diff --git a/exist-core/pom.xml b/exist-core/pom.xml index 3ced68c883..326f1f1220 100644 --- a/exist-core/pom.xml +++ b/exist-core/pom.xml @@ -740,6 +740,13 @@ src/main/java/org/exist/dom/memtree/reference/ProcessingInstructionReferenceImpl.java src/main/java/org/exist/dom/memtree/reference/TextReferenceImpl.java src/test/java/org/exist/http/urlrewrite/RedirectTest.java + src/main/java/org/exist/storage/io/AbstractVariableByteOutput.java + src/main/java/org/exist/storage/io/VariableByteArrayOutputStream.java + src/main/java/org/exist/storage/io/VariableByteBufferInput.java + src/main/java/org/exist/storage/io/VariableByteBufferOutput.java + src/main/java/org/exist/storage/io/VariableByteFilterInputStream.java + src/main/java/org/exist/storage/io/VariableByteFilterOutputStream.java + src/main/java/org/exist/storage/io/VariableByteOutput.java src/main/java/org/exist/util/ByteOrderMark.java src/main/java/org/exist/util/JREUtil.java src/main/java/org/exist/util/OSUtil.java @@ -824,6 +831,7 @@ src/main/java/org/exist/collections/Collection.java src/main/java/org/exist/collections/CollectionConfiguration.java src/main/java/org/exist/collections/CollectionConfigurationManager.java + src/main/java/org/exist/collections/LockedCollection.java src/main/java/org/exist/collections/MutableCollection.java src/main/java/org/exist/collections/triggers/CollectionTrigger.java src/main/java/org/exist/collections/triggers/DocumentTrigger.java @@ -832,6 +840,7 @@ src/main/java/org/exist/config/ConfigurationImpl.java src/main/java/org/exist/config/Configurator.java src/main/java/org/exist/dom/NodeListImpl.java + src/main/java/org/exist/dom/QName.java src/main/java/org/exist/dom/memtree/AbstractCharacterData.java src/main/java/org/exist/dom/memtree/AttrImpl.java src/main/java/org/exist/dom/memtree/DocumentImpl.java @@ -845,17 +854,22 @@ src/main/java/org/exist/dom/memtree/ProcessingInstructionImpl.java src/main/java/org/exist/dom/persistent/AbstractCharacterData.java src/main/java/org/exist/dom/persistent/AttrImpl.java + src/main/java/org/exist/dom/persistent/BinaryDocument.java src/main/java/org/exist/dom/persistent/CommentImpl.java src/main/java/org/exist/dom/persistent/DocumentImpl.java + src/main/java/org/exist/dom/persistent/DocumentMetadata.java + src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java src/main/java/org/exist/dom/persistent/DocumentSet.java src/main/java/org/exist/dom/persistent/ElementImpl.java src/main/java/org/exist/dom/persistent/NodeProxy.java src/test/java/org/exist/dom/persistent/NodeTest.java + src/main/java/org/exist/dom/persistent/LockToken.java src/test/java/org/exist/dom/persistent/PersistentDomTest.java src/main/java/org/exist/dom/persistent/ProcessingInstructionImpl.java src/main/java/org/exist/dom/persistent/SortedNodeSet.java src/main/java/org/exist/dom/persistent/StoredNode.java src/main/java/org/exist/dom/persistent/SymbolTable.java + src/test/java/org/exist/dom/persistent/SymbolTableTest.java src/main/java/org/exist/dom/persistent/TextImpl.java src/main/java/org/exist/dom/persistent/VirtualNodeSet.java src/main/java/org/exist/dom/persistent/XMLUtil.java @@ -871,6 +885,7 @@ src/main/java/org/exist/indexing/Index.java src/main/java/org/exist/indexing/IndexController.java src/main/java/org/exist/indexing/IndexManager.java + src/main/java/org/exist/indexing/ngram/NGramIndexWorker.java src/main/java/org/exist/jetty/JettyStart.java src/main/java/org/exist/jetty/ServerShutdown.java src/main/java/org/exist/jetty/WebAppContext.java @@ -892,6 +907,10 @@ src/main/java/org/exist/management/impl/DatabaseMXBean.java src/main/java/org/exist/management/impl/ExistMBean.java src/main/java/org/exist/management/impl/JMXAgent.java + src/main/java/org/exist/numbering/DLN.java + src/main/java/org/exist/numbering/DLNFactory.java + src/main/java/org/exist/numbering/NodeId.java + src/main/java/org/exist/numbering/NodeIdFactory.java src/main/java/org/exist/protocolhandler/xmldb/XmldbURL.java src/main/java/org/exist/protocolhandler/xmlrpc/XmlrpcUpload.java src/main/java/org/exist/repo/ClasspathHelper.java @@ -900,10 +919,15 @@ src/main/java/org/exist/scheduler/impl/QuartzSchedulerImpl.java src/main/java/org/exist/security/EffectiveSubject.java src/test/java/org/exist/security/FnDocSecurityTest.java + src/main/java/org/exist/security/Permission.java src/main/java/org/exist/security/SecurityManager.java src/main/java/org/exist/security/SimpleACLPermission.java + src/test/java/org/exist/security/SimpleACLPermissionTest.java + src/main/java/org/exist/security/UnixStylePermission.java + src/test/java/org/exist/security/UnixStylePermissionTest.java src/test/java/org/exist/security/XqueryApiTest.java src/main/java/org/exist/security/internal/AccountImpl.java + src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java src/main/java/org/exist/source/Source.java src/main/java/org/exist/source/SourceFactory.java src/test/java/org/exist/storage/BFileRecoverTest.java @@ -916,11 +940,20 @@ src/main/java/org/exist/storage/Indexable.java src/main/java/org/exist/storage/IndexSpec.java src/main/java/org/exist/storage/NativeBroker.java + src/main/java/org/exist/storage/NativeValueIndex.java src/main/java/org/exist/storage/ProcessMonitor.java + src/main/java/org/exist/storage/StorageAddress.java src/test/java/org/exist/storage/RecoveryTest.java src/test/java/org/exist/storage/RecoveryTest2.java src/test/java/org/exist/storage/btree/BTreeTest.java src/main/java/org/exist/storage/btree/TreeMetrics.java + src/main/java/org/exist/storage/index/BFile.java + src/main/java/org/exist/storage/io/AbstractVariableByteInput.java + src/main/java/org/exist/storage/io/VariableByteArrayInput.java + src/main/java/org/exist/storage/io/VariableByteInput.java + src/main/java/org/exist/storage/io/VariableByteInputToInputStream.java + src/main/java/org/exist/storage/io/VariableByteOutputToOutputStream.java + src/test/java/org/exist/storage/io/VariableByteStreamTest.java src/main/java/org/exist/storage/lock/FileLock.java src/main/java/org/exist/storage/recovery/RecoveryManager.java src/main/java/org/exist/storage/serializers/Serializer.java @@ -946,6 +979,7 @@ src/test/java/org/exist/util/serializer/json/JSONObjectTest.java src/main/java/org/exist/util/serializer/json/JSONSerializer.java src/test/java/org/exist/util/serializer/json/JSONWriterTest.java + src/test/java/org/exist/util/sorters/SortTestNodeId.java src/test/resources/org/exist/validation/catalog.xml src/test/java/org/exist/validation/CollectionConfigurationValidationModeTest.java src/main/java/org/exist/validation/resolver/SearchResourceResolver.java @@ -973,6 +1007,7 @@ src/main/java/org/exist/xquery/ErrorCodes.java src/test/java/org/exist/xquery/ForwardReferenceTest.java src/main/java/org/exist/xquery/FunctionFactory.java + src/main/java/org/exist/xquery/LocationStep.java src/main/java/org/exist/xquery/Optimizer.java src/main/java/org/exist/xquery/PerformanceStatsImpl.java src/main/java/org/exist/xquery/TryCatchExpression.java @@ -1020,8 +1055,30 @@ src/main/java/org/exist/xquery/util/ExpressionDumper.java src/main/java/org/exist/xquery/util/SerializerUtils.java src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java + src/main/java/org/exist/xquery/value/AnyURIValue.java src/test/java/org/exist/xquery/value/Base64BinaryValueTypeTest.java + src/main/java/org/exist/xquery/value/BinaryValue.java + src/main/java/org/exist/xquery/value/BooleanValue.java + src/main/java/org/exist/xquery/value/DateTimeStampValue.java + src/main/java/org/exist/xquery/value/DateTimeValue.java + src/main/java/org/exist/xquery/value/DateValue.java + src/main/java/org/exist/xquery/value/DayTimeDurationValue.java + src/main/java/org/exist/xquery/value/DecimalValue.java + src/main/java/org/exist/xquery/value/DoubleValue.java + src/main/java/org/exist/xquery/value/DurationValue.java + src/main/java/org/exist/xquery/value/FloatValue.java + src/main/java/org/exist/xquery/value/GDayValue.java + src/main/java/org/exist/xquery/value/GMonthDayValue.java + src/main/java/org/exist/xquery/value/GMonthValue.java + src/main/java/org/exist/xquery/value/GYearMonthValue.java + src/main/java/org/exist/xquery/value/GYearValue.java + src/main/java/org/exist/xquery/value/IntegerValue.java + src/main/java/org/exist/xquery/value/QNameValue.java + src/main/java/org/exist/xquery/value/StringValue.java + src/main/java/org/exist/xquery/value/TimeValue.java + src/main/java/org/exist/xquery/value/YearMonthDurationValue.java src/main/java/org/exist/xquery/value/SequenceType.java + src/main/java/org/exist/xquery/value/TimeUtils.java src/main/java/org/exist/xquery/value/Type.java src/main/java/org/exist/xslt/EXistURIResolver.java src/main/java/org/exist/xslt/XsltURIResolverHelper.java @@ -1110,6 +1167,7 @@ src/main/java/org/exist/collections/Collection.java src/main/java/org/exist/collections/CollectionConfiguration.java src/main/java/org/exist/collections/CollectionConfigurationManager.java + src/main/java/org/exist/collections/LockedCollection.java src/main/java/org/exist/collections/MutableCollection.java src/main/java/org/exist/collections/triggers/CollectionTrigger.java src/main/java/org/exist/collections/triggers/DocumentTrigger.java @@ -1118,6 +1176,7 @@ src/main/java/org/exist/config/ConfigurationImpl.java src/main/java/org/exist/config/Configurator.java src/main/java/org/exist/dom/NodeListImpl.java + src/main/java/org/exist/dom/QName.java src/main/java/org/exist/dom/memtree/AbstractCharacterData.java src/main/java/org/exist/dom/memtree/AttrImpl.java src/main/java/org/exist/dom/memtree/DocumentImpl.java @@ -1138,17 +1197,22 @@ src/main/java/org/exist/dom/memtree/reference/TextReferenceImpl.java src/main/java/org/exist/dom/persistent/AbstractCharacterData.java src/main/java/org/exist/dom/persistent/AttrImpl.java + src/main/java/org/exist/dom/persistent/BinaryDocument.java src/main/java/org/exist/dom/persistent/CommentImpl.java src/main/java/org/exist/dom/persistent/DocumentImpl.java + src/main/java/org/exist/dom/persistent/DocumentMetadata.java src/main/java/org/exist/dom/persistent/DocumentSet.java + src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java src/main/java/org/exist/dom/persistent/ElementImpl.java src/main/java/org/exist/dom/persistent/NodeProxy.java src/test/java/org/exist/dom/persistent/NodeTest.java + src/main/java/org/exist/dom/persistent/LockToken.java src/test/java/org/exist/dom/persistent/PersistentDomTest.java src/main/java/org/exist/dom/persistent/ProcessingInstructionImpl.java src/main/java/org/exist/dom/persistent/SortedNodeSet.java src/main/java/org/exist/dom/persistent/StoredNode.java src/main/java/org/exist/dom/persistent/SymbolTable.java + src/test/java/org/exist/dom/persistent/SymbolTableTest.java src/main/java/org/exist/dom/persistent/TextImpl.java src/main/java/org/exist/dom/persistent/VirtualNodeSet.java src/main/java/org/exist/dom/persistent/XMLDeclarationImpl.java @@ -1166,6 +1230,7 @@ src/main/java/org/exist/indexing/Index.java src/main/java/org/exist/indexing/IndexController.java src/main/java/org/exist/indexing/IndexManager.java + src/main/java/org/exist/indexing/ngram/NGramIndexWorker.java src/main/java/org/exist/jetty/JettyStart.java src/main/java/org/exist/jetty/ServerShutdown.java src/main/java/org/exist/jetty/WebAppContext.java @@ -1187,6 +1252,10 @@ src/main/java/org/exist/management/impl/DatabaseMXBean.java src/main/java/org/exist/management/impl/ExistMBean.java src/main/java/org/exist/management/impl/JMXAgent.java + src/main/java/org/exist/numbering/DLN.java + src/main/java/org/exist/numbering/DLNFactory.java + src/main/java/org/exist/numbering/NodeId.java + src/main/java/org/exist/numbering/NodeIdFactory.java src/main/java/org/exist/protocolhandler/xmldb/XmldbURL.java src/main/java/org/exist/protocolhandler/xmlrpc/XmlrpcUpload.java src/main/java/org/exist/repo/ClasspathHelper.java @@ -1197,10 +1266,15 @@ src/main/java/org/exist/scheduler/impl/QuartzSchedulerImpl.java src/main/java/org/exist/security/EffectiveSubject.java src/test/java/org/exist/security/FnDocSecurityTest.java + src/main/java/org/exist/security/Permission.java src/main/java/org/exist/security/SecurityManager.java src/main/java/org/exist/security/SimpleACLPermission.java + src/test/java/org/exist/security/SimpleACLPermissionTest.java + src/main/java/org/exist/security/UnixStylePermission.java + src/test/java/org/exist/security/UnixStylePermissionTest.java src/test/java/org/exist/security/XqueryApiTest.java src/main/java/org/exist/security/internal/AccountImpl.java + src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java src/main/java/org/exist/source/Source.java src/main/java/org/exist/source/SourceFactory.java src/test/java/org/exist/storage/AbstractRecoverTest.java @@ -1221,16 +1295,32 @@ src/main/java/org/exist/storage/IndexSpec.java src/test/java/org/exist/storage/MoveCollectionTest.java src/main/java/org/exist/storage/NativeBroker.java + src/main/java/org/exist/storage/NativeValueIndex.java src/main/java/org/exist/storage/ProcessMonitor.java src/test/java/org/exist/storage/RecoverBinaryTest.java src/test/java/org/exist/storage/RecoverXmlTest.java src/test/java/org/exist/storage/RecoveryTest.java src/test/java/org/exist/storage/RecoveryTest2.java + src/main/java/org/exist/storage/StorageAddress.java src/main/java/org/exist/storage/XQueryPool.java src/main/java/org/exist/storage/blob/** src/test/java/org/exist/storage/blob/** src/test/java/org/exist/storage/btree/BTreeTest.java src/main/java/org/exist/storage/btree/TreeMetrics.java + src/main/java/org/exist/storage/index/BFile.java + src/main/java/org/exist/storage/io/AbstractVariableByteInput.java + src/main/java/org/exist/storage/io/AbstractVariableByteOutput.java + src/main/java/org/exist/storage/io/VariableByteArrayInput.java + src/main/java/org/exist/storage/io/VariableByteArrayOutputStream.java + src/main/java/org/exist/storage/io/VariableByteBufferInput.java + src/main/java/org/exist/storage/io/VariableByteBufferOutput.java + src/main/java/org/exist/storage/io/VariableByteFilterInputStream.java + src/main/java/org/exist/storage/io/VariableByteFilterOutputStream.java + src/main/java/org/exist/storage/io/VariableByteInput.java + src/main/java/org/exist/storage/io/VariableByteInputToInputStream.java + src/main/java/org/exist/storage/io/VariableByteOutput.java + src/main/java/org/exist/storage/io/VariableByteOutputToOutputStream.java + src/test/java/org/exist/storage/io/VariableByteStreamTest.java src/test/java/org/exist/storage/journal/AbstractJournalTest.java src/test/java/org/exist/storage/journal/JournalBinaryTest.java src/main/java/org/exist/storage/journal/JournalManager.java @@ -1301,6 +1391,7 @@ src/test/java/org/exist/util/serializer/json/JSONObjectTest.java src/main/java/org/exist/util/serializer/json/JSONSerializer.java src/test/java/org/exist/util/serializer/json/JSONWriterTest.java + src/test/java/org/exist/util/sorters/SortTestNodeId.java src/test/resources/org/exist/validation/catalog.xml src/test/java/org/exist/validation/CollectionConfigurationValidationModeTest.java src/main/java/org/exist/validation/resolver/SearchResourceResolver.java @@ -1338,6 +1429,7 @@ src/main/java/org/exist/xquery/JavaBinding.java src/test/resources-filtered/org/exist/xquery/JavaBindingTest.conf.xml src/test/java/org/exist/xquery/JavaBindingTest.java + src/main/java/org/exist/xquery/LocationStep.java src/main/java/org/exist/xquery/Materializable.java src/main/java/org/exist/xquery/NameTest.java src/main/java/org/exist/xquery/Optimizer.java @@ -1405,18 +1497,40 @@ src/main/java/org/exist/xquery/util/SerializerUtils.java src/test/java/org/exist/xquery/util/URIUtilsTest.java src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java + src/main/java/org/exist/xquery/value/AnyURIValue.java src/main/java/org/exist/xquery/value/ArrayListValueSequence.java src/main/java/org/exist/xquery/value/AtomicValueComparator.java src/test/java/org/exist/xquery/value/Base64BinaryValueTypeTest.java src/test/java/org/exist/xquery/value/BifurcanMapTest.java + src/main/java/org/exist/xquery/value/BinaryValue.java + src/main/java/org/exist/xquery/value/BooleanValue.java + src/main/java/org/exist/xquery/value/DateTimeStampValue.java src/test/java/org/exist/xquery/value/DateTimeTypesTest.java + src/main/java/org/exist/xquery/value/DateTimeValue.java + src/main/java/org/exist/xquery/value/DateValue.java + src/main/java/org/exist/xquery/value/DayTimeDurationValue.java + src/main/java/org/exist/xquery/value/DecimalValue.java + src/main/java/org/exist/xquery/value/DoubleValue.java + src/main/java/org/exist/xquery/value/DurationValue.java + src/main/java/org/exist/xquery/value/FloatValue.java + src/main/java/org/exist/xquery/value/GDayValue.java + src/main/java/org/exist/xquery/value/GMonthDayValue.java + src/main/java/org/exist/xquery/value/GMonthValue.java + src/main/java/org/exist/xquery/value/GYearMonthValue.java + src/main/java/org/exist/xquery/value/GYearValue.java + src/main/java/org/exist/xquery/value/IntegerValue.java src/main/java/org/exist/xquery/value/ItemComparator.java + src/main/java/org/exist/xquery/value/QNameValue.java src/main/java/org/exist/xquery/value/SequenceComparator.java src/main/java/org/exist/xquery/value/SequenceType.java + src/main/java/org/exist/xquery/value/StringValue.java src/main/java/org/exist/xquery/value/SubSequence.java src/test/java/org/exist/xquery/value/SubSequenceRangeTest.java src/test/java/org/exist/xquery/value/SubSequenceTest.java + src/main/java/org/exist/xquery/value/TimeValue.java + src/main/java/org/exist/xquery/value/TimeUtils.java src/main/java/org/exist/xquery/value/Type.java + src/main/java/org/exist/xquery/value/YearMonthDurationValue.java src/main/java/org/exist/xslt/EXistURIResolver.java src/main/java/org/exist/xslt/XsltURIResolverHelper.java src/test/java/org/exist/xupdate/RemoveAppendTest.java diff --git a/exist-core/src/main/java/org/exist/collections/Collection.java b/exist-core/src/main/java/org/exist/collections/Collection.java index 0ffbb52cba..4e4c2b287f 100644 --- a/exist-core/src/main/java/org/exist/collections/Collection.java +++ b/exist-core/src/main/java/org/exist/collections/Collection.java @@ -54,7 +54,7 @@ import org.exist.security.SecurityManager; import org.exist.storage.*; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.storage.lock.*; import org.exist.storage.lock.Lock.LockMode; import org.exist.storage.txn.Txn; @@ -1207,7 +1207,7 @@ BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, BinaryDocumen * @throws IOException in case of I/O errors */ - @EnsureContainerLocked(mode=READ_LOCK) void serialize(final VariableByteOutputStream outputStream) throws IOException, LockException; + @EnsureContainerLocked(mode=READ_LOCK) void serialize(final VariableByteOutput outputStream) throws IOException, LockException; @Override void close(); diff --git a/exist-core/src/main/java/org/exist/collections/LockedCollection.java b/exist-core/src/main/java/org/exist/collections/LockedCollection.java index 10138374e3..758af4957b 100644 --- a/exist-core/src/main/java/org/exist/collections/LockedCollection.java +++ b/exist-core/src/main/java/org/exist/collections/LockedCollection.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -29,7 +53,7 @@ import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; import org.exist.storage.*; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.storage.lock.Lock; import org.exist.storage.lock.LockedDocumentMap; import org.exist.storage.lock.ManagedCollectionLock; @@ -473,7 +497,7 @@ public BinaryDocument addBinaryResource(final Txn transaction, final DBBroker br } @Override - public void serialize(final VariableByteOutputStream outputStream) throws IOException, LockException { + public void serialize(final VariableByteOutput outputStream) throws IOException, LockException { collection.serialize(outputStream); } diff --git a/exist-core/src/main/java/org/exist/collections/MutableCollection.java b/exist-core/src/main/java/org/exist/collections/MutableCollection.java index 1dcf59b202..81b51376a9 100644 --- a/exist-core/src/main/java/org/exist/collections/MutableCollection.java +++ b/exist-core/src/main/java/org/exist/collections/MutableCollection.java @@ -72,7 +72,7 @@ import org.exist.security.Subject; import org.exist.storage.*; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.storage.lock.*; import org.exist.storage.lock.Lock.LockMode; import org.exist.storage.lock.Lock.LockType; @@ -195,7 +195,7 @@ private MutableCollection(final DBBroker broker, final int collectionId, /** * Deserializes a Collection object * - * Counterpart method to {@link #serialize(VariableByteOutputStream)} + * Counterpart method to {@link #serialize(VariableByteOutput)} * * @param broker The database broker * @param path The path of the Collection @@ -884,7 +884,7 @@ public Iterator iteratorNoLock(final DBBroker broker) throws Permi * @param outputStream The output stream to write the collection contents to */ @Override - public void serialize(final VariableByteOutputStream outputStream) throws IOException, LockException { + public void serialize(final VariableByteOutput outputStream) throws IOException, LockException { outputStream.writeInt(collectionId); final int size; @@ -914,7 +914,7 @@ public void close() { /** * Read collection contents from the stream * - * Counterpart method to {@link #serialize(VariableByteOutputStream)} + * Counterpart method to {@link #serialize(VariableByteOutput)} * * @param broker The database broker * @param path The path of the Collection diff --git a/exist-core/src/main/java/org/exist/dom/QName.java b/exist-core/src/main/java/org/exist/dom/QName.java index 3ff7753c66..ef73420d91 100644 --- a/exist-core/src/main/java/org/exist/dom/QName.java +++ b/exist-core/src/main/java/org/exist/dom/QName.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -26,16 +50,19 @@ import org.exist.util.XMLNames; import org.exist.xquery.Constants; +import javax.annotation.Nullable; import javax.xml.XMLConstants; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.exist.dom.QName.Validity.*; +import static org.exist.util.StringUtil.isNullOrEmpty; /** * Represents a QName, consisting of a local name, a namespace URI and a prefix. * * @author Wolfgang + * @author Adam Retter */ public class QName implements Comparable { @@ -134,7 +161,18 @@ public byte getNameType() { * @return the string representation of this qualified name. * */ public String getStringValue() { - return getStringRepresentation(false); + return getStringRepresentation(false, false); + } + + /** + * Get an extended string representation of this qualified name. + * + * Will be of the format `local-name`, `{namespace}local-name`, or `{namespace}prefix:local-name`. + * + * @return the string representation of this qualified name. + */ + public String getExtendedStringValue() { + return getStringRepresentation(false, true); } /** @@ -146,23 +184,32 @@ public String getStringValue() { */ @Override public String toString() { - return getStringRepresentation(true); + return getStringRepresentation(true, false); } /** * Get a string representation of this qualified name. * * @param showNsWithoutPrefix true if the namespace should be shown even when there is no prefix, false otherwise. - * When shown, it will be output using Clark notation, e.g. `{http://namespace}local-name`. + * When shown, it will be output using Clark notation, e.g. `{namespace}local-name`. + * + * @param extended true if the namespace and prefix should be shown, requires showNsWithoutPrefix == false. * * @return the string representation of this qualified name. */ - private String getStringRepresentation(final boolean showNsWithoutPrefix) { + private String getStringRepresentation(final boolean showNsWithoutPrefix, final boolean extended) { if (prefix != null && !prefix.isEmpty()) { - return prefix + COLON + localPart; - } else if (showNsWithoutPrefix && namespaceURI != null && !XMLConstants.NULL_NS_URI.equals(namespaceURI)) { + if (extended) { + return LEFT_BRACE + namespaceURI + RIGHT_BRACE + prefix + COLON + localPart; + } else { + return prefix + COLON + localPart; + } + } + + if (showNsWithoutPrefix && namespaceURI != null && !XMLConstants.NULL_NS_URI.equals(namespaceURI)) { return LEFT_BRACE + namespaceURI + RIGHT_BRACE + localPart; } + return localPart; } @@ -330,6 +377,52 @@ public static QName parse(final String namespaceURI, final String qname) throws return new QName(qname.substring(p + 1), namespaceURI, qname.substring(0, p)); } + /** + * Extract a QName from a namespace and qualified name string. + * + * @param extendedStringValue a string representation as produced by {@link #getExtendedStringValue()}, i.e.: `local-name`, `{namespace}local-name`, or `{namespace}prefix:local-name`. + * @return The QName + * @throws IllegalQNameException if the qname component is invalid + */ + public static QName parse(String extendedStringValue) throws IllegalQNameException { + if (isNullOrEmpty(extendedStringValue)) { + throw new IllegalQNameException(ILLEGAL_FORMAT.val, "Illegal extended string QName is empty"); + } + + final String namespaceUri; + if (extendedStringValue.charAt(0) == LEFT_BRACE) { + final int idxNsEnd = extendedStringValue.indexOf(RIGHT_BRACE); + if (idxNsEnd == Constants.STRING_NOT_FOUND) { + throw new IllegalQNameException(ILLEGAL_FORMAT.val, "Illegal extended string QName, missing right brace: '" + extendedStringValue + "'"); + } + namespaceUri = extendedStringValue.substring(1, idxNsEnd); + extendedStringValue = extendedStringValue.substring(idxNsEnd + 1); + } else if (extendedStringValue.indexOf(RIGHT_BRACE) != Constants.STRING_NOT_FOUND) { + throw new IllegalQNameException(ILLEGAL_FORMAT.val, "Illegal extended string QName, missing left brace: '" + extendedStringValue + "'"); + } else { + namespaceUri = XMLConstants.NULL_NS_URI; + } + + @Nullable final String prefix; + final int idxColon = extendedStringValue.indexOf(COLON); + if (idxColon == Constants.STRING_NOT_FOUND) { + prefix = null; + } else { + prefix = extendedStringValue.substring(0, idxColon); + if (!XMLNames.isNCName(prefix)) { + throw new IllegalQNameException(INVALID_PREFIX.val, "Illegal extended string QName, invalid prefix: '" + extendedStringValue + "'"); + } + extendedStringValue = extendedStringValue.substring(idxColon + 1); + } + + final String localPart = extendedStringValue; + if (!XMLNames.isNCName(localPart)) { + throw new IllegalQNameException(INVALID_LOCAL_PART.val, "Illegal extended string QName, invalid prefix: '" + extendedStringValue + "'"); + } + + return new QName(localPart, namespaceUri, prefix); + } + /** * Parses the given string into a QName. The method uses context to look up * a namespace URI for an existing prefix. diff --git a/exist-core/src/main/java/org/exist/dom/persistent/BinaryDocument.java b/exist-core/src/main/java/org/exist/dom/persistent/BinaryDocument.java index f9c12b3666..be14a09b74 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/BinaryDocument.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/BinaryDocument.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -27,7 +51,7 @@ import org.exist.storage.BrokerPool; import org.exist.storage.blob.BlobId; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.xmldb.XmldbURI; import org.exist.xquery.Expression; import org.w3c.dom.DocumentType; @@ -167,7 +191,7 @@ public void setBlobId(final BlobId blobId) { } @Override - public void write(final VariableByteOutputStream ostream) throws IOException { + public void write(final VariableByteOutput ostream) throws IOException { ostream.writeInt(getDocId()); ostream.writeUTF(getFileURI().toString()); diff --git a/exist-core/src/main/java/org/exist/dom/persistent/DocumentImpl.java b/exist-core/src/main/java/org/exist/dom/persistent/DocumentImpl.java index 9489b17295..4da2b04a92 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/DocumentImpl.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/DocumentImpl.java @@ -59,7 +59,7 @@ import org.exist.security.*; import org.exist.storage.*; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.storage.lock.EnsureContainerLocked; import org.exist.storage.lock.EnsureLocked; import org.exist.storage.txn.Txn; @@ -859,11 +859,11 @@ public void appendChild(final NodeHandle child) throws DOMException { /** * The method write * - * @param ostream a VariableByteOutputStream value + * @param ostream a VariableByteOutput value * @throws IOException if an error occurs */ @EnsureContainerLocked(mode=READ_LOCK) - public void write(final VariableByteOutputStream ostream) throws IOException { + public void write(final VariableByteOutput ostream) throws IOException { try { ostream.writeInt(docId); ostream.writeUTF(fileURI.toString()); @@ -885,7 +885,7 @@ public void write(final VariableByteOutputStream ostream) throws IOException { } } - void writeDocumentAttributes(final SymbolTable symbolTable, final VariableByteOutputStream ostream) throws IOException { + void writeDocumentAttributes(final SymbolTable symbolTable, final VariableByteOutput ostream) throws IOException { ostream.writeLong(created); ostream.writeLong(lastModified); ostream.writeInt(symbolTable.getMimeTypeId(mimeType)); diff --git a/exist-core/src/main/java/org/exist/dom/persistent/DocumentMetadata.java b/exist-core/src/main/java/org/exist/dom/persistent/DocumentMetadata.java index 77bdb4bd2c..38fbe79c54 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/DocumentMetadata.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/DocumentMetadata.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,7 +47,7 @@ import org.exist.ResourceMetadata; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.w3c.dom.DocumentType; import java.io.IOException; @@ -100,7 +124,7 @@ public void decPageCount() { } @Deprecated - public void write(final SymbolTable symbolTable, final VariableByteOutputStream ostream) throws IOException { + public void write(final SymbolTable symbolTable, final VariableByteOutput ostream) throws IOException { doc.writeDocumentAttributes(symbolTable, ostream); } diff --git a/exist-core/src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java b/exist-core/src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java index 4fb48d3f7e..e1bfb1d470 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,7 +47,7 @@ import net.jcip.annotations.ThreadSafe; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.xquery.Expression; import org.w3c.dom.DocumentType; import org.w3c.dom.NamedNodeMap; @@ -79,7 +103,7 @@ public String getInternalSubset() { return null; } - protected void write(final VariableByteOutputStream ostream) throws IOException { + protected void write(final VariableByteOutput ostream) throws IOException { ostream.writeUTF(name); ostream.writeUTF(systemId != null ? systemId : ""); ostream.writeUTF(publicId != null ? publicId : ""); diff --git a/exist-core/src/main/java/org/exist/dom/persistent/ElementImpl.java b/exist-core/src/main/java/org/exist/dom/persistent/ElementImpl.java index 10514bc8ee..c58e77d777 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/ElementImpl.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/ElementImpl.java @@ -75,6 +75,7 @@ import org.exist.xquery.value.StringValue; import org.w3c.dom.*; +import javax.annotation.Nullable; import javax.xml.XMLConstants; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; @@ -1384,6 +1385,10 @@ public String getNamespaceForPrefix(final String prefix) { return namespaceMappings.get(prefix); } + public @Nullable Map getNamespaceMappings() { + return namespaceMappings; + } + /** * @see java.lang.Object#toString() */ diff --git a/exist-core/src/main/java/org/exist/dom/persistent/LockToken.java b/exist-core/src/main/java/org/exist/dom/persistent/LockToken.java index 7b9d60f926..57f5bd413d 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/LockToken.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/LockToken.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,7 +46,7 @@ package org.exist.dom.persistent; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.util.UUIDGenerator; import javax.annotation.Nullable; @@ -123,7 +147,8 @@ public static String generateUUID() { return UUIDGenerator.getUUID(); } - public void write(final VariableByteOutputStream ostream) throws IOException { + public void write(final VariableByteOutput + ostream) throws IOException { // TODO(AR) these 3 bytes could be encoded into 1 ostream.writeByte(type.getValue()); ostream.writeByte(depth.getValue()); diff --git a/exist-core/src/main/java/org/exist/dom/persistent/SymbolTable.java b/exist-core/src/main/java/org/exist/dom/persistent/SymbolTable.java index ba29a2df7b..5e536a0781 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/SymbolTable.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/SymbolTable.java @@ -52,10 +52,16 @@ import org.exist.EXistException; import org.exist.backup.RawDataBackup; import org.exist.dom.QName; -import org.exist.storage.*; + +import org.exist.storage.BrokerPool; +import org.exist.storage.BrokerPoolService; +import org.exist.storage.BrokerPoolServiceException; +import org.exist.storage.DBBroker; +import org.exist.storage.ElementValue; +import org.exist.storage.io.VariableByteFilterInputStream; +import org.exist.storage.io.VariableByteFilterOutputStream; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteInputStream; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.util.Configuration; import org.exist.util.FileUtils; import org.w3c.dom.Attr; @@ -135,8 +141,7 @@ public static SymbolType valueOf(final byte typeId) { * the underlying symbols.dbx file */ private Path file; - private final VariableByteOutputStream outBuffer = new VariableByteOutputStream(256); - private OutputStream os = null; + private VariableByteFilterOutputStream os = null; @Override public void configure(final Configuration configuration) { @@ -287,7 +292,7 @@ public synchronized String getNamespace(final short id) { * @param os outputstream * @throws IOException in response to an IO error */ - private synchronized void writeAll(final VariableByteOutputStream os) throws IOException { + private synchronized void writeAll(final VariableByteOutput os) throws IOException { os.writeFixedInt(FILE_FORMAT_VERSION_ID); localNameSymbols.write(os); namespaceSymbols.write(os); @@ -383,10 +388,8 @@ public final Path getFile() { * @throws EXistException in response to the error */ private void saveSymbols() throws EXistException { - try(final VariableByteOutputStream os = new VariableByteOutputStream(8192); - final OutputStream fos = new BufferedOutputStream(Files.newOutputStream(getFile()))) { + try (final VariableByteFilterOutputStream os = new VariableByteFilterOutputStream(new BufferedOutputStream(Files.newOutputStream(getFile())))) { writeAll(os); - fos.write(os.toByteArray()); } catch(final FileNotFoundException e) { throw new EXistException("File not found: " + this.getFile().toAbsolutePath().toString(), e); } catch(final IOException e) { @@ -402,9 +405,8 @@ private void saveSymbols() throws EXistException { * @throws EXistException in response to the error */ private synchronized void loadSymbols() throws EXistException { - try(final InputStream fis = new BufferedInputStream(Files.newInputStream(getFile()))) { + try (final VariableByteFilterInputStream is = new VariableByteFilterInputStream(new BufferedInputStream(Files.newInputStream(getFile())))) { - final VariableByteInput is = new VariableByteInputStream(fis); final int magic = is.readFixedInt(); if(magic == LEGACY_FILE_FORMAT_VERSION_ID) { LOG.info("Converting legacy symbols.dbx to new format..."); @@ -444,16 +446,15 @@ public void flush() throws EXistException { //Noting to do ? -pb } - private OutputStream getOutputStream() throws IOException { - if(os == null) { - os = new BufferedOutputStream(Files.newOutputStream(getFile(), StandardOpenOption.APPEND)); + private VariableByteFilterOutputStream getOutputStream() throws IOException { + if (os == null) { + os = new VariableByteFilterOutputStream(new BufferedOutputStream(Files.newOutputStream(getFile(), StandardOpenOption.APPEND))); } return os; } @Override public void close() throws IOException { - outBuffer.close(); if(os != null) { os.close(); } @@ -551,7 +552,7 @@ public synchronized int getId(final String name) { return id; } - protected final void write(final VariableByteOutputStream os) throws IOException { + protected final void write(final VariableByteOutput os) throws IOException { for (final String symbol : symbolsByName.keySet()) { final int id = symbolsByName.getInt(symbol); if (id < 0) { @@ -564,10 +565,8 @@ protected final void write(final VariableByteOutputStream os) throws IOException // Append a new entry to the .dbx file private void write(final int id, final String key) { - outBuffer.clear(); try { - writeEntry(id, key, outBuffer); - getOutputStream().write(outBuffer.toByteArray()); + writeEntry(id, key, getOutputStream()); getOutputStream().flush(); } catch(final FileNotFoundException e) { LOG.error("Symbol table: file not found!", e); @@ -578,7 +577,7 @@ private void write(final int id, final String key) { } } - private void writeEntry(final int id, final String key, final VariableByteOutputStream os) throws IOException { + private void writeEntry(final int id, final String key, final VariableByteOutput os) throws IOException { os.writeByte(getSymbolType().getTypeId()); os.writeInt(id); os.writeUTF(key); diff --git a/exist-core/src/main/java/org/exist/dom/persistent/XMLDeclarationImpl.java b/exist-core/src/main/java/org/exist/dom/persistent/XMLDeclarationImpl.java index 69804b88a4..3ba9a1e27b 100644 --- a/exist-core/src/main/java/org/exist/dom/persistent/XMLDeclarationImpl.java +++ b/exist-core/src/main/java/org/exist/dom/persistent/XMLDeclarationImpl.java @@ -33,7 +33,7 @@ package org.exist.dom.persistent; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import javax.annotation.Nullable; import java.io.IOException; @@ -93,7 +93,7 @@ public String getStandalone() { * * @throws IOException if an error occurs whilst writing to the output stream. */ - public void write(final VariableByteOutputStream ostream) throws IOException { + public void write(final VariableByteOutput ostream) throws IOException { ostream.writeUTF(version != null ? version : ""); ostream.writeUTF(encoding != null ? encoding : ""); ostream.writeUTF(standalone != null ? standalone : ""); diff --git a/exist-core/src/main/java/org/exist/numbering/DLN.java b/exist-core/src/main/java/org/exist/numbering/DLN.java index 8acad1bc27..023498f23d 100644 --- a/exist-core/src/main/java/org/exist/numbering/DLN.java +++ b/exist-core/src/main/java/org/exist/numbering/DLN.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,7 +48,7 @@ import java.io.IOException; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; /** * Represents a node id in the form of a dynamic level number (DLN). DLN's are @@ -110,7 +134,7 @@ public DLN(final int units, final byte[] data, final int startOffset) { /** * Reads a DLN from the given {@link VariableByteInput} stream. * - * @see #write(VariableByteOutputStream) + * @see #write(VariableByteOutput) * @param bitCnt total number of bits to read * @param is the input stream to read from * @throws IOException in case of an error reading the DLN @@ -358,19 +382,19 @@ public boolean before(final NodeId other, final boolean isPreceding) { } /** - * Write the node id to a {@link VariableByteOutputStream}. + * Write the node id to a {@link VariableByteOutput}. * * @param os the output stream to write to * @throws IOException in case of write error */ @Override - public void write(final VariableByteOutputStream os) throws IOException { + public void write(final VariableByteOutput os) throws IOException { os.writeShort((short) units()); os.write(bits, 0, bits.length); } @Override - public NodeId write(final NodeId prevId, final VariableByteOutputStream os) throws IOException { + public NodeId write(final NodeId prevId, final VariableByteOutput os) throws IOException { int i = 0; if(prevId != null) { final DLN previous = (DLN) prevId; diff --git a/exist-core/src/main/java/org/exist/numbering/DLNFactory.java b/exist-core/src/main/java/org/exist/numbering/DLNFactory.java index 9e7ebac060..a7be6d4ab2 100644 --- a/exist-core/src/main/java/org/exist/numbering/DLNFactory.java +++ b/exist-core/src/main/java/org/exist/numbering/DLNFactory.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,7 +48,7 @@ import java.io.IOException; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; /** * Implementation of {@link NodeIdFactory} for DLN-based @@ -71,7 +95,7 @@ public int lengthInBytes(final int units, final byte[] data, final int startOffs return DLNBase.getLengthInBytes(units, data, startOffset); } - public void writeEndOfDocument(final VariableByteOutputStream os) { + public void writeEndOfDocument(final VariableByteOutput os) throws IOException { os.writeByte((byte) 0); os.writeShort(0); } diff --git a/exist-core/src/main/java/org/exist/numbering/NodeId.java b/exist-core/src/main/java/org/exist/numbering/NodeId.java index e3dda63bb3..3fd30ebfa1 100644 --- a/exist-core/src/main/java/org/exist/numbering/NodeId.java +++ b/exist-core/src/main/java/org/exist/numbering/NodeId.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,7 +45,7 @@ */ package org.exist.numbering; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import java.io.IOException; @@ -197,15 +221,15 @@ public interface NodeId extends Comparable { void serialize(byte[] data, int offset); /** - * Write the node id to a {@link org.exist.storage.io.VariableByteOutputStream}. + * Write the node id to a {@link VariableByteOutput}. * * @param os the output stream * @throws java.io.IOException if there's a problem with the underlying output stream */ - void write(VariableByteOutputStream os) throws IOException; + void write(VariableByteOutput os) throws IOException; /** - * Write the node id to a {@link org.exist.storage.io.VariableByteOutputStream}. To save + * Write the node id to a {@link VariableByteOutput}. To save * storage space, only store those byte which are different from the previous node id. * * @param previous the node id previously written or null @@ -213,6 +237,6 @@ public interface NodeId extends Comparable { * @return this node id * @throws IOException if there's a problem with the underlying output stream */ - NodeId write(NodeId previous, VariableByteOutputStream os) throws IOException; + NodeId write(NodeId previous, VariableByteOutput os) throws IOException; } diff --git a/exist-core/src/main/java/org/exist/numbering/NodeIdFactory.java b/exist-core/src/main/java/org/exist/numbering/NodeIdFactory.java index b3fa62bf22..b9f2e0b0f7 100644 --- a/exist-core/src/main/java/org/exist/numbering/NodeIdFactory.java +++ b/exist-core/src/main/java/org/exist/numbering/NodeIdFactory.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,7 +48,7 @@ import java.io.IOException; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; /** * A factory for creating node ids. To support different numbering @@ -56,7 +80,7 @@ public interface NodeIdFactory { /** * Read a NodeId from the given input stream. * - * @see NodeId#write(org.exist.storage.io.VariableByteOutputStream) + * @see NodeId#write(VariableByteOutput) * * @param is the input stream to read from * @return the NodeId read @@ -69,7 +93,7 @@ public interface NodeIdFactory { * stored with prefix-compression, i.e. only the bytes differing from the previous * node were written out. * - * @see NodeId#write(NodeId, org.exist.storage.io.VariableByteOutputStream) + * @see NodeId#write(NodeId, VariableByteOutput) * * @param previous the previous node id read or null if there is none * @param is the input stream to read from @@ -121,6 +145,6 @@ public interface NodeIdFactory { */ NodeId documentNodeId(); - void writeEndOfDocument(VariableByteOutputStream os); + void writeEndOfDocument(VariableByteOutput os) throws IOException; } \ No newline at end of file diff --git a/exist-core/src/main/java/org/exist/security/Permission.java b/exist-core/src/main/java/org/exist/security/Permission.java index 5be941e751..5dd869743d 100644 --- a/exist-core/src/main/java/org/exist/security/Permission.java +++ b/exist-core/src/main/java/org/exist/security/Permission.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,7 +47,7 @@ import java.io.IOException; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.util.SyntaxException; @@ -242,7 +266,7 @@ public interface Permission { */ boolean validate(Subject user, int mode); - void write(VariableByteOutputStream ostream) throws IOException; + void write(VariableByteOutput ostream) throws IOException; void read(VariableByteInput istream) throws IOException; diff --git a/exist-core/src/main/java/org/exist/security/SimpleACLPermission.java b/exist-core/src/main/java/org/exist/security/SimpleACLPermission.java index ade15afef0..6ec73af4be 100644 --- a/exist-core/src/main/java/org/exist/security/SimpleACLPermission.java +++ b/exist-core/src/main/java/org/exist/security/SimpleACLPermission.java @@ -49,7 +49,7 @@ import java.util.Arrays; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import javax.annotation.Nullable; @@ -375,7 +375,7 @@ public void read(final VariableByteInput istream) throws IOException { } @Override - public void write(final VariableByteOutputStream ostream) throws IOException { + public void write(final VariableByteOutput ostream) throws IOException { super.write(ostream); ostream.write(acl.length); for (final int ace : acl) { diff --git a/exist-core/src/main/java/org/exist/security/UnixStylePermission.java b/exist-core/src/main/java/org/exist/security/UnixStylePermission.java index 7f2c578d9a..4dd2799d54 100644 --- a/exist-core/src/main/java/org/exist/security/UnixStylePermission.java +++ b/exist-core/src/main/java/org/exist/security/UnixStylePermission.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -28,7 +52,7 @@ import org.exist.security.internal.RealmImpl; import org.exist.storage.DBBroker; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import static org.exist.security.PermissionRequired.*; @@ -448,7 +472,7 @@ public void read(final VariableByteInput istream) throws IOException { } @Override - public void write(final VariableByteOutputStream ostream) throws IOException { + public void write(final VariableByteOutput ostream) throws IOException { ostream.writeLong(vector); } diff --git a/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java b/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java index bdef67b31a..23a5505ed5 100644 --- a/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java +++ b/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -27,7 +51,7 @@ import org.exist.security.SecurityManager; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.util.SyntaxException; /** @@ -350,7 +374,7 @@ public void setOwner(final int id) { } @Override - public void write(final VariableByteOutputStream ostream) { + public void write(final VariableByteOutput ostream) { throw new UnsupportedOperationException("Serialization of permission Aider is unsupported"); } diff --git a/exist-core/src/main/java/org/exist/storage/NativeBroker.java b/exist-core/src/main/java/org/exist/storage/NativeBroker.java index 9e53bb4128..947c1dea43 100644 --- a/exist-core/src/main/java/org/exist/storage/NativeBroker.java +++ b/exist-core/src/main/java/org/exist/storage/NativeBroker.java @@ -79,8 +79,8 @@ import org.exist.storage.dom.RawNodeIterator; import org.exist.storage.index.BFile; import org.exist.storage.index.CollectionStore; +import org.exist.storage.io.VariableByteArrayOutputStream; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; import org.exist.storage.lock.*; import org.exist.storage.lock.Lock.LockMode; import org.exist.storage.lock.Lock.LockType; @@ -2010,7 +2010,7 @@ public void saveCollection(final Txn transaction, final Collection collection) t try(final ManagedLock collectionsDbLock = lockManager.acquireBtreeWriteLock(collectionsDb.getLockName())) { final Value name = new CollectionStore.CollectionKey(collection.getURI().toString()); - try(final VariableByteOutputStream os = new VariableByteOutputStream(256)) { + try(final VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(256)) { collection.serialize(os); final long address = collectionsDb.put(transaction, name, os.data(), true); if (address == BFile.UNKNOWN_ADDRESS) { @@ -2326,8 +2326,8 @@ public void storeDocument(final Txn transaction, final XmldbURI name, final Node */ @Override public void storeXMLResource(final Txn transaction, final DocumentImpl doc) { - try(final VariableByteOutputStream os = new VariableByteOutputStream(256); - final ManagedLock collectionsDbLock = lockManager.acquireBtreeWriteLock(collectionsDb.getLockName())) { + try(final VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(256); + final ManagedLock collectionsDbLock = lockManager.acquireBtreeWriteLock(collectionsDb.getLockName())) { doc.write(os); final Value key = new CollectionStore.DocumentKey(doc.getCollection().getId(), doc.getResourceType(), doc.getDocId()); collectionsDb.put(transaction, key, os.data(), true); diff --git a/exist-core/src/main/java/org/exist/storage/NativeValueIndex.java b/exist-core/src/main/java/org/exist/storage/NativeValueIndex.java index 9bd4552f8b..f661cf2052 100644 --- a/exist-core/src/main/java/org/exist/storage/NativeValueIndex.java +++ b/exist-core/src/main/java/org/exist/storage/NativeValueIndex.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -43,8 +67,8 @@ import org.exist.storage.btree.Value; import org.exist.storage.index.BFile; import org.exist.storage.io.VariableByteArrayInput; +import org.exist.storage.io.VariableByteArrayOutputStream; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; import org.exist.storage.lock.LockManager; import org.exist.storage.lock.ManagedLock; import org.exist.storage.txn.Txn; @@ -169,7 +193,7 @@ public class NativeValueIndex implements ContentLoadingObserver { /** * Work output Stream that should be cleared before every use. */ - private VariableByteOutputStream os = new VariableByteOutputStream(); + private VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(); private final boolean caseSensitive; @@ -370,65 +394,69 @@ public void flush() { } private void flush(final PendingChanges pending, final FunctionE dbKeyFn) { - final VariableByteOutputStream nodeIdOs = new VariableByteOutputStream(); + try (final VariableByteArrayOutputStream nodeIdOs = new VariableByteArrayOutputStream()) { - for (final Map.Entry> entry : pending.changes.entrySet()) { - final T key = entry.getKey(); + for (final Map.Entry> entry : pending.changes.entrySet()) { + final T key = entry.getKey(); - final List gids = entry.getValue(); - final int gidsCount = gids.size(); + final List gids = entry.getValue(); + final int gidsCount = gids.size(); - //Don't forget this one - FastQSort.sort(gids, 0, gidsCount - 1); - os.clear(); - os.writeInt(this.doc.getDocId()); - os.writeInt(gidsCount); + //Don't forget this one + FastQSort.sort(gids, 0, gidsCount - 1); + os.clear(); + try { + os.writeInt(this.doc.getDocId()); + os.writeInt(gidsCount); - //Compute the GID list - try { - NodeId previous = null; - for (final NodeId nodeId : gids) { - previous = nodeId.write(previous, nodeIdOs); - } + //Compute the GID list + NodeId previous = null; + for (final NodeId nodeId : gids) { + previous = nodeId.write(previous, nodeIdOs); + } - final byte[] nodeIdsData = nodeIdOs.toByteArray(); + final byte[] nodeIdsData = nodeIdOs.toByteArray(); - // clear the buf for the next iteration - nodeIdOs.clear(); + // clear the buf for the next iteration + nodeIdOs.clear(); - // Write length of node IDs (bytes) - os.writeFixedInt(nodeIdsData.length); + // Write length of node IDs (bytes) + os.writeFixedInt(nodeIdsData.length); - // write the node IDs - os.write(nodeIdsData); + // write the node IDs + os.write(nodeIdsData); - } catch (final IOException e) { - LOG.warn("IO error while writing range index: {}", e.getMessage(), e); - //TODO : throw exception? - } + } catch (final IOException e) { + LOG.warn("IO error while writing range index: {}", e.getMessage(), e); + //TODO : throw exception? + } - try(final ManagedLock bfileLock = lockManager.acquireBtreeWriteLock(dbValues.getLockName())) { - final Value v = dbKeyFn.apply(key); + try (final ManagedLock bfileLock = lockManager.acquireBtreeWriteLock(dbValues.getLockName())) { + final Value v = dbKeyFn.apply(key); - if (dbValues.append(v, os.data()) == BFile.UNKNOWN_ADDRESS) { - LOG.warn("Could not append index data for key '{}'", key); - //TODO : throw exception ? + if (dbValues.append(v, os.data()) == BFile.UNKNOWN_ADDRESS) { + LOG.warn("Could not append index data for key '{}'", key); + //TODO : throw exception ? + } + } catch (final EXistException | IOException e) { + LOG.error(e.getMessage(), e); + } catch (final LockException e) { + LOG.warn("Failed to acquire lock for '{}'", FileUtils.fileName(dbValues.getFile()), e); + //TODO : return ? + } catch (final ReadOnlyException e) { + LOG.warn(e.getMessage(), e); + + //Return without clearing the pending entries + return; + } finally { + os.clear(); } - } catch (final EXistException | IOException e) { - LOG.error(e.getMessage(), e); - } catch (final LockException e) { - LOG.warn("Failed to acquire lock for '{}'", FileUtils.fileName(dbValues.getFile()), e); - //TODO : return ? - } catch (final ReadOnlyException e) { - LOG.warn(e.getMessage(), e); - - //Return without clearing the pending entries - return; - } finally { - os.clear(); } + pending.changes.clear(); + } catch (final IOException e) { + LOG.warn("IO error while writing range index: {}", e.getMessage(), e); + //TODO : throw exception? } - pending.changes.clear(); } @Override @@ -444,112 +472,116 @@ public void remove() { } private void remove(final PendingChanges pending, final FunctionE dbKeyFn) { - final VariableByteOutputStream nodeIdOs = new VariableByteOutputStream(); - for (final Map.Entry> entry : pending.changes.entrySet()) { - final T key = entry.getKey(); - final List storedGIDList = entry.getValue(); - final List newGIDList = new ArrayList<>(); - os.clear(); + try (final VariableByteArrayOutputStream nodeIdOs = new VariableByteArrayOutputStream()) { + for (final Map.Entry> entry : pending.changes.entrySet()) { + final T key = entry.getKey(); + final List storedGIDList = entry.getValue(); + final List newGIDList = new ArrayList<>(); + os.clear(); - try(final ManagedLock bfileLock = lockManager.acquireBtreeWriteLock(dbValues.getLockName())) { + try (final ManagedLock bfileLock = lockManager.acquireBtreeWriteLock(dbValues.getLockName())) { - //Compute a key for the value - final Value searchKey = dbKeyFn.apply(key); - final Value value = dbValues.get(searchKey); + //Compute a key for the value + final Value searchKey = dbKeyFn.apply(key); + final Value value = dbValues.get(searchKey); - //Does the value already has data in the index ? - if (value != null) { + //Does the value already has data in the index ? + if (value != null) { - //Add its data to the new list - final VariableByteArrayInput is = new VariableByteArrayInput(value.getData()); + //Add its data to the new list + final VariableByteArrayInput is = new VariableByteArrayInput(value.getData()); - while (is.available() > 0) { - final int storedDocId = is.readInt(); - final int gidsCount = is.readInt(); - final int size = is.readFixedInt(); + while (is.available() > 0) { + final int storedDocId = is.readInt(); + final int gidsCount = is.readInt(); + final int size = is.readFixedInt(); - if (storedDocId != this.doc.getDocId()) { + if (storedDocId != this.doc.getDocId()) { - // data are related to another document: - // append them to any existing data - os.writeInt(storedDocId); - os.writeInt(gidsCount); - os.writeFixedInt(size); - is.copyRaw(os, size); - } else { + // data are related to another document: + // append them to any existing data + os.writeInt(storedDocId); + os.writeInt(gidsCount); + os.writeFixedInt(size); + is.copyRaw(os, size); + } else { - // data are related to our document: - // feed the new list with the GIDs - NodeId previous = null; + // data are related to our document: + // feed the new list with the GIDs + NodeId previous = null; - for (int j = 0; j < gidsCount; j++) { - final NodeId nodeId = broker.getBrokerPool().getNodeFactory().createFromStream(previous, is); - previous = nodeId; + for (int j = 0; j < gidsCount; j++) { + final NodeId nodeId = broker.getBrokerPool().getNodeFactory().createFromStream(previous, is); + previous = nodeId; - // add the node to the new list if it is not - // in the list of removed nodes - if (!containsNode(storedGIDList, nodeId)) { - newGIDList.add(nodeId); + // add the node to the new list if it is not + // in the list of removed nodes + if (!containsNode(storedGIDList, nodeId)) { + newGIDList.add(nodeId); + } } } } - } - //append the data from the new list - if (newGIDList.size() > 0) { - final int gidsCount = newGIDList.size(); + //append the data from the new list + if (newGIDList.size() > 0) { + final int gidsCount = newGIDList.size(); - //Don't forget this one - FastQSort.sort(newGIDList, 0, gidsCount - 1); - os.writeInt(this.doc.getDocId()); - os.writeInt(gidsCount); + //Don't forget this one + FastQSort.sort(newGIDList, 0, gidsCount - 1); + os.writeInt(this.doc.getDocId()); + os.writeInt(gidsCount); - //Compute the new GID list - try { - NodeId previous = null; - for (final NodeId nodeId : newGIDList) { - previous = nodeId.write(previous, nodeIdOs); - } + //Compute the new GID list + try { + NodeId previous = null; + for (final NodeId nodeId : newGIDList) { + previous = nodeId.write(previous, nodeIdOs); + } - final byte[] nodeIdsData = nodeIdOs.toByteArray(); + final byte[] nodeIdsData = nodeIdOs.toByteArray(); - // clear the buf for the next iteration - nodeIdOs.clear(); + // clear the buf for the next iteration + nodeIdOs.clear(); - // Write length of node IDs (bytes) - os.writeFixedInt(nodeIdsData.length); + // Write length of node IDs (bytes) + os.writeFixedInt(nodeIdsData.length); - // write the node IDs - os.write(nodeIdsData); - } catch (final IOException e) { - LOG.warn("IO error while writing range index: {}", e.getMessage(), e); - //TODO : throw exception? + // write the node IDs + os.write(nodeIdsData); + } catch (final IOException e) { + LOG.warn("IO error while writing range index: {}", e.getMessage(), e); + //TODO : throw exception? + } } - } // if(os.data().size() == 0) // dbValues.remove(value); - if (dbValues.update(value.getAddress(), searchKey, os.data()) == BFile.UNKNOWN_ADDRESS) { - LOG.error("Could not update index data for value '{}'", searchKey); - //TODO: throw exception ? - } - } else { + if (dbValues.update(value.getAddress(), searchKey, os.data()) == BFile.UNKNOWN_ADDRESS) { + LOG.error("Could not update index data for value '{}'", searchKey); + //TODO: throw exception ? + } + } else { - if (dbValues.put(searchKey, os.data()) == BFile.UNKNOWN_ADDRESS) { - LOG.error("Could not put index data for value '{}'", searchKey); - //TODO : throw exception ? + if (dbValues.put(searchKey, os.data()) == BFile.UNKNOWN_ADDRESS) { + LOG.error("Could not put index data for value '{}'", searchKey); + //TODO : throw exception ? + } } + } catch (final EXistException | IOException e) { + LOG.error(e.getMessage(), e); + } catch (final LockException e) { + LOG.warn("Failed to acquire lock for '{}'", FileUtils.fileName(dbValues.getFile()), e); + //TODO : return ? + } finally { + os.clear(); } - } catch (final EXistException | IOException e) { - LOG.error(e.getMessage(), e); - } catch (final LockException e) { - LOG.warn("Failed to acquire lock for '{}'", FileUtils.fileName(dbValues.getFile()), e); - //TODO : return ? - } finally { - os.clear(); } + pending.changes.clear(); + } catch (final IOException e) { + LOG.warn("IO error while writing range index: {}", e.getMessage(), e); + //TODO : throw exception? } - pending.changes.clear(); } private static boolean containsNode(final List list, final NodeId nodeId) { @@ -1043,7 +1075,8 @@ public void close() throws DBException { try(final ManagedLock bfileLock = lockManager.acquireBtreeWriteLock(dbValues.getLockName())) { config.setProperty(getConfigKeyForFile(), null); dbValues.close(); - } catch (final LockException e) { + os.close(); + } catch (final LockException | IOException e) { LOG.warn("Failed to acquire lock for '{}'", FileUtils.fileName(dbValues.getFile()), e); } } diff --git a/exist-core/src/main/java/org/exist/storage/StorageAddress.java b/exist-core/src/main/java/org/exist/storage/StorageAddress.java index eefc3f39c8..b273cd604e 100644 --- a/exist-core/src/main/java/org/exist/storage/StorageAddress.java +++ b/exist-core/src/main/java/org/exist/storage/StorageAddress.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -26,7 +50,7 @@ import org.exist.dom.persistent.NodeHandle; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; /** * Represents a (virtual) storage address in the paged file, consisting @@ -114,7 +138,7 @@ public final static boolean equals(NodeHandle n0, NodeHandle n1) { return equals(n0.getInternalAddress(), n1.getInternalAddress()); } - public final static void write(long pointer, VariableByteOutputStream os) { + public final static void write(long pointer, VariableByteOutput os) throws IOException { os.writeInt(pageFromPointer(pointer)); os.writeShort(tidFromPointer(pointer)); os.writeShort(indexTypeFromPointer(pointer)); diff --git a/exist-core/src/main/java/org/exist/storage/index/BFile.java b/exist-core/src/main/java/org/exist/storage/index/BFile.java index b1913dd0d7..20ad855d07 100644 --- a/exist-core/src/main/java/org/exist/storage/index/BFile.java +++ b/exist-core/src/main/java/org/exist/storage/index/BFile.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -40,7 +64,7 @@ import org.exist.storage.cache.LRUCache; import org.exist.storage.io.VariableByteArrayInput; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.storage.journal.JournalException; import org.exist.storage.journal.LogEntryTypes; import org.exist.storage.journal.Loggable; @@ -56,6 +80,9 @@ import java.io.EOFException; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; import java.nio.file.Path; import java.text.NumberFormat; import java.util.ArrayList; @@ -2232,6 +2259,26 @@ public final short readShort() throws IOException { return i; } + @Override + public short readFixedShort() throws IOException { + if (offset == pageLen) { + advance(); + } + // do we have to read across a page boundary? + if (offset + 2 < pageLen) { + return (short) ((nextPage.data[offset++] & 0xff) | + ((nextPage.data[offset++] & 0xff) << 8)); + } + + short r = (short) (nextPage.data[offset++] & 0xff); + if (offset == pageLen) { + advance(); + } + r |= (nextPage.data[offset++] & 0xff) << 8; + + return r; + } + @Override public final int readInt() throws IOException { if (offset == pageLen) { @@ -2261,6 +2308,7 @@ public int readFixedInt() throws IOException { ( (nextPage.data[offset++] & 0xff) << 16 ) | ( (nextPage.data[offset++] & 0xff) << 24 ); } + int r = nextPage.data[offset++] & 0xff; int shift = 8; for (int i = 0; i < 3; i++) { @@ -2290,6 +2338,77 @@ public final long readLong() throws IOException { return i; } + @Override + public long readFixedLong() throws IOException { + if (offset == pageLen) { + advance(); + } + // do we have to read across a page boundary? + if (offset + 8 < pageLen) { + return ((nextPage.data[offset++] & 0xff) << 56) | + ((nextPage.data[offset++] & 0xff) << 48) | + ((nextPage.data[offset++] & 0xff) << 40) | + ((nextPage.data[offset++] & 0xff) << 32) | + ((nextPage.data[offset++] & 0xff) << 24) | + ((nextPage.data[offset++] & 0xff) << 16) | + ((nextPage.data[offset++] & 0xff) << 8) | + (nextPage.data[offset++] & 0xff); + } + + int r = (nextPage.data[offset++] & 0xff) << 56; + int shift = 48; + for (int i = 0; i < 7; i++) { + if (offset == pageLen) { + advance(); + } + r |= (nextPage.data[offset++] & 0xff) << shift; + shift -= 8; + } + return r; + } + + @Override + public BigInteger readBigInteger() throws IOException { + final int dataLength = readInt(); + final byte[] data = new byte[dataLength]; + read(data); + + return new BigInteger(data); + } + + @Override + public BigInteger readFixedBigInteger() throws IOException { + final int dataLength = readFixedInt(); + final byte[] data = new byte[dataLength]; + read(data); + + return new BigInteger(data); + } + + @Override + public BigDecimal readBigDecimal() throws IOException { + final int scale = readInt(); + final int precision = readInt(); + final int dataLength = readInt(); + final byte[] data = new byte[dataLength]; + read(data); + + final MathContext mathContext = new java.math.MathContext(precision); + return new BigDecimal(new BigInteger(data), scale, mathContext); + } + + @Override + public BigDecimal readFixedBigDecimal() throws IOException { + final int scale = readFixedInt(); + final int precision = readFixedInt(); + final int dataLength = readFixedInt(); + final byte[] data = new byte[dataLength]; + read(data); + + final MathContext mathContext = new java.math.MathContext(precision); + return new BigDecimal(new BigInteger(data), scale, mathContext); + } + @Override public final void skip(final int count) throws IOException { for (int i = 0; i < count; i++) { @@ -2383,7 +2502,7 @@ public final String readUTF() throws IOException { } @Override - public final void copyTo(final VariableByteOutputStream os) throws IOException { + public final void copyTo(final VariableByteOutput os) throws IOException { byte more; do { if (offset == pageLen) { @@ -2396,7 +2515,7 @@ public final void copyTo(final VariableByteOutputStream os) throws IOException { } @Override - public final void copyTo(final VariableByteOutputStream os, final int count) throws IOException { + public final void copyTo(final VariableByteOutput os, final int count) throws IOException { byte more; for (int i = 0; i < count; i++) { do { @@ -2410,7 +2529,7 @@ public final void copyTo(final VariableByteOutputStream os, final int count) thr } @Override - public void copyRaw(final VariableByteOutputStream os, final int count) throws IOException { + public void copyRaw(final VariableByteOutput os, final int count) throws IOException { for (int i = count; i != 0; ) { if (offset == pageLen) { advance(); diff --git a/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteInput.java b/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteInput.java index 4ba0aca507..8055bd58ff 100644 --- a/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteInput.java +++ b/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteInput.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,6 +47,9 @@ import java.io.EOFException; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; import static java.nio.charset.StandardCharsets.UTF_8; @@ -30,13 +57,16 @@ * Abstract base class for implementations of VariableByteInput. * * @author wolf + * @author Adam Retter */ public abstract class AbstractVariableByteInput implements VariableByteInput { @Override public byte readByte() throws IOException { final int i = read(); - if (i < 0) {throw new EOFException();} + if (i < 0) { + throw new EOFException(); + } return (byte) i; } @@ -51,6 +81,12 @@ public short readShort() throws IOException { return i; } + @Override + public short readFixedShort() throws IOException { + return (short) ((readByte() & 0xff) | + ((readByte() & 0xff) << 8)); + } + @Override public int readInt() throws IOException { byte b = readByte(); @@ -81,6 +117,61 @@ public long readLong() throws IOException { return i; } + @Override + public long readFixedLong() throws IOException { + return + ((readByte() & 0xff) << 56) | + ((readByte() & 0xff) << 48) | + ((readByte() & 0xff) << 40) | + ((readByte() & 0xff) << 32) | + ((readByte() & 0xff) << 24) | + ((readByte() & 0xff) << 16) | + ((readByte() & 0xff) << 8) | + (readByte() & 0xff); + } + + @Override + public BigInteger readBigInteger() throws IOException { + final int dataLength = readInt(); + final byte[] data = new byte[dataLength]; + read(data); + return new BigInteger(data); + } + + @Override + public BigInteger readFixedBigInteger() throws IOException { + final int dataLength = readFixedInt(); + final byte[] data = new byte[dataLength]; + read(data); + + return new BigInteger(data); + } + + @Override + public BigDecimal readBigDecimal() throws IOException { + final int scale = readInt(); + final int precision = readInt(); + final int dataLength = readInt(); + final byte[] data = new byte[dataLength]; + read(data); + + final MathContext mathContext = new java.math.MathContext(precision); + return new BigDecimal(new BigInteger(data), scale, mathContext); + } + + @Override + public BigDecimal readFixedBigDecimal() throws IOException { + final int scale = readFixedInt(); + final int precision = readFixedInt(); + final int dataLength = readFixedInt(); + final byte[] data = new byte[dataLength]; + read(data); + + final MathContext mathContext = new java.math.MathContext(precision); + return new BigDecimal(new BigInteger(data), scale, mathContext); + } + + @Override public String readUTF() throws IOException { final int len = readInt(); @@ -140,7 +231,8 @@ public int read(final byte b[], final int off, final int len) throws IOException return i; } - public void copyTo(final VariableByteOutputStream os) throws IOException { + @Override + public void copyTo(final VariableByteOutput os) throws IOException { int more; do { more = read(); @@ -150,7 +242,7 @@ public void copyTo(final VariableByteOutputStream os) throws IOException { } @Override - public void copyTo(final VariableByteOutputStream os, final int count) + public void copyTo(final VariableByteOutput os, final int count) throws IOException { int more; for (int i = 0; i < count; i++) { @@ -163,7 +255,7 @@ public void copyTo(final VariableByteOutputStream os, final int count) } @Override - public void copyRaw(final VariableByteOutputStream os, final int count) throws IOException { + public void copyRaw(final VariableByteOutput os, final int count) throws IOException { final byte buf[] = new byte[count]; int totalRead = 0; int read; @@ -172,8 +264,4 @@ public void copyRaw(final VariableByteOutputStream os, final int count) throws I totalRead += read; } } - - public void release() { - //Nothing to do - } } \ No newline at end of file diff --git a/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteOutput.java b/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteOutput.java new file mode 100644 index 0000000000..f7a19698c1 --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/AbstractVariableByteOutput.java @@ -0,0 +1,163 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Abstract base class for implementations of VariableByteOutput. + * + * Note that the VBE scheme used by this class + * does not offer any advantage for negative numbers, in fact + * it requires significantly more storage for those; see the javadoc + * on the appropriate encoding method for details. + * + * If support for negative numbers is desired then, the reader + * should look to zig-zag encoding as used in the varint's of + * Google's Protocol Buffers https://developers.google.com/protocol-buffers/docs/encoding#signed-integers + * or Hadoop's VarInt encoding, see org.apache.hadoop.io.file.tfile.Utils#writeVInt(java.io.DataOutput, int). + * + * VBE is never an alternative to having advance knowledge of number + * ranges and using fixed size byte arrays to represent them. + * + * Rather, for example, it is useful when you have an int that could be + * in any range between 0 and {@link Integer#MAX_VALUE}, but is likely + * less than 2,097,151, in that case you would save at least 1 byte for + * each int value that is written to the output stream that is + * less than 2,097,151. + * + * @author Adam Retter + */ +public abstract class AbstractVariableByteOutput implements VariableByteOutput { + + @Override + public void writeByte(final byte b) throws IOException { + write(b); + } + + @Override + public void write(final byte[] buf) throws IOException { + write(buf, 0, buf.length); + } + + @Override + public void write(final byte[] buf, final int off, final int len) throws IOException { + for (int i = off; i < off + len; i++) { + write(buf[i]); + } + } + + @Override + public void writeShort(int s) throws IOException { + while ((s & ~0177) != 0) { + write((byte) ((s & 0177) | 0200)); + s >>>= 7; + } + write((byte) s); + } + + @Override + public void writeFixedShort(final short s) throws IOException { + write((byte) ((s >>> 0) & 0xff)); + write((byte) ((s >>> 8) & 0xff)); + } + + @Override + public void writeInt(int i) throws IOException { + while ((i & ~0177) != 0) { + write((byte) ((i & 0177) | 0200)); + i >>>= 7; + } + write((byte) i); + } + + @Override + public void writeFixedInt(final int i) throws IOException { + write((byte) ((i >>> 0) & 0xff)); + write((byte) ((i >>> 8) & 0xff)); + write((byte) ((i >>> 16) & 0xff)); + write((byte) ((i >>> 24) & 0xff)); + } + + @Override + public void writeLong(long l) throws IOException { + while ((l & ~0177) != 0) { + write((byte) ((l & 0177) | 0200)); + l >>>= 7; + } + write((byte) l); + } + + @Override + public void writeFixedLong(final long l) throws IOException { + write((byte) ((l >>> 56) & 0xff)); + write((byte) ((l >>> 48) & 0xff)); + write((byte) ((l >>> 40) & 0xff)); + write((byte) ((l >>> 32) & 0xff)); + write((byte) ((l >>> 24) & 0xff)); + write((byte) ((l >>> 16) & 0xff)); + write((byte) ((l >>> 8) & 0xff)); + write((byte) ((l >>> 0) & 0xff)); + } + + @Override + public void writeBigInteger(final BigInteger bi) throws IOException { + final byte[] data = bi.toByteArray(); + writeInt(data.length); + write(data); + } + + @Override + public void writeFixedBigInteger(final BigInteger bi) throws IOException { + final byte[] data = bi.toByteArray(); + writeFixedInt(data.length); + write(data); + } + + @Override + public void writeBigDecimal(final BigDecimal bd) throws IOException { + final byte[] data = bd.unscaledValue().toByteArray(); + writeInt(bd.scale()); + writeInt(bd.precision()); + writeInt(data.length); + write(data); + } + + @Override + public void writeFixedBigDecimal(final BigDecimal bd) throws IOException { + final byte[] data = bd.unscaledValue().toByteArray(); + writeFixedInt(bd.scale()); + writeFixedInt(bd.precision()); + writeFixedInt(data.length); + write(data); + } + + @Override + public void writeUTF(final String s) throws IOException { + final byte[] data = s.getBytes(UTF_8); + writeInt(data.length); + write(data, 0, data.length); + } +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayInput.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayInput.java index 3b67302b33..5a2343c573 100644 --- a/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayInput.java +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayInput.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,15 +45,10 @@ */ package org.exist.storage.io; -import java.io.EOFException; -import java.io.IOException; - -import static java.nio.charset.StandardCharsets.UTF_8; - /** - * Implements VariableByteInput on top of a byte array. + * A byte array input using VBE (Variable Byte Encoding). * - * @author wolf + * @author Adam Retter */ public class VariableByteArrayInput extends AbstractVariableByteInput { @@ -37,10 +56,6 @@ public class VariableByteArrayInput extends AbstractVariableByteInput { protected int position; private int end; - public VariableByteArrayInput() { - super(); - } - public VariableByteArrayInput(final byte[] data) { super(); this.data = data; @@ -62,15 +77,7 @@ public void initialize(final byte[] data, final int offset, final int length) { } @Override - public byte readByte() throws IOException { - if (position == end) { - throw new EOFException(); - } - return data[position++]; - } - - @Override - public int read() throws IOException { + public int read() { if (position == end) { return -1; } @@ -78,121 +85,23 @@ public int read() throws IOException { } @Override - public int available() throws IOException { + public int available() { return end - position; } @Override - public short readShort() throws IOException { - if (position == end) { - throw new EOFException(); - } - byte b = data[position++]; - short i = (short) (b & 0177); - for (int shift = 7; (b & 0200) != 0; shift += 7) { - if (position == end) { - throw new EOFException(); - } - b = data[position++]; - i |= (b & 0177) << shift; - } - return i; - } - - @Override - public int readInt() throws IOException { - if (position == end) { - throw new EOFException(); - } - byte b = data[position++]; - int i = b & 0177; - for (int shift = 7; (b & 0200) != 0; shift += 7) { - if (position == end) { - throw new EOFException(); - } - b = data[position++]; - i |= (b & 0177) << shift; - } - return i; - } - - @Override - public int readFixedInt() throws IOException { - return ( data[position++] & 0xff ) | - ( ( data[position++] & 0xff ) << 8 ) | - ( ( data[position++] & 0xff ) << 16 ) | - ( ( data[position++] & 0xff ) << 24 ); - } - - @Override - public long readLong() throws IOException { - if (position == end) { - throw new EOFException(); - } - byte b = data[position++]; - long i = b & 0177L; - for (int shift = 7; (b & 0200) != 0; shift += 7) { - if (position == end) { - throw new EOFException(); - } - b = data[position++]; - i |= (b & 0177L) << shift; - } - return i; - } - - @Override - public String readUTF() throws IOException { - final int len = readInt(); - final String s = new String(data, position, len, UTF_8); - position += len; - return s; - } - - @Override - public void copyTo(final VariableByteOutputStream os, final int count) throws IOException { - byte more; - for (int i = 0; i < count; i++) { - do { - more = data[position++]; - os.write(more); - } while ((more & 0x200) > 0); - } - } - - @Override - public void copyRaw(final VariableByteOutputStream os, final int count) throws IOException { - os.write(data, position, count); - position += count; - } - - @Override - public void skip(final int count) throws IOException { + public void skip(final int count) { for (int i = 0; i < count; i++) { - while (position < end && (data[position++] & 0200) > 0) { + while (position < end && (data[position++] & 128) > 0) { //Nothing to do } } } @Override - public void skipBytes(final long count) throws IOException { - for(long i = 0; i < count && position < end; i++) { + public void skipBytes(final long count) { + for (long i = 0; i < count && position < end; i++) { position++; } } - - public String toString(final int len) { - final byte[] subArray = new byte[len]; - System.arraycopy(data, position, subArray, 0, len); - final StringBuilder buf = new StringBuilder("["); - for (int i = 0 ; i < len; i++) { - if (i > 0) { - buf.append(" "); - } - buf.append(subArray[i]); - } - buf.append("]"); - return buf.toString(); - } } \ No newline at end of file diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayOutputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayOutputStream.java new file mode 100644 index 0000000000..2f30b0e53b --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteArrayOutputStream.java @@ -0,0 +1,169 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; +import org.exist.util.ByteArray; +import org.exist.util.FixedByteArray; + +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * A byte array output stream using VBE (Variable Byte Encoding). + * + * The choice of the backing buffer is quite tricky, we have two easy options: + * + * 1. org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream + * This allocates multiple underlying buffers in sequence, which means that appends to the buffer always allocate + * a new buffer, and so there is no GC overhead for appending. However, for serialization #toArray() involves + * allocating a new array and copying data from those multiple buffers into the new array, this requires 2x + * memory. + * NOTE: Previously this classes {@link VariableByteArrayOutputStream#toByteArray()} made a copy anyway, and so + * previously required 2x memory. + * + * 2. it.unimi.dsi.fastutil.io.UnsynchronizedByteArrayOutputStream + * This allocates a single underlying buffer, appends that + * would overflow the underlying buffer cause a new buffer to be allocated, data copied, and the old buffer left + * to GC. This means that appends which require resizing the buffer can be expensive. However, #toArray() is not + * needed as access to the underlying array is permitted, so this is very cheap for serializing. + * + * Likely there are different scenarios where each is more appropriate. + * + * @author Adam Retter + */ +public class VariableByteArrayOutputStream extends OutputStream implements VariableByteOutput { + + private final UnsynchronizedByteArrayOutputStream os; + private final VariableByteFilterOutputStream vbfo; + + public VariableByteArrayOutputStream() { + this(512); + } + + public VariableByteArrayOutputStream(final int size) { + this.os = UnsynchronizedByteArrayOutputStream.builder().setBufferSize(size).get(); + this.vbfo = new VariableByteFilterOutputStream(os); + } + + public void clear() { + os.reset(); + } + + @Override + public void close() throws IOException { + vbfo.close(); + } + + public int size() { + return os.size(); + } + + public byte[] toByteArray() { + return os.toByteArray(); + } + + public ByteArray data() { + return new FixedByteArray(toByteArray()); + } + + @Override + public void write(final int b) throws IOException { + vbfo.write(b); + } + + @Override + public void write(final byte[] b) throws IOException { + vbfo.write(b); + } + + @Override + public void write(final byte[] b, final int off, final int len) throws IOException { + vbfo.write(b, off, len); + } + + @Override + public void writeByte(final byte b) throws IOException { + vbfo.writeByte(b); + } + + @Override + public void writeShort(final int s) throws IOException { + vbfo.writeShort(s); + } + + @Override + public void writeFixedShort(final short s) throws IOException { + vbfo.writeFixedShort(s); + } + + @Override + public void writeInt(final int i) throws IOException { + vbfo.writeInt(i); + } + + @Override + public void writeFixedInt(final int i) throws IOException { + vbfo.writeFixedInt(i); + } + + @Override + public void writeLong(final long l) throws IOException { + vbfo.writeLong(l); + } + + @Override + public void writeFixedLong(final long l) throws IOException { + vbfo.writeFixedLong(l); + } + + @Override + public void writeBigInteger(final BigInteger bi) throws IOException { + vbfo.writeBigInteger(bi); + } + + @Override + public void writeFixedBigInteger(final BigInteger bi) throws IOException { + vbfo.writeFixedBigInteger(bi); + } + + @Override + public void writeBigDecimal(final BigDecimal bd) throws IOException { + vbfo.writeBigDecimal(bd); + } + + @Override + public void writeFixedBigDecimal(final BigDecimal bd) throws IOException { + vbfo.writeFixedBigDecimal(bd); + } + + @Override + public void writeUTF(final String s) throws IOException { + vbfo.writeUTF(s); + } + + @Override + public void flush() throws IOException { + vbfo.flush(); + } +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteInputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteBufferInput.java similarity index 53% rename from exist-core/src/main/java/org/exist/storage/io/VariableByteInputStream.java rename to exist-core/src/main/java/org/exist/storage/io/VariableByteBufferInput.java index ce660233e7..9bc0e976ff 100644 --- a/exist-core/src/main/java/org/exist/storage/io/VariableByteInputStream.java +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteBufferInput.java @@ -1,14 +1,13 @@ /* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd * - * info@exist-db.org - * http://www.exist-db.org + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. + * License as published by the Free Software Foundation; version 2.1. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -22,36 +21,32 @@ package org.exist.storage.io; import java.io.IOException; -import java.io.InputStream; - +import java.nio.ByteBuffer; /** - * Implements VariableByteInput on top of an InputStream. - * - * @author wolf + * Implements VariableByteInput on top of a ByteBuffer. + * + * @author Adam Retter */ -public class VariableByteInputStream extends AbstractVariableByteInput { +public class VariableByteBufferInput extends AbstractVariableByteInput { - private InputStream is; - - public VariableByteInputStream(InputStream is) { + private final ByteBuffer buf; + + public VariableByteBufferInput(final ByteBuffer buf) { super(); - this.is = is; + this.buf = buf; } - /* (non-Javadoc) - * @see java.io.InputStream#read() - */ @Override public int read() throws IOException { - return is.read(); + if (buf.remaining() == 0) { + return -1; + } + return buf.get() & 0xFF; } - /* (non-Javadoc) - * @see java.io.InputStream#available() - */ @Override public int available() throws IOException { - return is.available(); - } + return buf.remaining(); + } } diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteBufferOutput.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteBufferOutput.java new file mode 100644 index 0000000000..5067ccc138 --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteBufferOutput.java @@ -0,0 +1,54 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Implements VariableByteOutput on top of a ByteBuffer. + * + * @author Adam Retter + */ +public class VariableByteBufferOutput extends AbstractVariableByteOutput { + + private final ByteBuffer buf; + + public VariableByteBufferOutput(final ByteBuffer buf) { + super(); + this.buf = buf; + } + + @Override + public void write(final int b) throws IOException { + buf.put((byte) b); + } + + @Override + public void write(final byte[] buf) throws IOException { + this.buf.put(buf); + } + + @Override + public void write(final byte[] buf, final int off, final int len) throws IOException { + this.buf.put(buf, off, len); + } +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteFilterInputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteFilterInputStream.java new file mode 100644 index 0000000000..7c82188527 --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteFilterInputStream.java @@ -0,0 +1,142 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * An input stream filter using VBE (Variable Byte Encoding). + * + * @author Adam Retter + */ +public class VariableByteFilterInputStream extends FilterInputStream implements VariableByteInput { + + private final VariableByteInput vbi; + + public VariableByteFilterInputStream(final InputStream is) { + super(is); + vbi = new VariableByteInputToInputStream(is); + } + + @Override + public int read() throws IOException { + return vbi.read(); + } + + @Override + public int read(final byte[] b) throws IOException { + return vbi.read(b); + } + + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + return vbi.read(b, off, len); + } + + @Override + public byte readByte() throws IOException { + return vbi.readByte(); + } + + @Override + public short readShort() throws IOException { + return vbi.readShort(); + } + + @Override + public short readFixedShort() throws IOException { + return vbi.readFixedShort(); + } + + @Override + public int readInt() throws IOException { + return vbi.readInt(); + } + + @Override + public int readFixedInt() throws IOException { + return vbi.readFixedInt(); + } + + @Override + public long readLong() throws IOException { + return vbi.readLong(); + } + + @Override + public long readFixedLong() throws IOException { + return vbi.readFixedLong(); + } + + @Override + public BigInteger readBigInteger() throws IOException { + return vbi.readBigInteger(); + } + + @Override + public BigInteger readFixedBigInteger() throws IOException { + return vbi.readFixedBigInteger(); + } + + @Override + public BigDecimal readBigDecimal() throws IOException { + return vbi.readBigDecimal(); + } + + @Override + public BigDecimal readFixedBigDecimal() throws IOException { + return vbi.readFixedBigDecimal(); + } + + @Override + public String readUTF() throws IOException { + return vbi.readUTF(); + } + + @Override + public void skip(final int count) throws IOException { + vbi.skip(count); + } + + @Override + public void skipBytes(final long count) throws IOException { + vbi.skipBytes(count); + } + + @Override + public void copyTo(final VariableByteOutput os) throws IOException { + vbi.copyTo(os); + } + + @Override + public void copyTo(final VariableByteOutput os, final int count) throws IOException { + vbi.copyTo(os, count); + } + + @Override + public void copyRaw(final VariableByteOutput os, final int bytes) throws IOException { + vbi.copyRaw(os, bytes); + } +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteFilterOutputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteFilterOutputStream.java new file mode 100644 index 0000000000..e33bce6207 --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteFilterOutputStream.java @@ -0,0 +1,117 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * An output stream filter using VBE (Variable Byte Encoding). + * + * @author Adam Retter + */ +public class VariableByteFilterOutputStream extends FilterOutputStream implements VariableByteOutput { + + private final VariableByteOutput vbo; + + public VariableByteFilterOutputStream(final OutputStream os) { + super(os); + this.vbo = new VariableByteOutputToOutputStream(os); + } + + @Override + public void write(final int b) throws IOException { + vbo.write(b); + } + + @Override + public void write(final byte[] b) throws IOException { + vbo.write(b); + } + + @Override + public void write(final byte[] b, final int off, final int len) throws IOException { + vbo.write(b, off, len); + } + + @Override + public void writeByte(final byte b) throws IOException { + vbo.writeByte(b); + } + + @Override + public void writeShort(final int s) throws IOException { + vbo.writeShort(s); + } + + @Override + public void writeFixedShort(final short s) throws IOException { + vbo.writeFixedShort(s); + } + + @Override + public void writeInt(final int i) throws IOException { + vbo.writeInt(i); + } + + @Override + public void writeFixedInt(final int i) throws IOException { + vbo.writeFixedInt(i); + } + + @Override + public void writeLong(final long l) throws IOException { + vbo.writeLong(l); + } + + @Override + public void writeFixedLong(final long l) throws IOException { + vbo.writeFixedLong(l); + } + + @Override + public void writeBigInteger(final BigInteger bi) throws IOException { + vbo.writeBigInteger(bi); + } + + @Override + public void writeFixedBigInteger(final BigInteger bi) throws IOException { + vbo.writeFixedBigInteger(bi); + } + + @Override + public void writeBigDecimal(final BigDecimal bd) throws IOException { + vbo.writeBigDecimal(bd); + } + + @Override + public void writeFixedBigDecimal(final BigDecimal bd) throws IOException { + vbo.writeFixedBigDecimal(bd); + } + + @Override + public void writeUTF(final String s) throws IOException { + vbo.writeUTF(s); + } +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteInput.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteInput.java index 2b5c8daf55..78edbeb820 100644 --- a/exist-core/src/main/java/org/exist/storage/io/VariableByteInput.java +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteInput.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,8 +45,9 @@ */ package org.exist.storage.io; -import java.io.EOFException; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; /** * Interface for reading variable byte encoded values. @@ -32,6 +57,7 @@ * the given type. * * @author wolf + * @author Adam Retter */ public interface VariableByteInput { @@ -41,7 +67,7 @@ public interface VariableByteInput { * @return the byte value as int or -1 if no more bytes are available. * @throws IOException in case of an I/O error */ - public int read() throws IOException; + int read() throws IOException; /** * Fill the provided byte array with data from the input. @@ -50,9 +76,9 @@ public interface VariableByteInput { * @throws IOException in case of an I/O error * @return the number of bytes read */ - public int read(byte[] data) throws IOException; + int read(byte[] data) throws IOException; - public int read(byte b[], int off, int len) throws IOException; + int read(byte b[], int off, int len) throws IOException; /** * Returns a value > 0 if more bytes can be read @@ -61,7 +87,7 @@ public interface VariableByteInput { * @throws IOException in case of an I/O error * @return the number of bytes available */ - public int available() throws IOException; + int available() throws IOException; /** * Read a single byte. Throws EOFException if no @@ -70,7 +96,7 @@ public interface VariableByteInput { * @throws IOException in case of an I/O error * @return the byte read */ - public byte readByte() throws IOException; + byte readByte() throws IOException; /** * Read a short value in variable byte encoding. @@ -78,7 +104,16 @@ public interface VariableByteInput { * @throws IOException in case of an I/O error * @return the short read */ - public short readShort() throws IOException; + short readShort() throws IOException; + + /** + * Read a fixed size short from the input. + * + * Requires 2 bytes. + * + * @return the short. + */ + short readFixedShort() throws IOException; /** * Read an integer value in variable byte encoding. @@ -86,9 +121,16 @@ public interface VariableByteInput { * @throws IOException in case of an I/O error * @return the int read */ - public int readInt() throws IOException; + int readInt() throws IOException; - public int readFixedInt() throws IOException; + /** + * Read a fixed size int from the input. + * + * Requires 4 bytes. + * + * @return the int. + */ + int readFixedInt() throws IOException; /** * Read a long value in variable byte encoding. @@ -96,9 +138,59 @@ public interface VariableByteInput { * @throws IOException in case of an I/O error * @return the long read */ - public long readLong() throws IOException; + long readLong() throws IOException; - public String readUTF() throws IOException, EOFException; + /** + * Read a fixed size long from the input. + * + * Requires 8 bytes. + * + * @return the long. + */ + long readFixedLong() throws IOException; + + /** + * Read a big integer in variable byte encoding. + * + * @throws IOException in case of an I/O error + * + * @return the big integer read + */ + BigInteger readBigInteger() throws IOException; + + /** + * Read a fixed size big integer from input. + * + * @throws IOException in case of an I/O error + * + * @return the big integer read + */ + BigInteger readFixedBigInteger() throws IOException; + + /** + * Read a big decimal in variable byte encoding. + * + * @throws IOException in case of an I/O error + * + * @return the big decimal read + */ + BigDecimal readBigDecimal() throws IOException; + + /** + * Read a fixed size big decimal from input. + * + * @throws IOException in case of an I/O error + * + * @return the big decimal read + */ + BigDecimal readFixedBigDecimal() throws IOException; + + /** + * Read a string as UTF-8 encoded bytes from the input. + * + * @return the string. + */ + String readUTF() throws IOException; /** * Read the following count numeric values from the input @@ -107,30 +199,30 @@ public interface VariableByteInput { * @param count the number of bytes to skip * @throws IOException in case of an I/O error */ - public void skip(int count) throws IOException; + void skip(int count) throws IOException; - public void skipBytes(long count) throws IOException; + void skipBytes(long count) throws IOException; /** * Copy the next numeric value from the input to the * specified output stream. * - * @param os the output stream to copy the data to + * @param output the output destination to copy the data to * @throws IOException in case of an I/O error */ - public void copyTo(VariableByteOutputStream os) throws IOException; + void copyTo(VariableByteOutput output) throws IOException; /** * Copy the count next numeric values from the input to * the specified output stream. * - * @param os the output stream to copy the data to + * @param os the output destination to copy the data to * @param count the number of bytes to copy * @throws IOException in case of an I/O error */ - public void copyTo(VariableByteOutputStream os, int count) + void copyTo(VariableByteOutput os, int count) throws IOException; - public void copyRaw(VariableByteOutputStream os, int bytes) + void copyRaw(VariableByteOutput os, int bytes) throws IOException; } \ No newline at end of file diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteInputToInputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteInputToInputStream.java new file mode 100644 index 0000000000..8bff416807 --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteInputToInputStream.java @@ -0,0 +1,76 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.IOException; +import java.io.InputStream; + + +/** + * Implements VariableByteInput on top of an InputStream. + * + * @author wolf + * @author Adam Retter + */ +public class VariableByteInputToInputStream extends AbstractVariableByteInput { + + private final InputStream is; + + public VariableByteInputToInputStream(final InputStream is) { + super(); + this.is = is; + } + + @Override + public int read() throws IOException { + return is.read(); + } + + @Override + public int available() throws IOException { + return is.available(); + } +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteOutput.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteOutput.java new file mode 100644 index 0000000000..d959439a67 --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteOutput.java @@ -0,0 +1,199 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * Interface for writing variable byte encoded values. + * + * @author Adam Retter + */ +public interface VariableByteOutput { + + /** + * Write a byte to the output. + * + * @param b the byte to write. + */ + void write(final int b) throws IOException; + + /** + * Write a byte to the output. + * + * @param b the byte to write. + */ + void writeByte(final byte b) throws IOException; + + /** + * Write bytes to the output. + * + * @param buf the bytes to write. + */ + void write(final byte[] buf) throws IOException; + + /** + * Write bytes to the output. + * + * @param buf the bytes to write. + * @param off the offset to read the bytes from. + * @param len the length of bytes to read. + */ + void write(final byte[] buf, final int off, final int len) throws IOException; + + /** + * Writes a VBE short to the output. + * + * The encoding scheme requires the following storage + * for numbers between (inclusive): + * + * {@link Short#MIN_VALUE} and -1, 5 bytes + * 0 and 127, 1 byte + * 128 and 16383, 2 bytes + * 16384 and {@link Short#MAX_VALUE}, 3 bytes + * + * @param s the short to write. + */ + void writeShort(int s) throws IOException; + + /** + * Write a fixed size short to the output. + * + * Requires 2 bytes. + * + * @param s the short to write. + */ + void writeFixedShort(final short s) throws IOException; + + /** + * Writes a VBE int to the output. + * + * The encoding scheme requires the following storage + * for numbers between (inclusive): + * + * {@link Integer#MIN_VALUE} and -1, 5 bytes + * 0 and 127, 1 byte + * 128 and 16383, 2 bytes + * 16384 and 2097151, 3 bytes + * 2097152 and 268435455, is 4 bytes + * 268435456 and {@link Integer#MAX_VALUE}, 5 bytes + * + * @param i the integer to write. + */ + void writeInt(int i) throws IOException; + + /** + * Write a fixed size int to the output. + * + * Requires 4 bytes. + * + * @param i the integer to write. + */ + void writeFixedInt(final int i) throws IOException; + + /** + * Writes a VBE long to the output. + * + * The encoding scheme requires the following storage + * for numbers between (inclusive): + * + * {@link Long#MIN_VALUE} and -1, 10 bytes + * 0 and 127, 1 byte + * 128 and 16383, 2 bytes + * 16384 and 2097151, 3 bytes + * 2097152 and 268435455, is 4 bytes + * 268435456 and 34359738367, 5 bytes + * 34359738368 and 4398046511103, 6 bytes + * 4398046511104 and 562949953421311, 7 bytes + * 562949953421312 and 72057594037927935, 8 bytes + * 72057594037927936 and 9223372036854775807, 9 bytes + * 9223372036854775808 and {@link Long#MAX_VALUE}, 10 bytes + * + * @param l the long to write. + */ + void writeLong(long l) throws IOException; + + /** + * Write a fixed size long to the output. + * + * Requires 8 bytes. + * + * @param l the long to write. + */ + void writeFixedLong(final long l) throws IOException; + + /** + * Writes a VBE BigInteger to the output. + * + * The BigInteger will be written as: + * VBE int - data length + * byte[] - data + * + * @param bi the big integer to write + */ + void writeBigInteger(final BigInteger bi) throws IOException; + + /** + * Writes a BigInteger to the output. + * + * The BigInteger will be written as: + * int - data length + * byte[] - data + * + * @param bi the big integer to write + */ + void writeFixedBigInteger(final BigInteger bi) throws IOException; + + /** + * Writes a VBE BigDecimal to the output. + * + * The BigInteger will be written as: + * VBE int - scale + * VBE int - precision + * VBE int - data length + * byte[] - data + * + * @param bd the big decimal to write. + */ + void writeBigDecimal(final BigDecimal bd) throws IOException; + + /** + * Writes a BigDecimal to the output. + * + * The BigInteger will be written as: + * int - scale + * int - precision + * int - data length + * byte[] - data + * + * @param bd the big decimal to write. + */ + void writeFixedBigDecimal(final BigDecimal bd) throws IOException; + + /** + * Write a string as UTF-8 encoded bytes to the output. + * + * @param s the string to write. + */ + void writeUTF(final String s) throws IOException; +} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteOutputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteOutputStream.java deleted file mode 100644 index ea030fcf6f..0000000000 --- a/exist-core/src/main/java/org/exist/storage/io/VariableByteOutputStream.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.storage.io; - -import java.io.IOException; -import java.io.OutputStream; - -import org.exist.util.ByteArray; -import org.exist.util.FixedByteArray; -import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * A byte array output stream using VBE (Variable Byte Encoding). - * - * Note that the VBE scheme used by this class - * does not offer any advantage for negative numbers, in fact - * it requires significantly more storage for those; see the javadoc - * on the appropriate encoding method for details. - * - * If support for negative numbers is desired then, the reader - * should look to zig-zag encoding as used in the varint's of - * Google's Protocol Buffers https://developers.google.com/protocol-buffers/docs/encoding#signed-integers - * or Hadoop's VarInt encoding, see org.apache.hadoop.io.file.tfile.Utils#writeVInt(java.io.DataOutput, int). - * - * VBE is never an alternative to having advance knowledge of number - * ranges and using fixed size byte arrays to represent them. - * - * Rather, for example, it is useful when you have an int that could be - * in any range between 0 and {@link Integer#MAX_VALUE}, but is likely - * less than 2,097,151, in that case you would save at least 1 byte for - * each int value that is written to the output stream that is - * less than 2,097,151. - * - * @author Adam Retter - */ -public class VariableByteOutputStream extends OutputStream { - - /** - * The choice of the backing buffer is quite tricky, we have two easy options: - * - * 1. org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream - * This allocates multiple underlying buffers in sequence, which means that appends to the buffer always allocate - * a new buffer, and so there is no GC overhead for appending. However, for serialization #toArray() involves - * allocating a new array and copying data from those multiple buffers into the new array, this requires 2x - * memory. - * NOTE: Previously this classes {@link VariableByteOutputStream#toByteArray()} made a copy anyway, and so - * previously required 2x memory. - * - * 2. it.unimi.dsi.fastutil.io.UnsynchronizedByteArrayOutputStream - * This allocates a single underlying buffer, appends that - * would overflow the underlying buffer cause a new buffer to be allocated, data copied, and the old buffer left - * to GC. This means that appends which require resizing the buffer can be expensive. However, #toArray() is not - * needed as access to the underlying array is permitted, so this is very cheap for serializing. - * - * Likely there are different scenarios where each is more appropriate. - */ - private final UnsynchronizedByteArrayOutputStream buf; - - public VariableByteOutputStream() { - super(); - buf = new UnsynchronizedByteArrayOutputStream(512); - } - - public VariableByteOutputStream(final int bytes) { - super(); - buf = new UnsynchronizedByteArrayOutputStream(bytes); - } - - public void clear() { - buf.reset(); - } - - @Override - public void close() throws IOException { - buf.close(); - } - - public int size() { - return buf.size(); - } - - @Override - public void flush() throws IOException { - buf.flush(); - } - - public byte[] toByteArray() { - return buf.toByteArray(); - } - - public ByteArray data() { - return new FixedByteArray(buf.toByteArray()); - } - - @Override - public void write(final int b) throws IOException { - buf.write(b); - } - - @Override - public void write(final byte[] b) throws IOException { - buf.write(b); - } - - @Override - public void write(final byte[] b, final int off, final int len) throws IOException { - buf.write(b, off, len); - } - -// public void write(final ByteArray b) { -// b.copyTo(buf); -// } - - public void writeByte(final byte b) { - buf.write(b); - } - - /** - * Writes a VBE short to the output stream - * - * The encoding scheme requires the following storage - * for numbers between (inclusive): - * - * {@link Short#MIN_VALUE} and -1, 5 bytes - * 0 and 127, 1 byte - * 128 and 16383, 2 bytes - * 16384 and {@link Short#MAX_VALUE}, 3 bytes - * - * @param s the short to write - */ - public void writeShort(int s) { - while ((s & ~0177) != 0) { - buf.write((byte) ((s & 0177) | 0200)); - s >>>= 7; - } - buf.write((byte) s); - } - - /** - * Writes a VBE int to the output stream - * - * The encoding scheme requires the following storage - * for numbers between (inclusive): - * - * {@link Integer#MIN_VALUE} and -1, 5 bytes - * 0 and 127, 1 byte - * 128 and 16383, 2 bytes - * 16384 and 2097151, 3 bytes - * 2097152 and 268435455, is 4 bytes - * 268435456 and {@link Integer#MAX_VALUE}, 5 bytes - * - * @param i the integer to write - */ - public void writeInt(int i) { - while ((i & ~0177) != 0) { - buf.write((byte) ((i & 0177) | 0200)); - i >>>= 7; - } - buf.write((byte) i); - } - - public void writeFixedInt(final int i) { - buf.write((byte) ( ( i >>> 0 ) & 0xff )); - buf.write((byte) ( ( i >>> 8 ) & 0xff )); - buf.write((byte) ( ( i >>> 16 ) & 0xff )); - buf.write((byte) ( ( i >>> 24 ) & 0xff )); - } - - /** - * Writes a VBE long to the output stream - * - * The encoding scheme requires the following storage - * for numbers between (inclusive): - * - * {@link Long#MIN_VALUE} and -1, 10 bytes - * 0 and 127, 1 byte - * 128 and 16383, 2 bytes - * 16384 and 2097151, 3 bytes - * 2097152 and 268435455, is 4 bytes - * 268435456 and 34359738367, 5 bytes - * 34359738368 and 4398046511103, 6 bytes - * 4398046511104 and 562949953421311, 7 bytes - * 562949953421312 and 72057594037927935, 8 bytes - * 72057594037927936 and 9223372036854775807, 9 bytes - * 9223372036854775808 and {@link Long#MAX_VALUE}, 10 bytes - * - * @param l the long to write - */ - public void writeLong(long l) { - while ((l & ~0177) != 0) { - buf.write((byte) ((l & 0177) | 0200)); - l >>>= 7; - } - buf.write((byte) l); - } - - public void writeFixedLong(final long l) { - buf.write((byte) ((l >>> 56) & 0xff)); - buf.write((byte) ((l >>> 48) & 0xff)); - buf.write((byte) ((l >>> 40) & 0xff)); - buf.write((byte) ((l >>> 32) & 0xff)); - buf.write((byte) ((l >>> 24) & 0xff)); - buf.write((byte) ((l >>> 16) & 0xff)); - buf.write((byte) ((l >>> 8) & 0xff)); - buf.write((byte) ((l >>> 0) & 0xff)); - } - - public void writeUTF(final String s) throws IOException { - final byte[] data = s.getBytes(UTF_8); - writeInt(data.length); - write(data, 0, data.length); - } -} diff --git a/exist-core/src/main/java/org/exist/storage/io/VariableByteOutputToOutputStream.java b/exist-core/src/main/java/org/exist/storage/io/VariableByteOutputToOutputStream.java new file mode 100644 index 0000000000..a3d27bac5e --- /dev/null +++ b/exist-core/src/main/java/org/exist/storage/io/VariableByteOutputToOutputStream.java @@ -0,0 +1,79 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.storage.io; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Variable Byte Output to an Output Stream. + * + * @author Adam Retter + */ +public class VariableByteOutputToOutputStream extends AbstractVariableByteOutput { + + private OutputStream os; + + public VariableByteOutputToOutputStream(final OutputStream os) { + super(); + this.os = os; + } + + @Override + public void write(final int b) throws IOException { + os.write(b); + } + + @Override + public void write(final byte[] b) throws IOException { + os.write(b); + } + + @Override + public void write(final byte[] b, final int off, final int len) throws IOException { + os.write(b, off, len); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/LocationStep.java b/exist-core/src/main/java/org/exist/xquery/LocationStep.java index b16f78b3a4..31a3bea7d6 100644 --- a/exist-core/src/main/java/org/exist/xquery/LocationStep.java +++ b/exist-core/src/main/java/org/exist/xquery/LocationStep.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -47,6 +71,7 @@ * to the {@link #eval(Sequence, Item)} method) has changed. * * @author wolf + * @author Adam Retter */ public class LocationStep extends Step { @@ -501,6 +526,9 @@ private Sequence getSelf(final XQueryContext context, final Sequence contextSequ if (ns != null) { @Nullable final NodeProxy np = ns.get(p); if (np != null) { + if (p.getMatches() != null) { + np.addMatch(p.getMatches()); + } p = np; } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/AnyURIValue.java b/exist-core/src/main/java/org/exist/xquery/value/AnyURIValue.java index e25227af33..9d01928532 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/AnyURIValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/AnyURIValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,6 +46,9 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.xmldb.XmldbURI; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; @@ -29,16 +56,20 @@ import org.exist.xquery.XPathException; import org.exist.xquery.functions.fn.FunEscapeURI; +import javax.annotation.Nullable; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.ByteBuffer; import java.util.BitSet; /** * @author Wolfgang Meier + * @author Adam Retter */ -public class AnyURIValue extends AtomicValue { +public class AnyURIValue extends AtomicValue { public static final AnyURIValue EMPTY_URI = new AnyURIValue(); static final int caseDiff = ('a' - 'A'); @@ -377,31 +408,36 @@ public int conversionPreference(Class javaClass) { */ @Override public T toJavaObject(final Class target) throws XPathException { - if (target.isAssignableFrom(AnyURIValue.class)) { - return (T) this; - } else if (target == XmldbURI.class) { - return (T) toXmldbURI(); - } else if (target == URI.class) { - return (T) toURI(); - } else if (target == URL.class) { - try { - return (T) new URL(uri); - } catch (final MalformedURLException e) { - throw new XPathException(getExpression(), ErrorCodes.FORG0001, + Throwable throwable = null; + try { + if (target.isAssignableFrom(AnyURIValue.class)) { + return (T) this; + } else if (target == XmldbURI.class) { + return (T) toXmldbURI(); + } else if (target == URI.class) { + return (T) toURI(); + } else if (target == URL.class) { + try { + return (T) new URL(uri); + } catch (final MalformedURLException e) { + throw new XPathException(getExpression(), ErrorCodes.FORG0001, "failed to convert " + uri + " into a Java URL: " + e.getMessage(), e); + } + } else if (target == String.class || target == CharSequence.class) { + return (T) uri; + } else if (target == Object.class) { + return (T) uri; + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serialize()); } - } else if (target == String.class || target == CharSequence.class) { - return (T) uri; - } else if (target == Object.class) { - return (T) uri; + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), - "cannot convert value of type " - + Type.getTypeName(getType()) - + " to Java object of type " - + target.getName()); + throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); } public XmldbURI toXmldbURI() throws XPathException { @@ -457,4 +493,40 @@ private String normalizeEscaped(String in) { } return builder.toString(); } + + /** + * Serializes to a byte array. + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream()) { + vbos.writeUTF(uri); + return vbos.toByteArray(); + } + } + + /** + * Serializes to a ByteBuffer. + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + vbb.writeUTF(uri); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the AnyURIValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the AnyURIValue. + */ + public static AnyURIValue deserialize(@Nullable Expression expression, final ByteBuffer buf) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + final String uri = vbbi.readUTF(); + return new AnyURIValue(expression, uri); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/BinaryValue.java b/exist-core/src/main/java/org/exist/xquery/value/BinaryValue.java index bff4a04ad5..3520d3d94a 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/BinaryValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/BinaryValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,21 +46,25 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; +import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import org.apache.commons.io.output.CloseShieldOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; +import org.exist.util.ByteConversion; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import java.io.*; +import java.nio.ByteBuffer; import static java.nio.charset.StandardCharsets.UTF_8; /** - * @author Adam Retter + * @author Adam Retter */ public abstract class BinaryValue extends AtomicValue implements Closeable { @@ -138,22 +166,24 @@ private int compareTo(BinaryValue otherValue) { } @Override - public T toJavaObject(Class target) throws XPathException { - if (target.isAssignableFrom(getClass())) { - return (T) this; - } + public T toJavaObject(final Class target) throws XPathException { + Throwable throwable = null; + try { + if (target.isAssignableFrom(getClass())) { + return (T) this; - if (target == byte[].class) { - try (final UnsynchronizedByteArrayOutputStream baos = new UnsynchronizedByteArrayOutputStream()) { - streamBinaryTo(baos); - return (T) baos.toByteArray(); - } catch (final IOException ioe) { - LOG.error("Unable to Stream BinaryValue to byte[]: {}", ioe.getMessage(), ioe); - throw new XPathException(getExpression(), "Unable to Stream BinaryValue to byte[]: " + ioe.getMessage(), ioe); + } else if (target == byte[].class) { + return (T) serializeRaw(); + + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serializeRaw()); } + } catch (final IOException e) { + LOG.error("Unable to Stream BinaryValue to byte[]: {}", e.getMessage(), e); + throwable = e; } - throw new XPathException(getExpression(), "Cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName()); + throw new XPathException(getExpression(), "Cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); } /** @@ -306,4 +336,64 @@ public int hashCode() { } return hash; } + + /** + * Serializes to a byte array. + * + * @return the serialized data. + */ + private byte[] serializeRaw() throws IOException { + try (final UnsynchronizedByteArrayOutputStream baos = new UnsynchronizedByteArrayOutputStream()) { + streamBinaryTo(baos); + return baos.toByteArray(); + } + } + + /** + * Serializes to a byte array. + * + * Return value is formatted like: + * byte[0..3] int encoded length of the data + * byte[4..] the data + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final byte[] serialized; + try (final UnsynchronizedByteArrayOutputStream baos = new UnsynchronizedByteArrayOutputStream()) { + // NOTE(AR) leave space for 4 bytes for the data len to be added later + baos.write(0); + baos.write(0); + baos.write(0); + baos.write(0); + + streamBinaryTo(baos); // write the data + + serialized = baos.toByteArray(); + } + + // calculate the length of the data written + final int dataLen = serialized.length - 4; + + ByteConversion.intToByteH(dataLen, serialized, 0); // NOTE(AR) now write the data len + + return serialized; + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the IntegerValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the IntegerValue. + */ + public static BinaryValue deserialize(@Nullable final Expression expression, final BinaryValueType binaryValueType, final ByteBuffer buf) throws XPathException { + final int dataLen = ByteConversion.byteToIntH(buf); + final byte[] data = new byte[dataLen]; + buf.get(data); + + final InputStream is = new UnsynchronizedByteArrayInputStream(data); + return BinaryValueFromInputStream.getInstance(expression != null ? expression.getContext() : null, binaryValueType, is, expression); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/BooleanValue.java b/exist-core/src/main/java/org/exist/xquery/value/BooleanValue.java index 07db3c5b77..94ed5b6f44 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/BooleanValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/BooleanValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -28,8 +52,18 @@ import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class BooleanValue extends AtomicValue { + public static final int SERIALIZED_SIZE = 1; + public final static BooleanValue TRUE = new BooleanValue(true); public final static BooleanValue FALSE = new BooleanValue(false); @@ -48,27 +82,50 @@ public BooleanValue(final Expression expression, boolean bool) { * Returns one of the static fields TRUE or FALSE depending on * the value of the parameter. * - * @param bool the boolean value to map + * @param value the boolean value to map * @return either {@link #TRUE} or {@link #FALSE} */ - public final static BooleanValue valueOf(boolean bool) { - return bool ? TRUE : FALSE; + public static BooleanValue valueOf(final boolean value) { + return value ? TRUE : FALSE; } - /* (non-Javadoc) - * @see org.exist.xquery.value.AtomicValue#getType() + /** + * Returns one of the static fields TRUE or FALSE depending on + * the value of the parameter. + * + * @param expression the calling expression + * @param value the string value to map + * + * @return either {@link #TRUE} or {@link #FALSE} + * + * @throws XPathException if the value cannot be converted to an xs:boolean */ + public static BooleanValue valueOf(@Nullable final Expression expression, String value) throws XPathException { + if (value != null) { + value = value.trim(); + + if("true".equals(value)) { + return BooleanValue.TRUE; + + } else if ("false".equals(value)) { + return BooleanValue.FALSE; + } + } + + throw new XPathException(expression, ErrorCodes.FORG0001, "can not convert '" + value + "' to xs:boolean"); + } + + @Override public int getType() { return Type.BOOLEAN; } - /* (non-Javadoc) - * @see org.exist.xquery.value.Item#getStringValue() - */ + @Override public String getStringValue() throws XPathException { return value ? "true" : "false"; } + @Override public AtomicValue convertTo(final int requiredType) throws XPathException { switch (requiredType) { case Type.BOOLEAN: @@ -113,7 +170,8 @@ public boolean compareTo(final Collator collator, final Comparison operator, fin "cannot convert 'xs:boolean(" + value + ")' to " + Type.getTypeName(other.getType())); } - public int compareTo(Collator collator, AtomicValue other) throws XPathException { + @Override + public int compareTo(final Collator collator, final AtomicValue other) throws XPathException { if (Type.subTypeOf(other.getType(), Type.BOOLEAN)) { final boolean otherVal = other.effectiveBooleanValue(); if (otherVal == value) { @@ -127,9 +185,7 @@ public int compareTo(Collator collator, AtomicValue other) throws XPathException return Constants.INFERIOR; } - /* (non-Javadoc) - * @see org.exist.xquery.value.AtomicValue#effectiveBooleanValue() - */ + @Override public boolean effectiveBooleanValue() throws XPathException { return value; } @@ -138,10 +194,8 @@ public boolean getValue() { return value; } - /* (non-Javadoc) - * @see org.exist.xquery.value.AtomicValue#max(org.exist.xquery.value.AtomicValue) - */ - public AtomicValue max(Collator collator, AtomicValue other) throws XPathException { + @Override + public AtomicValue max(final Collator collator, final AtomicValue other) throws XPathException { if (other.getType() == Type.BOOLEAN) { boolean otherValue = ((BooleanValue) other).value; return value && (!otherValue) ? this : other; @@ -152,7 +206,8 @@ public AtomicValue max(Collator collator, AtomicValue other) throws XPathExcepti } } - public AtomicValue min(Collator collator, AtomicValue other) throws XPathException { + @Override + public AtomicValue min(final Collator collator, final AtomicValue other) throws XPathException { if (other.getType() == Type.BOOLEAN) { final boolean otherValue = ((BooleanValue) other).value; return (!value) && otherValue ? this : other; @@ -163,10 +218,8 @@ public AtomicValue min(Collator collator, AtomicValue other) throws XPathExcepti } } - /* (non-Javadoc) - * @see org.exist.xquery.value.Item#conversionPreference(java.lang.Class) - */ - public int conversionPreference(Class javaClass) { + @Override + public int conversionPreference(final Class javaClass) { if (javaClass.isAssignableFrom(BooleanValue.class)) { return 0; } @@ -183,28 +236,33 @@ public int conversionPreference(Class javaClass) { return Integer.MAX_VALUE; } - /* (non-Javadoc) - * @see org.exist.xquery.value.Item#toJavaObject(java.lang.Class) - */ @Override public T toJavaObject(final Class target) throws XPathException { - if (target.isAssignableFrom(BooleanValue.class)) { - return (T) this; - } else if (target == Boolean.class || target == boolean.class || target == Object.class) { - return (T) Boolean.valueOf(value); - } else if (target == String.class || target == CharSequence.class) { - final StringValue v = (StringValue) convertTo(Type.STRING); - return (T) v.value; + Throwable throwable = null; + try { + if (target.isAssignableFrom(BooleanValue.class)) { + return (T) this; + } else if (target == Boolean.class || target == boolean.class || target == Object.class) { + return (T) Boolean.valueOf(value); + } else if (target == String.class || target == CharSequence.class) { + final StringValue v = (StringValue) convertTo(Type.STRING); + return (T) v.value; + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(getType()) + - " to Java object of type " + target.getName()); + throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); } - /* (non-Javadoc) - * @see java.lang.Comparable#compareTo(java.lang.Object) - */ - public int compareTo(Object o) { + @Override + public int compareTo(final Object o) { final AtomicValue other = (AtomicValue) o; if (Type.subTypeOf(other.getType(), Type.BOOLEAN)) { if (value == ((BooleanValue) other).value) { @@ -226,4 +284,45 @@ public boolean equals(Object obj) { } return false; } + + /** + * Serializes to a ByteBuffer. + * + * 1 byte. + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); + } + + /** + * Serializes to a ByteBuffer. + * + * 1 byte. + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) { + buf.put((byte) (value == true ? 1 : 0)); + } + + /** + * Deserializes from a BooleanValue. + * + * @param expression the expression that creates the BooleanValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the BooleanValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + final byte b = buf.get(); + if (b == 1) { + return BooleanValue.TRUE; + } else { + return BooleanValue.FALSE; + } + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DateTimeStampValue.java b/exist-core/src/main/java/org/exist/xquery/value/DateTimeStampValue.java index d7eb922816..5e9f71e943 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DateTimeStampValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DateTimeStampValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -25,12 +49,18 @@ import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.XMLConstants; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.nio.ByteBuffer; +/** + * @author Radek Hübner + * @author Adam Retter + */ public class DateTimeStampValue extends DateTimeValue { private static final QName XML_SCHEMA_TYPE = new QName(XMLConstants.W3C_XML_SCHEMA_NS_URI, "dateTimeStamp"); @@ -82,4 +112,8 @@ public int getType() { protected QName getXMLSchemaType() { return XML_SCHEMA_TYPE; } + + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + return deserialize(expression, buf, DateTimeStampValue::new); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DateTimeValue.java b/exist-core/src/main/java/org/exist/xquery/value/DateTimeValue.java index fadaac67da..1c87052679 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DateTimeValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DateTimeValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,14 +45,17 @@ */ package org.exist.xquery.value; +import com.evolvedbinary.j8fu.function.BiFunctionE; import org.exist.util.ByteConversion; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.io.IOException; import java.nio.ByteBuffer; import java.time.Instant; import java.util.Date; @@ -39,6 +66,7 @@ * * @author Wolfgang Meier * @author Piotr Kaminski + * @author Adam Retter */ public class DateTimeValue extends AbstractDateTimeValue { @@ -58,7 +86,7 @@ public DateTimeValue(final XMLGregorianCalendar calendar) { this(null, calendar); } - public DateTimeValue(final Expression expression, XMLGregorianCalendar calendar) { + public DateTimeValue(@Nullable final Expression expression, final XMLGregorianCalendar calendar) { super(expression, fillCalendar(cloneXMLGregorianCalendar(calendar))); normalize(); } @@ -81,10 +109,6 @@ public DateTimeValue(final Expression expression, Date date) { normalize(); } - public DateTimeValue(final int year, final int month, final int day, final int hour, final int minute, final int second, final int millisecond, final int timezone) { - super(TimeUtils.getInstance().newXMLGregorianCalendar(year, month, day, hour, minute, second, millisecond, timezone)); - } - public DateTimeValue(String dateTime) throws XPathException { this(null, dateTime); } @@ -196,17 +220,34 @@ public Date getDate() { @Override public T toJavaObject(final Class target) throws XPathException { - if (target == byte[].class) { - final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); - serialize(buf); - return (T) buf.array(); - } else if (target == ByteBuffer.class) { - final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); - serialize(buf); - return (T) buf; - } else { - return super.toJavaObject(target); + Throwable throwable = null; + try { + if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } else { + return super.toJavaObject(target); + } + } catch (final IOException e) { + throwable = e; } + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); + } + + /** + * Serializes to a ByteBuffer. + * + * 13 bytes where: [0-3 (Year), 4 (Month), 5 (Day), 6 (Hour), 7 (Minute), 8 (Second), 9-10 (Milliseconds), 11-12 (Timezone)] + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); } /** @@ -237,7 +278,27 @@ public void serialize(final ByteBuffer buf) { ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); } - public static AtomicValue deserialize(final ByteBuffer buf) { + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the DateTimeValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the DateTimeValue. + */ + public static AtomicValue deserialize(final @Nullable Expression expression, final ByteBuffer buf) throws XPathException { + return deserialize(expression, buf, DateTimeValue::new); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the DateTimeValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the DateTimeValue. + */ + protected static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf, final BiFunctionE cstr) throws XPathException { final int year = ByteConversion.byteToIntH(buf); final int month = buf.get(); final int day = buf.get(); @@ -245,13 +306,14 @@ public static AtomicValue deserialize(final ByteBuffer buf) { final int minute = buf.get(); final int second = buf.get(); - final int ms = ByteConversion.byteToShortH(buf); + final int millisecond = ByteConversion.byteToShortH(buf); int timezone = ByteConversion.byteToShortH(buf); if (timezone == Short.MAX_VALUE) { timezone = DatatypeConstants.FIELD_UNDEFINED; } - return new DateTimeValue(year, month, day, hour, minute, second, ms, timezone); + final XMLGregorianCalendar xmlGregorianCalendar = TimeUtils.getInstance().newXMLGregorianCalendar(year, month, day, hour, minute, second, millisecond, timezone); + return cstr.apply(expression, xmlGregorianCalendar); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DateValue.java b/exist-core/src/main/java/org/exist/xquery/value/DateValue.java index 2701d6da74..a364dfd053 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DateValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DateValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -26,6 +50,7 @@ import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; @@ -35,6 +60,7 @@ /** * @author Wolfgang Meier * @author Piotr Kaminski + * @author Adam Retter */ public class DateValue extends AbstractDateTimeValue { @@ -71,8 +97,8 @@ public DateValue(final Expression expression, XMLGregorianCalendar calendar) thr super(expression, stripCalendar(cloneXMLGregorianCalendar(calendar))); } - public DateValue(final int year, final int month, final int day, final int timezone) { - super(TimeUtils.getInstance().newXMLGregorianCalendarDate(year, month, day, timezone)); + public DateValue(@Nullable final Expression expression, final int year, final int month, final int day, final int timezone) { + super(expression, TimeUtils.getInstance().newXMLGregorianCalendarDate(year, month, day, timezone)); } private static XMLGregorianCalendar stripCalendar(XMLGregorianCalendar calendar) { @@ -141,6 +167,7 @@ public ComputableValue minus(ComputableValue other) throws XPathException { } @Override + @SuppressWarnings("unchecked") public T toJavaObject(final Class target) throws XPathException { if (target == byte[].class) { final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); @@ -150,6 +177,8 @@ public T toJavaObject(final Class target) throws XPathException { final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); serialize(buf); return (T) buf; + } else if (target == Long.class || target == long.class) { + return (T) Long.valueOf(serializeToLong()); } else { return super.toJavaObject(target); } @@ -173,7 +202,7 @@ public void serialize(final ByteBuffer buf) { ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); } - public static AtomicValue deserialize(final ByteBuffer buf) { + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) { final int year = ByteConversion.byteToIntH(buf); final int month = buf.get(); final int day = buf.get(); @@ -183,6 +212,53 @@ public static AtomicValue deserialize(final ByteBuffer buf) { timezone = DatatypeConstants.FIELD_UNDEFINED; } - return new DateValue(year, month, day, timezone); + return new DateValue(expression, year, month, day, timezone); + } + + /** + * Bit-packs a DateValue into a long (64 bits) + * + * @return the long value + */ + public long serializeToLong() { + final int year = calendar.getYear(); + final int month = calendar.getMonth(); + final int day = calendar.getDay(); + int timezone = calendar.getTimezone(); + + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + if (timezone == DatatypeConstants.FIELD_UNDEFINED) { + timezone = Short.MAX_VALUE; + } + + return ((long) year & 0xFFFFFFFFL) << 32 + | ((long) month & 0xFFL) << 24 + | ((long) day & 0xFFL) << 16 + | ((long) timezone & 0xFFFFL); + } + + /** + * Deserializes a DateValue that has been bit-packed into a long (64 bits) + * + * @return the DateValue + */ + public static DateValue deserialize(@Nullable final Expression expression, final long l) { + final int year = (int) (l >>> 32); + final int month = (int) ((l >>> 24) & 0xFFL); + final int day = (int) ((l >>> 16) & 0xFFL); + int timezone = (int) (l & 0xFFFFL); + // manual sign extension as timezone can be negative + timezone = (timezone >= 0x8000) + ? (short)(timezone - 0x10000) + : (short) timezone; + + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + return new DateValue(expression, year, month, day, timezone); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java b/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java index cf960f108f..8f34cd65b5 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,18 +47,25 @@ import net.sf.saxon.tree.util.FastStringBuffer; import net.sf.saxon.value.FloatingPointConverter; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.Duration; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.nio.ByteBuffer; /** * @author Piotr Kaminski + * @author Adam Retter */ public class DayTimeDurationValue extends OrderedDurationValue { @@ -282,4 +313,73 @@ public boolean effectiveBooleanValue() throws XPathException { throw new XPathException(getExpression(), ErrorCodes.FORG0006, "value of type " + Type.getTypeName(getType()) + " has no boolean value."); } + + /** + * Serializes to a byte array. + * + * Return value is formatted like: + * byte[0] sign of the duration, -1, 0, or 1. + * byte[...] VBE BigInteger encoded day + * byte[...] VBE BigInteger encoded hour + * byte[...] VBE BigInteger encoded minute + * byte[...] VBE BigDecimal encoded seconds + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream(37)) { + + vbos.write((byte) duration.getSign()); + serializeBigIntegerField(vbos, DatatypeConstants.DAYS); + serializeBigIntegerField(vbos, DatatypeConstants.HOURS); + serializeBigIntegerField(vbos, DatatypeConstants.MINUTES); + serializeBigDecimalField(vbos, DatatypeConstants.SECONDS); + + return vbos.toByteArray(); + } + } + + /** + * Serializes to a ByteBuffer. + * + * Return value is formatted like: + * byte[0] sign of the duration, -1, 0, or 1. + * byte[...] VBE BigInteger encoded day + * byte[...] VBE BigInteger encoded hour + * byte[...] VBE BigInteger encoded minute + * byte[...] VBE BigDecimal encoded seconds + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + + vbb.write((byte) duration.getSign()); + serializeBigIntegerField(vbb, DatatypeConstants.DAYS); + serializeBigIntegerField(vbb, DatatypeConstants.HOURS); + serializeBigIntegerField(vbb, DatatypeConstants.MINUTES); + serializeBigDecimalField(vbb, DatatypeConstants.SECONDS); + } + + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the DayTimeDurationValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the DayTimeDurationValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + + final boolean isPositive = vbbi.read() != -1; + final BigInteger days = vbbi.readBigInteger(); + final BigInteger hours = vbbi.readBigInteger(); + final BigInteger minutes = vbbi.readBigInteger(); + final BigDecimal seconds = vbbi.readBigDecimal(); + + final Duration duration = TimeUtils.getInstance().newDuration(isPositive, null ,null, days, hours, minutes, seconds); + return new DayTimeDurationValue(expression, duration); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java b/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java index b465de5423..38c455b9a4 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,13 +46,16 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; -import org.exist.util.ByteConversion; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.xquery.Constants; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; import javax.annotation.Nullable; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.math.BigDecimal; @@ -40,11 +67,10 @@ /** * @author wolf + * @author Adam Retter */ public class DecimalValue extends NumericValue { - public static final int SERIALIZED_SIZE = 8; - public static final BigInteger BIG_INTEGER_TEN = BigInteger.valueOf(10); // i × 10^-n where i, n = integers and n >= 0 // All ·minimally conforming· processors ·must· support decimal numbers @@ -551,67 +577,97 @@ public int conversionPreference(Class javaClass) { return Integer.MAX_VALUE; } - /* (non-Javadoc) - * @see org.exist.xquery.value.Item#toJavaObject(java.lang.Class) - */ @Override + @SuppressWarnings("unchecked") public T toJavaObject(final Class target) throws XPathException { - if (target.isAssignableFrom(DecimalValue.class)) { - return (T) this; - } else if (target == BigDecimal.class) { - return (T) value; - } else if (target == Double.class || target == double.class) { - return (T) Double.valueOf(value.doubleValue()); - } else if (target == Float.class || target == float.class) { - return (T) Float.valueOf(value.floatValue()); - } else if (target == Long.class || target == long.class) { - return (T) Long.valueOf(((IntegerValue) convertTo(Type.LONG)).getValue()); - } else if (target == Integer.class || target == int.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.INT); - return (T) Integer.valueOf((int) v.getValue()); - } else if (target == Short.class || target == short.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.SHORT); - return (T) Short.valueOf((short) v.getValue()); - } else if (target == Byte.class || target == byte.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.BYTE); - return (T) Byte.valueOf((byte) v.getValue()); - } else if (target == byte[].class) { - final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); - serialize(buf); - return (T) buf.array(); - } else if (target == ByteBuffer.class) { - final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); - serialize(buf); - return (T) buf; - } else if (target == String.class) { - return (T) getStringValue(); - } else if (target == Boolean.class) { - return (T) Boolean.valueOf(effectiveBooleanValue()); + Throwable throwable = null; + try { + if (target.isAssignableFrom(DecimalValue.class)) { + return (T) this; + } else if (target == BigDecimal.class) { + return (T) value; + } else if (target == Double.class || target == double.class) { + return (T) Double.valueOf(value.doubleValue()); + } else if (target == Float.class || target == float.class) { + return (T) Float.valueOf(value.floatValue()); + } else if (target == Long.class || target == long.class) { + return (T) Long.valueOf(((IntegerValue) convertTo(Type.LONG)).getValue()); + } else if (target == Integer.class || target == int.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.INT); + return (T) Integer.valueOf((int) v.getValue()); + } else if (target == Short.class || target == short.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.SHORT); + return (T) Short.valueOf((short) v.getValue()); + } else if (target == Byte.class || target == byte.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.BYTE); + return (T) Byte.valueOf((byte) v.getValue()); + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serialize()); + } else if (target == String.class) { + return (T) getStringValue(); + } else if (target == Boolean.class) { + return (T) Boolean.valueOf(effectiveBooleanValue()); + } + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), - "cannot convert value of type " - + Type.getTypeName(getType()) - + " to Java object of type " - + target.getName()); + throw new XPathException(getExpression(), "Cannot convert value of type " + Type.getTypeName(getType()) + + " to Java object of type " + target.getName(), throwable); } //End of copy + /** + * Serializes to a byte array. + * + * Return value is formatted like: + * byte[0..] VBE int encoded scale of the big decimal value + * byte[...] VBE int encoded precision of the big decimal value + * byte[...] VBE int encoded length of the following big decimal byte[] value + * byte[...] the big decimal byte[] value + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final byte[] data = value.unscaledValue().toByteArray(); + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream(16)) { + vbos.writeInt(value.scale()); + vbos.writeInt(value.precision()); + vbos.writeInt(data.length); + vbos.write(data); + return vbos.toByteArray(); + } + } + /** * Serializes to a ByteBuffer. * - * 8 bytes. + * Return value is formatted like: + * byte[0..] VBE int encoded scale of the big decimal value + * byte[...] VBE int encoded precision of the big decimal value + * byte[...] VBE int encoded length of the following big decimal byte[] value + * byte[...] the big decimal byte[] value * * @param buf the ByteBuffer to serialize to. */ - public void serialize(final ByteBuffer buf) { - final long ddBits = Double.doubleToLongBits(value.doubleValue()) ^ 0x8000000000000000L; - ByteConversion.longToByte(ddBits, buf); + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + vbb.writeBigDecimal(value); } - public static DecimalValue deserialize(final ByteBuffer buf) { - final long dBits = ByteConversion.byteToLong(buf) ^ 0x8000000000000000L; - final double d = Double.longBitsToDouble(dBits); - return new DecimalValue(d); + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the DecimalValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the DecimalValue. + */ + public static DecimalValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + final BigDecimal value = vbbi.readBigDecimal(); + return new DecimalValue(expression, value); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DoubleValue.java b/exist-core/src/main/java/org/exist/xquery/value/DoubleValue.java index 3cd6cd2409..63d26a0f8f 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DoubleValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DoubleValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -36,6 +60,10 @@ import java.nio.ByteBuffer; import java.util.function.IntSupplier; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class DoubleValue extends NumericValue { public static final int SERIALIZED_SIZE = 8; @@ -74,12 +102,24 @@ public DoubleValue(final String stringValue) throws XPathException { public DoubleValue(final Expression expression, final String stringValue) throws XPathException { super(expression); try { - value = switch (stringValue) { - case "INF" -> Double.POSITIVE_INFINITY; - case "-INF" -> Double.NEGATIVE_INFINITY; - case "NaN" -> Double.NaN; - default -> Double.parseDouble(stringValue); - }; + switch (stringValue) { + case "INF": + this.value = Double.POSITIVE_INFINITY; + break; + + case "-INF": + this.value = Double.NEGATIVE_INFINITY; + break; + + case "NaN": + this.value = Double.NaN; + break; + + default: + this.value = Double.parseDouble(stringValue); + break; + + } } catch (final NumberFormatException e) { throw new XPathException(getExpression(), ErrorCodes.FORG0001, "Cannot construct " + Type.getTypeName(getItemType()) + " from '" + stringValue + "'"); @@ -506,9 +546,9 @@ public void serialize(final ByteBuffer buf) { ByteConversion.longToByte(dBits, buf); } - public static AtomicValue deserialize(final ByteBuffer buf) { + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) { final long bits = ByteConversion.byteToLong(buf) ^ 0x8000000000000000L; final double d = Double.longBitsToDouble(bits); - return new DoubleValue(d); + return new DoubleValue(expression, d); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java b/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java index 192d8bf853..f7147d5257 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,20 +46,28 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; +import org.exist.storage.io.VariableByteOutput; import org.exist.xquery.Constants; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.Duration; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.nio.ByteBuffer; /** * @author Piotr Kaminski + * @author Adam Retter */ public class DurationValue extends ComputableValue { @@ -398,13 +430,23 @@ public int conversionPreference(Class target) { } @Override + @SuppressWarnings("unchecked") public T toJavaObject(Class target) throws XPathException { - if (target.isAssignableFrom(getClass())) { - return (T) this; - } else if (target.isAssignableFrom(Duration.class)) { - return (T) duration; + Throwable throwable = null; + try { + if (target.isAssignableFrom(getClass())) { + return (T) this; + } else if (target.isAssignableFrom(Duration.class)) { + return (T) duration; + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serialize()); + } + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName()); + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); } public boolean effectiveBooleanValue() throws XPathException { @@ -422,4 +464,98 @@ public boolean equals(Object obj) { } return false; } + + /** + * Serializes to a byte array. + * + * Return value is formatted like: + * byte[0] sign of the duration, -1, 0, or 1. + * byte[...] VBE BigInteger encoded year + * byte[...] VBE BigInteger encoded month + * byte[...] VBE BigInteger encoded day + * byte[...] VBE BigInteger encoded hour + * byte[...] VBE BigInteger encoded minute + * byte[...] VBE BigDecimal encoded seconds + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream(37)) { + + vbos.write((byte) duration.getSign()); + serializeBigIntegerField(vbos, DatatypeConstants.YEARS); + serializeBigIntegerField(vbos, DatatypeConstants.MONTHS); + serializeBigIntegerField(vbos, DatatypeConstants.DAYS); + serializeBigIntegerField(vbos, DatatypeConstants.HOURS); + serializeBigIntegerField(vbos, DatatypeConstants.MINUTES); + serializeBigDecimalField(vbos, DatatypeConstants.SECONDS); + + return vbos.toByteArray(); + } + } + + protected void serializeBigIntegerField(final VariableByteOutput vbo, final DatatypeConstants.Field field) throws IOException { + @Nullable BigInteger value = (BigInteger) duration.getField(field); + if (value == null) { + value = BigInteger.ZERO; + } + vbo.writeBigInteger(value); + } + + protected void serializeBigDecimalField(final VariableByteOutput vbo, final DatatypeConstants.Field field) throws IOException { + @Nullable BigDecimal value = (BigDecimal) duration.getField(field); + if (value == null) { + value = BigDecimal.ZERO; + } + vbo.writeBigDecimal(value); + } + + /** + * Serializes to a ByteBuffer. + * + * Return value is formatted like: + * byte[0] sign of the duration, -1, 0, or 1. + * byte[...] VBE BigInteger encoded year + * byte[...] VBE BigInteger encoded month + * byte[...] VBE BigInteger encoded day + * byte[...] VBE BigInteger encoded hour + * byte[...] VBE BigInteger encoded minute + * byte[...] VBE BigDecimal encoded seconds + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + + vbb.write((byte) duration.getSign()); + serializeBigIntegerField(vbb, DatatypeConstants.YEARS); + serializeBigIntegerField(vbb, DatatypeConstants.MONTHS); + serializeBigIntegerField(vbb, DatatypeConstants.DAYS); + serializeBigIntegerField(vbb, DatatypeConstants.HOURS); + serializeBigIntegerField(vbb, DatatypeConstants.MINUTES); + serializeBigDecimalField(vbb, DatatypeConstants.SECONDS); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the DurationValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the DurationValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + + final boolean isPositive = vbbi.read() != -1; + final BigInteger years = vbbi.readBigInteger(); + final BigInteger months = vbbi.readBigInteger(); + final BigInteger days = vbbi.readBigInteger(); + final BigInteger hours = vbbi.readBigInteger(); + final BigInteger minutes = vbbi.readBigInteger(); + final BigDecimal seconds = vbbi.readBigDecimal(); + + final Duration duration = TimeUtils.getInstance().newDuration(isPositive, years,months, days, hours, minutes, seconds); + return new DurationValue(expression, duration); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/FloatValue.java b/exist-core/src/main/java/org/exist/xquery/value/FloatValue.java index 17a59c0714..fc1ec67300 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/FloatValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/FloatValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -38,6 +62,7 @@ /** * @author wolf + * @author Adam Retter */ public class FloatValue extends NumericValue { @@ -549,9 +574,9 @@ public void serialize(final ByteBuffer buf) { ByteConversion.intToByteH(fBits, buf); } - public static FloatValue deserialize(final ByteBuffer buf) { + public static FloatValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) { final int fBits = ByteConversion.byteToIntH(buf) ^ 0x80000000; final float f = Float.intBitsToFloat(fBits); - return new FloatValue(f); + return new FloatValue(expression, f); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/GDayValue.java b/exist-core/src/main/java/org/exist/xquery/value/GDayValue.java index 373e0292b5..3a3ef9be89 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/GDayValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/GDayValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,18 +46,28 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; +import org.exist.util.ByteConversion; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.GregorianCalendar; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class GDayValue extends AbstractDateTimeValue { + public static final int SERIALIZED_SIZE = 3; + public GDayValue() throws XPathException { super(null, stripCalendar(TimeUtils.getInstance().newXMLGregorianCalendar(new GregorianCalendar()))); } @@ -144,4 +178,70 @@ public int compareTo(Collator collator, AtomicValue other) throws XPathException + Type.getTypeName(other.getType())); } + @Override + public T toJavaObject(final Class target) throws XPathException { + Throwable throwable = null; + try { + if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } else { + return super.toJavaObject(target); + } + } catch (final IOException e) { + throwable = e; + } + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); + } + + /** + * Serializes to a ByteBuffer. + * + * 3 bytes where: [0 (Day), 1-2 (Timezone)] + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); + } + + /** + * Serializes to a ByteBuffer. + * + * 3 bytes where: [0 (Day), 1-2 (Timezone)] + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) { + buf.put((byte) calendar.getDay()); + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + final int timezone = calendar.getTimezone(); + ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the GDayValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the GDayValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + final int day = buf.get(); + int timezone = ByteConversion.byteToShortH(buf); + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + final XMLGregorianCalendar xmlGregorianCalendar = TimeUtils.getInstance().newXMLGregorianCalendar(DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, day, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, timezone); + return new GDayValue(expression, xmlGregorianCalendar); + } + } diff --git a/exist-core/src/main/java/org/exist/xquery/value/GMonthDayValue.java b/exist-core/src/main/java/org/exist/xquery/value/GMonthDayValue.java index b81fb399e9..1ce504090e 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/GMonthDayValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/GMonthDayValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,17 +45,27 @@ */ package org.exist.xquery.value; +import org.exist.util.ByteConversion; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.GregorianCalendar; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class GMonthDayValue extends AbstractDateTimeValue { + public static final int SERIALIZED_SIZE = 4; + public GMonthDayValue() throws XPathException { super(null, stripCalendar(TimeUtils.getInstance().newXMLGregorianCalendar(new GregorianCalendar()))); } @@ -108,4 +142,72 @@ public ComputableValue minus(ComputableValue other) throws XPathException { throw new XPathException(getExpression(), "Subtraction is not supported on values of type " + Type.getTypeName(getType())); } + + @Override + public T toJavaObject(final Class target) throws XPathException { + Throwable throwable = null; + try { + if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } else { + return super.toJavaObject(target); + } + } catch (final IOException e) { + throwable = e; + } + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); + } + + /** + * Serializes to a ByteBuffer. + * + * 4 bytes where: [0 (Month), 1 (Day), 2-3 (Timezone)] + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); + } + + /** + * Serializes to a ByteBuffer. + * + * 4 bytes where: [0 (Month), 1 (Day), 2-3 (Timezone)] + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) { + buf.put((byte) calendar.getMonth()); + buf.put((byte) calendar.getDay()); + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + final int timezone = calendar.getTimezone(); + ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the GMonthDayValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the GMonthDayValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + final int month = buf.get(); + final int day = buf.get(); + int timezone = ByteConversion.byteToShortH(buf); + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + final XMLGregorianCalendar xmlGregorianCalendar = TimeUtils.getInstance().newXMLGregorianCalendar(DatatypeConstants.FIELD_UNDEFINED, month, day, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, timezone); + return new GMonthDayValue(expression, xmlGregorianCalendar); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/GMonthValue.java b/exist-core/src/main/java/org/exist/xquery/value/GMonthValue.java index 69e54cf525..6907aef205 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/GMonthValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/GMonthValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,19 +46,29 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; +import org.exist.util.ByteConversion; import org.exist.xquery.Constants; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.GregorianCalendar; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class GMonthValue extends AbstractDateTimeValue { + public static final int SERIALIZED_SIZE = 3; + protected boolean addTrailingZ = false; public GMonthValue() throws XPathException { @@ -187,4 +221,70 @@ public int compareTo(Collator collator, AtomicValue other) throws XPathException "Type error: cannot compare " + Type.getTypeName(getType()) + " to " + Type.getTypeName(other.getType())); } + + @Override + public T toJavaObject(final Class target) throws XPathException { + Throwable throwable = null; + try { + if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } else { + return super.toJavaObject(target); + } + } catch (final IOException e) { + throwable = e; + } + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); + } + + /** + * Serializes to a ByteBuffer. + * + * 3 bytes where: [0 (Month), 1-2 (Timezone)] + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); + } + + /** + * Serializes to a ByteBuffer. + * + * 3 bytes where: [0 (Month), 1-2 (Timezone)] + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) { + buf.put((byte) calendar.getMonth()); + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + final int timezone = calendar.getTimezone(); + ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the GMonthValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the GMonthValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + final int month = buf.get(); + int timezone = ByteConversion.byteToShortH(buf); + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + final XMLGregorianCalendar xmlGregorianCalendar = TimeUtils.getInstance().newXMLGregorianCalendar(DatatypeConstants.FIELD_UNDEFINED, month, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, timezone); + return new GMonthValue(expression, xmlGregorianCalendar); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/GYearMonthValue.java b/exist-core/src/main/java/org/exist/xquery/value/GYearMonthValue.java index 722af98332..6780f58820 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/GYearMonthValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/GYearMonthValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,18 +46,28 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; +import org.exist.util.ByteConversion; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.GregorianCalendar; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class GYearMonthValue extends AbstractDateTimeValue { + public static final int SERIALIZED_SIZE = 7; + public GYearMonthValue() throws XPathException { super(null, stripCalendar(TimeUtils.getInstance().newXMLGregorianCalendar(new GregorianCalendar()))); } @@ -146,4 +180,72 @@ public ComputableValue minus(ComputableValue other) throws XPathException { throw new XPathException(getExpression(), "Subtraction is not supported on values of type " + Type.getTypeName(getType())); } + + @Override + public T toJavaObject(final Class target) throws XPathException { + Throwable throwable = null; + try { + if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } else { + return super.toJavaObject(target); + } + } catch (final IOException e) { + throwable = e; + } + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); + } + + /** + * Serializes to a ByteBuffer. + * + * 7 bytes where: [0-3 (Year), 4 (Month), 5-6 (Timezone)] + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); + } + + /** + * Serializes to a ByteBuffer. + * + * 7 bytes where: [0-3 (Year), 4 (Month), 5-6 (Timezone)] + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) { + ByteConversion.intToByteH(calendar.getYear(), buf); + buf.put((byte) calendar.getMonth()); + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + final int timezone = calendar.getTimezone(); + ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the GYearMonthValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the GYearMonthValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + final int year = ByteConversion.byteToIntH(buf); + final int month = buf.get(); + int timezone = ByteConversion.byteToShortH(buf); + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + final XMLGregorianCalendar xmlGregorianCalendar = TimeUtils.getInstance().newXMLGregorianCalendar(year, month, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, timezone); + return new GYearMonthValue(expression, xmlGregorianCalendar); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/GYearValue.java b/exist-core/src/main/java/org/exist/xquery/value/GYearValue.java index b1f67a4122..8039fa130d 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/GYearValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/GYearValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,17 +45,27 @@ */ package org.exist.xquery.value; +import org.exist.util.ByteConversion; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.GregorianCalendar; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class GYearValue extends AbstractDateTimeValue { + public static final int SERIALIZED_SIZE = 6; + public GYearValue() throws XPathException { super(null, stripCalendar(TimeUtils.getInstance().newXMLGregorianCalendar(new GregorianCalendar()))); } @@ -109,4 +143,70 @@ public ComputableValue minus(ComputableValue other) throws XPathException { throw new XPathException(getExpression(), "Subtraction is not supported on values of type " + Type.getTypeName(getType())); } + + @Override + public T toJavaObject(final Class target) throws XPathException { + Throwable throwable = null; + try { + if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return (T) buf; + } else { + return super.toJavaObject(target); + } + } catch (final IOException e) { + throwable = e; + } + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); + } + + /** + * Serializes to a ByteBuffer. + * + * 6 bytes where: [0-3 (Year), 4-5 (Timezone)] + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); + serialize(buf); + return buf.array(); + } + + /** + * Serializes to a ByteBuffer. + * + * 6 bytes where: [0-3 (Year), 4-5 (Timezone)] + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) { + ByteConversion.intToByteH(calendar.getYear(), buf); + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + final int timezone = calendar.getTimezone(); + ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the GYearValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the GYearValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws XPathException { + final int year = ByteConversion.byteToIntH(buf); + int timezone = ByteConversion.byteToShortH(buf); + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + final XMLGregorianCalendar xmlGregorianCalendar = TimeUtils.getInstance().newXMLGregorianCalendar(year, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, DatatypeConstants.FIELD_UNDEFINED, timezone); + return new GYearValue(expression, xmlGregorianCalendar); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/IntegerValue.java b/exist-core/src/main/java/org/exist/xquery/value/IntegerValue.java index 9bfe550fbc..e1f5867982 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/IntegerValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/IntegerValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -22,7 +46,9 @@ package org.exist.xquery.value; import com.ibm.icu.text.Collator; -import org.exist.util.ByteConversion; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; @@ -41,6 +67,9 @@ * The value space of integer is the infinite set {...,-2,-1,0,1,2,...}. * The base type of integer is decimal. * See http://www.w3.org/TR/xmlschema-2/#integer + * + * @author Wolfgang Meier + * @author Adam Retter */ public class IntegerValue extends NumericValue { @@ -521,41 +550,46 @@ public int conversionPreference(final Class javaClass) { @Override @SuppressWarnings("unchecked") public T toJavaObject(final Class target) throws XPathException { - if (target.isAssignableFrom(IntegerValue.class)) { - return (T) this; - } else if (target == Long.class || target == long.class) { - return (T) Long.valueOf(value.longValue()); - } else if (target == Integer.class || target == int.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.INT); - return (T) Integer.valueOf(v.value.intValue()); - } else if (target == Short.class || target == short.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.SHORT); - return (T) Short.valueOf(v.value.shortValue()); - } else if (target == Byte.class || target == byte.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.BYTE); - return (T) Byte.valueOf(v.value.byteValue()); - } else if (target == Double.class || target == double.class) { - final DoubleValue v = (DoubleValue) convertTo(Type.DOUBLE); - return (T) Double.valueOf(v.getValue()); - } else if (target == Float.class || target == float.class) { - final FloatValue v = (FloatValue) convertTo(Type.FLOAT); - return (T) Float.valueOf(v.value); - } else if (target == Boolean.class || target == boolean.class) { - return (T) new BooleanValue(getExpression(), effectiveBooleanValue()); - } else if (target == byte[].class) { - return (T) serialize(); - } else if (target == ByteBuffer.class) { - return (T) ByteBuffer.wrap(serialize()); - } else if (target == String.class) { - return (T) value.toString(); - } else if (target == BigInteger.class) { - return (T) new BigInteger(value.toByteArray()); - } else if (target == Object.class) { - return (T) value; + Throwable throwable = null; + try { + if (target.isAssignableFrom(IntegerValue.class)) { + return (T) this; + } else if (target == Long.class || target == long.class) { + return (T) Long.valueOf(value.longValue()); + } else if (target == Integer.class || target == int.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.INT); + return (T) Integer.valueOf(v.value.intValue()); + } else if (target == Short.class || target == short.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.SHORT); + return (T) Short.valueOf(v.value.shortValue()); + } else if (target == Byte.class || target == byte.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.BYTE); + return (T) Byte.valueOf(v.value.byteValue()); + } else if (target == Double.class || target == double.class) { + final DoubleValue v = (DoubleValue) convertTo(Type.DOUBLE); + return (T) Double.valueOf(v.getValue()); + } else if (target == Float.class || target == float.class) { + final FloatValue v = (FloatValue) convertTo(Type.FLOAT); + return (T) Float.valueOf(v.value); + } else if (target == Boolean.class || target == boolean.class) { + return (T) new BooleanValue(getExpression(), effectiveBooleanValue()); + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serialize()); + } else if (target == String.class) { + return (T) value.toString(); + } else if (target == BigInteger.class) { + return (T) new BigInteger(value.toByteArray()); + } else if (target == Object.class) { + return (T) value; + } + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(getType()) + - " to Java object of type " + target.getName()); + throw new XPathException(getExpression(), "Cannot convert value of type " + Type.getTypeName(getType()) + + " to Java object of type " + target.getName(), throwable); } @Override @@ -573,37 +607,68 @@ public int hashCode() { return value.hashCode(); } - //TODO(AR) this is not a very good serialization method, the size of the IntegerValue is unbounded and may not fit in 8 bytes. /** * Serializes to a byte array. * - * 8 bytes. + * Return value is formatted like: + * byte[0] indicates the {@link Type} + * byte[...] VBE int encoded length of the following big integer byte[] value + * byte[...] the big integer byte[] value * * @return the serialized data. */ - public byte[] serialize() { - final byte[] buf = new byte[8]; - final long l = value.longValue() - Long.MIN_VALUE; - ByteConversion.longToByte(l, buf, 0); - return buf; + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream(6)) { + vbos.writeByte((byte) (type & 0xFF)); + vbos.writeBigInteger(value); + return vbos.toByteArray(); + } } - //TODO(AR) this is not a very good serialization method, the size of the IntegerValue is unbounded and may not fit in 8 bytes. /** * Serializes to a ByteBuffer. * - * 8 bytes. + * Return value is formatted like: + * byte[0] indicates the {@link Type} + *. byte[...] VBE int encoded length of the following big integer byte[] value + * byte[...] the big integer byte[] value * * @param buf the ByteBuffer to serialize to. */ public void serialize(final ByteBuffer buf) throws IOException { - final long l = value.longValue() - Long.MIN_VALUE; - ByteConversion.longToByte(l, buf); + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + vbb.writeByte((byte) (type & 0xFF)); + vbb.writeBigInteger(value); } - //TODO(AR) this is not a very good deserialization method, the size of the IntegerValue is unbounded and may not fit in 8 bytes. - public static IntegerValue deserialize(final ByteBuffer buf) { - final long l = ByteConversion.byteToLong(buf) ^ 0x8000000000000000L; - return new IntegerValue(l); + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the IntegerValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the IntegerValue. + */ + public static IntegerValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws IOException, XPathException { + return deserialize(expression, buf, null); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the IntegerValue object. + * @param buf the ByteBuffer to deserialize from. + * @param checkType an XDM type to check that matches against the deserialized IntegerValue type. + * + * @return the IntegerValue. + */ + public static IntegerValue deserialize(@Nullable Expression expression, final ByteBuffer buf, @Nullable final Integer checkType) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + final int type = vbbi.read(); + if (checkType != null && type != checkType.intValue()) { + throw new XPathException(expression, "Expected deserialized IntegerValue of type: " + Type.getTypeName(checkType) + ", but found: " + Type.getTypeName(type)); + } + final BigInteger value = vbbi.readBigInteger(); + return new IntegerValue(expression, value, type); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/QNameValue.java b/exist-core/src/main/java/org/exist/xquery/value/QNameValue.java index 05e94c3720..812312890d 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/QNameValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/QNameValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,19 +47,28 @@ import com.ibm.icu.text.Collator; import org.exist.dom.QName; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.xquery.Constants.Comparison; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; + /** * Wrapper class around a {@link org.exist.dom.QName} value which extends * {@link org.exist.xquery.value.AtomicValue}. * * @author wolf + * @author Adam Retter */ + public class QNameValue extends AtomicValue { private final QName qname; @@ -214,19 +247,24 @@ public int conversionPreference(Class javaClass) { */ @Override public T toJavaObject(final Class target) throws XPathException { - if (target.isAssignableFrom(QNameValue.class)) { - return (T) this; - } else if (target == String.class) { - return (T) getStringValue(); - } else if (target == Object.class) { - return (T) qname; + Throwable throwable = null; + try { + if (target.isAssignableFrom(QNameValue.class)) { + return (T) this; + } else if (target == String.class) { + return (T) getStringValue(); + } else if (target == Object.class) { + return (T) qname; + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serialize()); + } + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), - "cannot convert value of type " - + Type.getTypeName(getType()) - + " to Java object of type " - + target.getName()); + throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(getType()) + " to Java object of type " + target.getName(), throwable); } public String toString() { @@ -258,4 +296,45 @@ public boolean equals(Object obj) { public int hashCode() { return qname.hashCode(); } + + /** + * Serializes to a byte array. + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream()) { + vbos.writeUTF(qname.getExtendedStringValue()); + return vbos.toByteArray(); + } + } + + /** + * Serializes to a ByteBuffer. + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + vbb.writeUTF(qname.getExtendedStringValue()); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the AnyURIValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the AnyURIValue. + */ + public static QNameValue deserialize(@Nullable Expression expression, final ByteBuffer buf) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + final String extendedStringValue = vbbi.readUTF(); + try { + final QName qname = QName.parse(extendedStringValue); + return new QNameValue(expression, expression != null ? expression.getContext() : null, qname); + } catch (final QName.IllegalQNameException e) { + throw new XPathException(expression, "Unable to deserialize QNameValue", e); + } + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/StringValue.java b/exist-core/src/main/java/org/exist/xquery/value/StringValue.java index c92621850a..3867baa50d 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/StringValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/StringValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,6 +48,9 @@ import com.ibm.icu.text.Collator; import org.apache.xerces.util.XMLChar; import org.exist.dom.QName; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.util.Collations; import org.exist.util.UTF8; import org.exist.util.XMLCharUtil; @@ -34,11 +61,18 @@ import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.exist.dom.QName.Validity.VALID; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class StringValue extends AtomicValue { public final static StringValue EMPTY_STRING = new StringValue(""); @@ -603,39 +637,47 @@ public int conversionPreference(Class javaClass) { @Override public T toJavaObject(final Class target) throws XPathException { - if (target.isAssignableFrom(StringValue.class)) { - return (T) this; - } else if (target == Object.class || target == String.class || target == CharSequence.class) { - return (T) value; - } else if (target == double.class || target == Double.class) { - final DoubleValue v = (DoubleValue) convertTo(Type.DOUBLE); - return (T) Double.valueOf(v.getValue()); - } else if (target == float.class || target == Float.class) { - final FloatValue v = (FloatValue) convertTo(Type.FLOAT); - return (T) Float.valueOf(v.value); - } else if (target == long.class || target == Long.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.LONG); - return (T) Long.valueOf(v.getInt()); - } else if (target == int.class || target == Integer.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.INT); - return (T) Integer.valueOf(v.getInt()); - } else if (target == short.class || target == Short.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.SHORT); - return (T) Short.valueOf((short) v.getInt()); - } else if (target == byte.class || target == Byte.class) { - final IntegerValue v = (IntegerValue) convertTo(Type.BYTE); - return (T) Byte.valueOf((byte) v.getInt()); - } else if (target == boolean.class || target == Boolean.class) { - return (T) Boolean.valueOf(effectiveBooleanValue()); - } else if (target == char.class || target == Character.class) { - if (value.length() > 1 || value.length() == 0) { - throw new XPathException(getExpression(), "cannot convert string with length = 0 or length > 1 to Java character"); + Throwable throwable = null; + try { + if (target.isAssignableFrom(StringValue.class)) { + return (T) this; + } else if (target == Object.class || target == String.class || target == CharSequence.class) { + return (T) value; + } else if (target == double.class || target == Double.class) { + final DoubleValue v = (DoubleValue) convertTo(Type.DOUBLE); + return (T) Double.valueOf(v.getValue()); + } else if (target == float.class || target == Float.class) { + final FloatValue v = (FloatValue) convertTo(Type.FLOAT); + return (T) Float.valueOf(v.value); + } else if (target == long.class || target == Long.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.LONG); + return (T) Long.valueOf(v.getInt()); + } else if (target == int.class || target == Integer.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.INT); + return (T) Integer.valueOf(v.getInt()); + } else if (target == short.class || target == Short.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.SHORT); + return (T) Short.valueOf((short) v.getInt()); + } else if (target == byte.class || target == Byte.class) { + final IntegerValue v = (IntegerValue) convertTo(Type.BYTE); + return (T) Byte.valueOf((byte) v.getInt()); + } else if (target == boolean.class || target == Boolean.class) { + return (T) Boolean.valueOf(effectiveBooleanValue()); + } else if (target == char.class || target == Character.class) { + if (value.length() > 1 || value.length() == 0) { + throw new XPathException(getExpression(), "cannot convert string with length = 0 or length > 1 to Java character"); + } + return (T) Character.valueOf(value.charAt(0)); + } else if (target == byte[].class) { + return (T) serialize(); + } else if (target == ByteBuffer.class) { + return (T) ByteBuffer.wrap(serialize()); } - return (T) Character.valueOf(value.charAt(0)); + } catch (final IOException e) { + throwable = e; } - throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(type) + - " to Java object of type " + target.getName()); + throw new XPathException(getExpression(), "cannot convert value of type " + Type.getTypeName(type) + " to Java object of type " + target.getName(), throwable); } @Override @@ -804,4 +846,69 @@ public boolean equals(Object obj) { } return value.equals(obj.toString()); } + + /** + * Serializes to a byte array. + * + * Return value is formatted like: + * byte[0] indicates the {@link Type} + * byte[...] VBE int encoded length of the following string + * byte[...] the string as UTF-8 + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream(6)) { + vbos.writeByte((byte) (type & 0xFF)); + vbos.writeUTF(value); + return vbos.toByteArray(); + } + } + + /** + * Serializes to a ByteBuffer. + * + * Return value is formatted like: + * byte[0] indicates the {@link Type} + * byte[...] VBE int encoded length of the following string + * byte[...] the string as UTF-8 + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + vbb.writeByte((byte) (type & 0xFF)); + vbb.writeUTF(value); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the StringValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the StringValue. + */ + public static StringValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws IOException, XPathException { + return deserialize(expression, buf, null); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the StringValue object. + * @param buf the ByteBuffer to deserialize from. + * @param checkType an XDM type to check that matches against the deserialized StringValue type. + * + * @return the StringValue. + */ + public static StringValue deserialize(@Nullable Expression expression, final ByteBuffer buf, @Nullable final Integer checkType) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + final int type = vbbi.read(); + if (checkType != null && type != checkType.intValue()) { + throw new XPathException(expression, "Expected deserialized StringValue of type: " + Type.getTypeName(checkType) + ", but found: " + Type.getTypeName(type)); + } + final String value = vbbi.readUTF(); + return new StringValue(expression, value, type); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/TimeUtils.java b/exist-core/src/main/java/org/exist/xquery/value/TimeUtils.java index d1986b9681..4c51d47075 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/TimeUtils.java +++ b/exist-core/src/main/java/org/exist/xquery/value/TimeUtils.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -98,97 +122,87 @@ public int getLocalTimezoneOffsetMinutes() { return getLocalTimezoneOffsetMillis() / 60000; } - public Duration newDuration(long arg0) { - return factory.newDuration(arg0); + public Duration newDuration(final long durationInMilliSeconds) { + return factory.newDuration(durationInMilliSeconds); } - public Duration newDuration(String arg0) { - return factory.newDuration(arg0); + public Duration newDuration(final String lexicalRepresentation) { + return factory.newDuration(lexicalRepresentation); } - public Duration newDuration( - final boolean isPositive, - final int years, - final int months, - final int days, - final int hours, - final int minutes, - final int seconds) { - + public Duration newDuration(final boolean isPositive, final int years, final int months, final int days, final int hours, final int minutes, final int seconds) { return factory.newDuration(isPositive, years, months, days, hours, minutes, seconds); } - public Duration newDuration(boolean arg0, BigInteger arg1, BigInteger arg2, BigInteger arg3, BigInteger arg4, BigInteger arg5, BigDecimal arg6) { - return factory.newDuration(arg0, arg1, arg2, arg3, arg4, arg5, arg6); + public Duration newDuration(final boolean isPositive, final BigInteger years, final BigInteger months, final BigInteger days, final BigInteger hours, final BigInteger minutes, final BigDecimal seconds) { + return factory.newDuration(isPositive, years, months, days, hours, minutes, seconds); } - public Duration newDurationDayTime(long arg0) { - return factory.newDurationDayTime(arg0); + public Duration newDurationDayTime(final long durationInMilliseconds) { + return factory.newDurationDayTime(durationInMilliseconds); } - public Duration newDurationDayTime(String arg0) { - return factory.newDurationDayTime(arg0); + public Duration newDurationDayTime(final String lexicalRepresentation) { + return factory.newDurationDayTime(lexicalRepresentation); } - public Duration newDurationDayTime(boolean arg0, int arg1, int arg2, int arg3, int arg4) { - return factory.newDurationDayTime(arg0, arg1, arg2, arg3, arg4); + public Duration newDurationDayTime(final boolean isPositive, final int day, final int hour, final int minute, final int second) { + return factory.newDurationDayTime(isPositive, day, hour, minute, second); } - public Duration newDurationDayTime(boolean arg0, BigInteger arg1, BigInteger arg2, BigInteger arg3, BigInteger arg4) { - return factory.newDurationDayTime(arg0, arg1, arg2, arg3, arg4); + public Duration newDurationDayTime(final boolean isPositive, final BigInteger day, final BigInteger hour, final BigInteger minute, final BigInteger second) { + return factory.newDurationDayTime(isPositive, day, hour, minute, second); } - public Duration newDurationYearMonth(long arg0) { - return factory.newDurationYearMonth(arg0); + public Duration newDurationYearMonth(final long durationInMilliseconds) { + return factory.newDurationYearMonth(durationInMilliseconds); } - public Duration newDurationYearMonth(String arg0) { - return factory.newDurationYearMonth(arg0); + public Duration newDurationYearMonth(final String lexicalRepresentation) { + return factory.newDurationYearMonth(lexicalRepresentation); } - public Duration newDurationYearMonth(boolean arg0, int arg1, int arg2) { - return factory.newDurationYearMonth(arg0, arg1, arg2); + public Duration newDurationYearMonth(final boolean isPositive, final int year, final int month) { + return factory.newDurationYearMonth(isPositive, year, month); } - public Duration newDurationYearMonth(boolean arg0, BigInteger arg1, BigInteger arg2) { - return factory.newDurationYearMonth(arg0, arg1, arg2); + public Duration newDurationYearMonth(final boolean isPositive, final BigInteger year, final BigInteger month) { + return factory.newDurationYearMonth(isPositive, year, month); } public XMLGregorianCalendar newXMLGregorianCalendar() { return factory.newXMLGregorianCalendar(); } - public XMLGregorianCalendar newXMLGregorianCalendar(int arg0, int arg1, int arg2, int arg3, int arg4, int arg5, int arg6, int arg7) { - return factory.newXMLGregorianCalendar(arg0, arg1, arg2, arg3, arg4, arg5, - arg6, arg7); + public XMLGregorianCalendar newXMLGregorianCalendar(final int year, final int month, final int day, final int hour, final int minute, final int second, final int millisecond, final int timezone) { + return factory.newXMLGregorianCalendar(year, month, day, hour, minute, second, millisecond, timezone); } - public XMLGregorianCalendar newXMLGregorianCalendar(String lexicalRepresentation) { + public XMLGregorianCalendar newXMLGregorianCalendar(final String lexicalRepresentation) { return factory.newXMLGregorianCalendar(lexicalRepresentation); } - public XMLGregorianCalendar newXMLGregorianCalendar(BigInteger arg0, int arg1, int arg2, int arg3, int arg4, int arg5, BigDecimal arg6, int arg7) { - return factory.newXMLGregorianCalendar(arg0, arg1, arg2, arg3, arg4, arg5, - arg6, arg7); + public XMLGregorianCalendar newXMLGregorianCalendar(final BigInteger year, final int month, final int day, final int hour, final int minute, final int second, final BigDecimal fractionalSecond, final int timezone) { + return factory.newXMLGregorianCalendar(year, month, day, hour, minute, second, fractionalSecond, timezone); } - public XMLGregorianCalendar newXMLGregorianCalendar(GregorianCalendar arg0) { - return factory.newXMLGregorianCalendar(arg0); + public XMLGregorianCalendar newXMLGregorianCalendar(final GregorianCalendar cal) { + return factory.newXMLGregorianCalendar(cal); } - public XMLGregorianCalendar newXMLGregorianCalendarDate(int arg0, int arg1, int arg2, int arg3) { - return factory.newXMLGregorianCalendarDate(arg0, arg1, arg2, arg3); + public XMLGregorianCalendar newXMLGregorianCalendarDate(final int year, final int month, final int day, final int timezone) { + return factory.newXMLGregorianCalendarDate(year, month, day, timezone); } - public XMLGregorianCalendar newXMLGregorianCalendarTime(int arg0, int arg1, int arg2, int arg3) { - return factory.newXMLGregorianCalendarTime(arg0, arg1, arg2, arg3); + public XMLGregorianCalendar newXMLGregorianCalendarTime(final int hours, final int minutes, final int seconds, final int timezone) { + return factory.newXMLGregorianCalendarTime(hours, minutes, seconds, timezone); } - public XMLGregorianCalendar newXMLGregorianCalendarTime(int arg0, int arg1, int arg2, int arg3, int arg4) { - return factory.newXMLGregorianCalendarTime(arg0, arg1, arg2, arg3, arg4); + public XMLGregorianCalendar newXMLGregorianCalendarTime(final int hours, final int minutes, final int seconds, final int milliseconds, final int timezone) { + return factory.newXMLGregorianCalendarTime(hours, minutes, seconds, milliseconds, timezone); } - public XMLGregorianCalendar newXMLGregorianCalendarTime(int arg0, int arg1, int arg2, BigDecimal arg3, int arg4) { - return factory.newXMLGregorianCalendarTime(arg0, arg1, arg2, arg3, arg4); + public XMLGregorianCalendar newXMLGregorianCalendarTime(final int hours, final int minutes, final int seconds, final BigDecimal fractionalSecond, final int timezone) { + return factory.newXMLGregorianCalendarTime(hours, minutes, seconds, fractionalSecond, timezone); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/TimeValue.java b/exist-core/src/main/java/org/exist/xquery/value/TimeValue.java index ae15414c30..dc09293273 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/TimeValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/TimeValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -26,6 +50,7 @@ import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; @@ -35,6 +60,7 @@ /** * @author Wolfgang Meier * @author Piotr Kaminski + * @author Adam Retter */ public class TimeValue extends AbstractDateTimeValue { @@ -71,8 +97,8 @@ public TimeValue(final Expression expression, String timeValue) throws XPathExce } } - public TimeValue(final int hour, final int minute, final int second, final int millisecond, final int timezone) { - super(TimeUtils.getInstance().newXMLGregorianCalendarTime(hour, minute, second, millisecond, timezone)); + public TimeValue(@Nullable final Expression expression, final int hour, final int minute, final int second, final int millisecond, final int timezone) { + super(expression, TimeUtils.getInstance().newXMLGregorianCalendarTime(hour, minute, second, millisecond, timezone)); } private static XMLGregorianCalendar stripCalendar(XMLGregorianCalendar calendar) { calendar = (XMLGregorianCalendar) calendar.clone(); @@ -144,6 +170,8 @@ public T toJavaObject(final Class target) throws XPathException { final ByteBuffer buf = ByteBuffer.allocate(SERIALIZED_SIZE); serialize(buf); return (T) buf; + } else if (target == Long.class || target == long.class) { + return (T) Long.valueOf(serializeToLong()); } else { return super.toJavaObject(target); } @@ -174,7 +202,7 @@ public void serialize(final ByteBuffer buf) { ByteConversion.shortToByteH((short) (timezone == DatatypeConstants.FIELD_UNDEFINED ? Short.MAX_VALUE : timezone), buf); } - public static TimeValue deserialize(final ByteBuffer buf) { + public static TimeValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) { final int hour = buf.get(); final int minute = buf.get(); final int second = buf.get(); @@ -186,6 +214,60 @@ public static TimeValue deserialize(final ByteBuffer buf) { timezone = DatatypeConstants.FIELD_UNDEFINED; } - return new TimeValue(hour, minute, second, ms, timezone); + return new TimeValue(expression, hour, minute, second, ms, timezone); + } + + /** + * Bit-packs a TimeValue into a long (64 bits) + * + * @return the long value + */ + public long serializeToLong() { + final int hour = calendar.getHour(); + final int minute = calendar.getMinute(); + final int second = calendar.getSecond(); + int ms = calendar.getMillisecond(); + if (ms == DatatypeConstants.FIELD_UNDEFINED) { + ms = 0; + } + + int timezone = calendar.getTimezone(); + + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + if (timezone == DatatypeConstants.FIELD_UNDEFINED) { + timezone = Short.MAX_VALUE; + } + + return ((long) hour & 0xFFL) << 48 + | ((long) minute & 0xFFL) << 40 + | ((long) second & 0xFFL) << 32 + | ((long) ms & 0xFFFFL) << 16 + | ((long) timezone & 0xFFFFL); + } + + /** + * Deserializes a TimeValue that has been bit-packed into a long (64 bits) + * + * @return the TimeValue + */ + public static TimeValue deserialize(@Nullable final Expression expression, final long l) { + final int hour = (int) (l >>> 48); + final int minute = (int) ((l >>> 40) & 0xFFL); + final int second = (int) ((l >>> 32) & 0xFFL); + final int ms = (int) ((l >>> 16) & 0xFFFFL); + int timezone = (int) (l & 0xFFFFL); + // manual sign extension as timezone can be negative + timezone = (timezone >= 0x8000) + ? (short)(timezone - 0x10000) + : (short) timezone; + + // values for timezone range from -14*60 to 14*60, so we can use a short, but + // need to choose a different value for FIELD_UNDEFINED, which is not the same as 0 (= UTC) + if (timezone == Short.MAX_VALUE) { + timezone = DatatypeConstants.FIELD_UNDEFINED; + } + + return new TimeValue(expression, hour, minute, second, ms, timezone); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/YearMonthDurationValue.java b/exist-core/src/main/java/org/exist/xquery/value/YearMonthDurationValue.java index 4c54b223ed..3095a4e4c9 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/YearMonthDurationValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/YearMonthDurationValue.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,20 +45,29 @@ */ package org.exist.xquery.value; +import org.exist.storage.io.VariableByteArrayOutputStream; +import org.exist.storage.io.VariableByteBufferInput; +import org.exist.storage.io.VariableByteBufferOutput; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; +import javax.annotation.Nullable; import javax.xml.datatype.DatatypeConstants; import javax.xml.datatype.Duration; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; /** * @author Piotr Kaminski + * @author Adam Retter */ public class YearMonthDurationValue extends OrderedDurationValue { + public static final int SERIALIZED_SIZE = 9; + public static final Duration CANONICAL_ZERO_DURATION = TimeUtils.getInstance().newDuration(true, null, BigInteger.ZERO, null, null, null, null); @@ -228,4 +261,62 @@ public boolean effectiveBooleanValue() throws XPathException { "value of type " + Type.getTypeName(getType()) + " has no boolean value."); } + + /** + * Serializes to a byte array. + * + * Return value is formatted like: + * byte[0] sign of the duration, -1, 0, or 1. + * byte[...] VBE BigInteger encoded year + * byte[...] VBE BigInteger encoded month + * + * @return the serialized data. + */ + public byte[] serialize() throws IOException { + try (final VariableByteArrayOutputStream vbos = new VariableByteArrayOutputStream(37)) { + + vbos.write((byte) duration.getSign()); + serializeBigIntegerField(vbos, DatatypeConstants.YEARS); + serializeBigIntegerField(vbos, DatatypeConstants.MONTHS); + + return vbos.toByteArray(); + } + } + + /** + * Serializes to a ByteBuffer. + * + * Return value is formatted like: + * byte[0] sign of the duration, -1, 0, or 1. + * byte[...] VBE BigInteger encoded year + * byte[...] VBE BigInteger encoded month + * + * @param buf the ByteBuffer to serialize to. + */ + public void serialize(final ByteBuffer buf) throws IOException { + final VariableByteBufferOutput vbb = new VariableByteBufferOutput(buf); + + vbb.write((byte) duration.getSign()); + serializeBigIntegerField(vbb, DatatypeConstants.YEARS); + serializeBigIntegerField(vbb, DatatypeConstants.MONTHS); + } + + /** + * Deserializes from a ByteBuffer. + * + * @param expression the expression that creates the YearMonthDurationValue object. + * @param buf the ByteBuffer to deserialize from. + * + * @return the YearMonthDurationValue. + */ + public static AtomicValue deserialize(@Nullable final Expression expression, final ByteBuffer buf) throws IOException, XPathException { + final VariableByteBufferInput vbbi = new VariableByteBufferInput(buf); + + final boolean isPositive = vbbi.read() != -1; + final BigInteger years = vbbi.readBigInteger(); + final BigInteger months = vbbi.readBigInteger(); + + final Duration duration = TimeUtils.getInstance().newDuration(isPositive, years, months, null, null, null, null); + return new YearMonthDurationValue(duration); + } } diff --git a/exist-core/src/test/java/org/exist/dom/persistent/SymbolTableTest.java b/exist-core/src/test/java/org/exist/dom/persistent/SymbolTableTest.java index 6480218c91..bade12edc0 100644 --- a/exist-core/src/test/java/org/exist/dom/persistent/SymbolTableTest.java +++ b/exist-core/src/test/java/org/exist/dom/persistent/SymbolTableTest.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -26,12 +50,12 @@ import org.exist.storage.BrokerPoolServiceException; import org.exist.storage.io.VariableByteInput; import org.easymock.Capture; -import org.exist.storage.io.VariableByteOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.exist.storage.io.VariableByteOutput; import org.exist.util.Configuration; import org.junit.Test; import org.junit.runner.RunWith; @@ -148,7 +172,7 @@ public void write_and_read_are_balanced() throws IOException, BrokerPoolServiceE final SymbolTable symbolTable = createSymbolTable(createTempDir()); symbolTable.getSymbol("some-name"); - VariableByteOutputStream mockOs = createMock(VariableByteOutputStream.class); + VariableByteOutput mockOs = createMock(VariableByteOutput.class); VariableByteInput mockIs = createMock(VariableByteInput.class); final Capture byteCapture = newCapture(); diff --git a/exist-core/src/test/java/org/exist/security/SimpleACLPermissionTest.java b/exist-core/src/test/java/org/exist/security/SimpleACLPermissionTest.java index 3bbd16aede..535f7ce91c 100644 --- a/exist-core/src/test/java/org/exist/security/SimpleACLPermissionTest.java +++ b/exist-core/src/test/java/org/exist/security/SimpleACLPermissionTest.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,17 +47,17 @@ import com.googlecode.junittoolbox.ParallelRunner; import org.exist.storage.DBBroker; -import org.exist.storage.io.VariableByteInputStream; import java.io.IOException; -import org.exist.storage.io.VariableByteOutputStream; + +import org.exist.storage.io.VariableByteArrayInput; import org.exist.Database; import org.exist.security.ACLPermission.ACE_TARGET; import org.exist.security.ACLPermission.ACE_ACCESS_TYPE; import org.exist.security.internal.SecurityManagerImpl; import java.util.Random; import org.easymock.EasyMock; -import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; +import org.exist.storage.io.VariableByteArrayOutputStream; import org.junit.Test; import org.junit.runner.RunWith; @@ -682,7 +706,7 @@ public void roundtrip_write_read() throws PermissionDeniedException, IOException final int mode2 = Permission.READ; permission.addGroupACE(ACE_ACCESS_TYPE.DENIED, groupId2, mode2); - final VariableByteOutputStream os = new VariableByteOutputStream(); + final VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(); //write the acl out permission.write(os); @@ -708,7 +732,7 @@ public void roundtrip_write_read() throws PermissionDeniedException, IOException SimpleACLPermission permission2 = new SimpleACLPermission(mockSecurityManager); //read the acl in - permission2.read(new VariableByteInputStream(new UnsynchronizedByteArrayInputStream(data))); + permission2.read(new VariableByteArrayInput(data)); assertEquals(2, permission2.getACECount()); diff --git a/exist-core/src/test/java/org/exist/security/UnixStylePermissionTest.java b/exist-core/src/test/java/org/exist/security/UnixStylePermissionTest.java index 7b1f139313..6a2d4b9fd8 100644 --- a/exist-core/src/test/java/org/exist/security/UnixStylePermissionTest.java +++ b/exist-core/src/test/java/org/exist/security/UnixStylePermissionTest.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -33,7 +57,7 @@ import org.exist.security.internal.RealmImpl; import org.exist.security.internal.SecurityManagerImpl; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; import org.exist.util.SyntaxException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -58,7 +82,7 @@ public void writeRead_roundtrip() throws IOException { final int mode = 0700; final int ownerGroupId = new Random().nextInt(); - final VariableByteOutputStream mockOstream = EasyMock.createMock(VariableByteOutputStream.class); + final VariableByteOutput mockOstream = EasyMock.createMock(VariableByteOutput.class); final VariableByteInput mockIstream = EasyMock.createMock(VariableByteInput.class); final TestableUnixStylePermission permission = new TestableUnixStylePermission(mockSecurityManager, ownerId, ownerGroupId, mode); diff --git a/exist-core/src/test/java/org/exist/storage/io/VariableByteStreamTest.java b/exist-core/src/test/java/org/exist/storage/io/VariableByteStreamTest.java index f0488545ae..883819584a 100644 --- a/exist-core/src/test/java/org/exist/storage/io/VariableByteStreamTest.java +++ b/exist-core/src/test/java/org/exist/storage/io/VariableByteStreamTest.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -47,7 +71,7 @@ public void setUp() { @Test public void inOutLong() throws IOException { - VariableByteOutputStream os = new VariableByteOutputStream(); + VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(); for(int i = 0; i < SIZE * 3; i++) { os.writeLong(values[i++]); os.writeInt((int)values[i++]); @@ -74,7 +98,7 @@ public void copyTo() throws IOException { Random rand = new Random(System.currentTimeMillis()); int valuesWritten = 0; int dataLen = 0; - VariableByteOutputStream os = new VariableByteOutputStream(); + VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(); for(int i = 0; i < 1000; i++) { int count = rand.nextInt(0xfff); os.writeShort((short)count); @@ -92,7 +116,7 @@ public void copyTo() throws IOException { int valuesCopied = 0; dataLen = 0; VariableByteArrayInput is = new VariableByteArrayInput(data); - os = new VariableByteOutputStream(); + os = new VariableByteArrayOutputStream(); while(is.available() > 0) { int count = is.readShort(); boolean skip = rand.nextBoolean(); diff --git a/exist-core/src/test/java/org/exist/storage/journal/AbstractJournalTest.java b/exist-core/src/test/java/org/exist/storage/journal/AbstractJournalTest.java index 137647c1ee..2d082531b9 100644 --- a/exist-core/src/test/java/org/exist/storage/journal/AbstractJournalTest.java +++ b/exist-core/src/test/java/org/exist/storage/journal/AbstractJournalTest.java @@ -49,13 +49,13 @@ import org.exist.storage.dom.RemovePageLoggable; import org.exist.storage.dom.WriteOverflowPageLoggable; import org.exist.storage.index.StoreValueLoggable; -import org.exist.storage.io.VariableByteInputStream; +import org.exist.storage.io.VariableByteArrayInput; +import org.exist.storage.io.VariableByteInput; import org.exist.storage.lock.Lock; import org.exist.storage.txn.*; import org.exist.test.ExistEmbeddedServer; import org.exist.test.TestConstants; import org.exist.util.*; -import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import org.exist.xmldb.XmldbURI; import org.junit.After; import org.junit.Ignore; @@ -1886,7 +1886,7 @@ protected boolean equalsStoreValue(final StoreValueLoggable o) { } try { - final VariableByteInputStream vis = new VariableByteInputStream(new UnsynchronizedByteArrayInputStream(o.getValue())); + final VariableByteInput vis = new VariableByteArrayInput(o.getValue()); final int thatDocId = vis.readInt(); final String thatDocName = vis.readUTF(); @@ -1947,7 +1947,7 @@ protected boolean equalsRemoveValue(final org.exist.storage.index.RemoveValueLog } try { - final VariableByteInputStream vis = new VariableByteInputStream(new UnsynchronizedByteArrayInputStream(o.getOldData(), o.getOffset(), o.getLen())); + final VariableByteInput vis = new VariableByteArrayInput(o.getOldData(), o.getOffset(), o.getLen()); final int thatDocId = vis.readInt(); final String thatDocName = vis.readUTF(); diff --git a/exist-core/src/test/java/org/exist/util/sorters/SortTestNodeId.java b/exist-core/src/test/java/org/exist/util/sorters/SortTestNodeId.java index b74324336d..417a2b272c 100644 --- a/exist-core/src/test/java/org/exist/util/sorters/SortTestNodeId.java +++ b/exist-core/src/test/java/org/exist/util/sorters/SortTestNodeId.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -19,13 +43,12 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ - package org.exist.util.sorters; import java.io.IOException; import org.exist.numbering.NodeId; -import org.exist.storage.io.VariableByteOutputStream; +import org.exist.storage.io.VariableByteOutput; /** * Mock NodeId. @@ -135,11 +158,11 @@ public int units() { throw new UnsupportedOperationException(); } - public void write(VariableByteOutputStream arg0) { + public void write(VariableByteOutput arg0) { throw new UnsupportedOperationException(); } - public NodeId write(NodeId arg0, VariableByteOutputStream arg1) + public NodeId write(NodeId arg0, VariableByteOutput arg1) throws IOException { throw new UnsupportedOperationException(); } diff --git a/extensions/indexes/lucene/pom.xml b/extensions/indexes/lucene/pom.xml index d2824a3fdd..11b72bc034 100644 --- a/extensions/indexes/lucene/pom.xml +++ b/extensions/indexes/lucene/pom.xml @@ -202,7 +202,12 @@
${project.parent.relativePath}/../../elemental-parent/elemental-LGPL-21-ONLY-license.template.txt
pom.xml - src/** + src/main/** + src/test/java/** + src/test/resources/** + src/test/resources-filtered/** + src/test/xquery/lucene/*.xml + src/test/xquery/lucene/*.xql @@ -224,14 +229,21 @@ src/test/resources-filtered/conf.xml src/test/resources/log4j2.xml src/main/java/org/exist/indexing/lucene/AbstractFieldConfig.java + src/main/java/org/exist/indexing/lucene/AbstractTextExtractor.java src/main/java/org/exist/indexing/lucene/AnalyzerConfig.java + src/main/java/org/exist/indexing/lucene/DefaultTextExtractor.java src/main/java/org/exist/indexing/lucene/FieldType.java src/main/java/org/exist/indexing/lucene/LuceneConfig.java src/main/java/org/exist/indexing/lucene/LuceneFacetConfig.java src/main/java/org/exist/indexing/lucene/LuceneFieldConfig.java src/main/java/org/exist/indexing/lucene/LuceneIndex.java src/main/java/org/exist/indexing/lucene/LuceneIndexConfig.java + src/main/java/org/exist/indexing/lucene/LuceneIndexWorker.java + src/main/java/org/exist/indexing/lucene/LuceneMatchListener.java + src/main/java/org/exist/indexing/lucene/TextExtractor.java src/main/java/org/exist/indexing/lucene/XMLToQuery.java + src/main/java/org/exist/xquery/modules/lucene/Field.java + src/main/java/org/exist/xquery/modules/lucene/LuceneModule.java @@ -244,15 +256,23 @@ pom.xml src/test/resources-filtered/conf.xml src/test/resources/log4j2.xml + src/test/xquery/lucene/fields.xqm src/main/java/org/exist/indexing/lucene/AbstractFieldConfig.java + src/main/java/org/exist/indexing/lucene/AbstractTextExtractor.java src/main/java/org/exist/indexing/lucene/AnalyzerConfig.java + src/main/java/org/exist/indexing/lucene/DefaultTextExtractor.java src/main/java/org/exist/indexing/lucene/FieldType.java src/main/java/org/exist/indexing/lucene/LuceneConfig.java src/main/java/org/exist/indexing/lucene/LuceneFacetConfig.java src/main/java/org/exist/indexing/lucene/LuceneFieldConfig.java src/main/java/org/exist/indexing/lucene/LuceneIndex.java src/main/java/org/exist/indexing/lucene/LuceneIndexConfig.java + src/main/java/org/exist/indexing/lucene/LuceneIndexWorker.java + src/main/java/org/exist/indexing/lucene/LuceneMatchListener.java + src/main/java/org/exist/indexing/lucene/TextExtractor.java src/main/java/org/exist/indexing/lucene/XMLToQuery.java + src/main/java/org/exist/xquery/modules/lucene/Field.java + src/main/java/org/exist/xquery/modules/lucene/LuceneModule.java diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractFieldConfig.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractFieldConfig.java index 2cf6d27f8a..42c264fdce 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractFieldConfig.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractFieldConfig.java @@ -75,6 +75,7 @@ * on an arbitrary XQuery expression. * * @author Wolfgang Meier + * @author Adam Retter */ public abstract class AbstractFieldConfig { @@ -117,16 +118,16 @@ public Analyzer getAnalyzer() { return null; } - protected abstract void processResult(Sequence result, Document luceneDoc) throws XPathException; + protected abstract void processResult(final Sequence result, final Map prefixToNamespaceMappings, final Document luceneDoc) throws XPathException; - protected abstract void processText(CharSequence text, Document luceneDoc); + protected abstract void processText(final NodeId nodeId, final CharSequence text, final Map prefixToNamespaceMappings, final Document luceneDoc); - protected abstract void build(DBBroker broker, DocumentImpl document, NodeId nodeId, Document luceneDoc, CharSequence text); + protected abstract void build( final DBBroker broker, final DocumentImpl document, final NodeId nodeId, final Document luceneDoc, final CharSequence text, final Map prefixToNamespaceMappings); - protected void doBuild(DBBroker broker, DocumentImpl document, NodeId nodeId, Document luceneDoc, CharSequence text) + protected void doBuild(final DBBroker broker, final DocumentImpl document, final NodeId nodeId, final Document luceneDoc, final CharSequence text, final Map prefixToNamespaceMappings) throws PermissionDeniedException, XPathException { if (!expression.isPresent()) { - processText(text, luceneDoc); + processText(nodeId, text, prefixToNamespaceMappings, luceneDoc); return; } @@ -142,7 +143,7 @@ protected void doBuild(DBBroker broker, DocumentImpl document, NodeId nodeId, Do Sequence result = xquery.execute(broker, compiled, currentNode); if (!result.isEmpty()) { - processResult(result, luceneDoc); + processResult(result, prefixToNamespaceMappings, luceneDoc); } } catch (PermissionDeniedException | XPathException e) { isValid = false; diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractTextExtractor.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractTextExtractor.java index cb342fe216..0e5225592e 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractTextExtractor.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/AbstractTextExtractor.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,23 +47,40 @@ import org.exist.util.XMLString; +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.Map; + +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public abstract class AbstractTextExtractor implements TextExtractor { - protected LuceneConfig config; - protected LuceneIndexConfig idxConfig; + protected final LuceneConfig config; + protected final LuceneIndexConfig idxConfig; + protected final Map prefixToNamespaceMappings; protected XMLString buffer = new XMLString(); - public void configure(LuceneConfig config, LuceneIndexConfig idxConfig) { + public AbstractTextExtractor(final LuceneConfig config, final LuceneIndexConfig idxConfig, @Nullable final Map prefixToNamespaceMappings) { this.config = config; this.idxConfig = idxConfig; + this.prefixToNamespaceMappings = prefixToNamespaceMappings != null ? prefixToNamespaceMappings : Collections.emptyMap(); } + @Override public LuceneIndexConfig getIndexConfig() { return idxConfig; } + @Override public XMLString getText() { return buffer; } + + @Override + public Map getPrefixToNamespaceMappings() { + return prefixToNamespaceMappings; + } } diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/DefaultTextExtractor.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/DefaultTextExtractor.java index 33c7b01d38..c37ee8d9b8 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/DefaultTextExtractor.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/DefaultTextExtractor.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,12 +48,24 @@ import org.exist.dom.QName; import org.exist.util.XMLString; +import javax.annotation.Nullable; +import java.util.Map; + +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class DefaultTextExtractor extends AbstractTextExtractor { private int stack = 0; private boolean addSpaceBeforeNext = false; - - public int startElement(QName name) { + + public DefaultTextExtractor(final LuceneConfig config, final LuceneIndexConfig idxConfig, @Nullable final Map prefixToNamespaceMappings) { + super(config, idxConfig, prefixToNamespaceMappings); + } + + @Override + public int startElement(final QName name) { if (isInlineNode(name)) { // discard not yet applied whitespaces addSpaceBeforeNext = false; @@ -52,6 +88,7 @@ private boolean isInlineNode(final QName name) { return (config.isInlineNode(name) || (idxConfig != null && idxConfig.isInlineNode(name))); } + @Override public int endElement(final QName name) { if (isIgnoredNode(name)) { stack--; @@ -62,6 +99,7 @@ public int endElement(final QName name) { return 0; } + @Override public int beforeCharacters() { if (addSpaceBeforeNext && !buffer.isEmpty() && buffer.charAt(buffer.length() - 1) != ' ') { // separate the previous element's text from following text @@ -71,8 +109,9 @@ public int beforeCharacters() { } return 0; } - - public int characters(XMLString text) { + + @Override + public int characters(final XMLString text) { if (stack == 0) { buffer.append(text); return text.length(); diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFacetConfig.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFacetConfig.java index 669f34adef..fb2272dfc1 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFacetConfig.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFacetConfig.java @@ -46,6 +46,7 @@ package org.exist.indexing.lucene; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.facet.FacetField; import org.exist.dom.persistent.DocumentImpl; import org.exist.numbering.NodeId; @@ -69,6 +70,7 @@ * A facet has a dimension and content returned by an XQuery expression. * * @author Wolfgang Meier + * @author Adam Retter */ public class LuceneFacetConfig extends AbstractFieldConfig { @@ -98,7 +100,7 @@ public String getDimension() { } @Override - protected void processResult(final Sequence result, final Document luceneDoc) throws XPathException { + protected void processResult(final Sequence result, final Map prefixToNamespaceMappings, final Document luceneDoc) throws XPathException { if (isHierarchical) { // hierarchical facets may be multi-valued, so if we receive an array, // create one hierarchical facet for each member @@ -135,15 +137,16 @@ private void createHierarchicalFacet(Document luceneDoc, Sequence seq) throws XP } @Override - protected void processText(CharSequence text, Document luceneDoc) { + protected void processText(final NodeId nodeId, final CharSequence text, final Map prefixToNamespaceMappings, final Document luceneDoc) { if (!text.isEmpty()) { - luceneDoc.add(new FacetField(dimension, text.toString())); + final Field field = new FacetField(dimension, text.toString()); + luceneDoc.add(field); } } - public void build(DBBroker broker, DocumentImpl document, NodeId nodeId, Document luceneDoc, CharSequence text) { + public void build(final DBBroker broker, final DocumentImpl document, final NodeId nodeId, final Document luceneDoc, final CharSequence text, final Map prefixToNamespaceMappings) { try { - doBuild(broker, document, nodeId, luceneDoc, text); + doBuild(broker, document, nodeId, luceneDoc, text, prefixToNamespaceMappings); } catch (PermissionDeniedException e) { LOG.warn("Permission denied while evaluating expression for facet '{}': {}", dimension, expression, e); } catch (XPathException e) { diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFieldConfig.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFieldConfig.java index 2cfeda2717..2d302d2e29 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFieldConfig.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneFieldConfig.java @@ -48,11 +48,13 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.*; import org.apache.lucene.util.BytesRef; +import org.exist.dom.QName; import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.persistent.NodeProxy; import org.exist.numbering.NodeId; import org.exist.security.PermissionDeniedException; import org.exist.storage.DBBroker; +import org.exist.storage.ElementValue; import org.exist.util.DatabaseConfigurationException; import org.exist.xquery.CompiledXQuery; import org.exist.xquery.XPathException; @@ -62,7 +64,10 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.xml.datatype.XMLGregorianCalendar; +import javax.xml.XMLConstants; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Map; import java.util.Optional; @@ -78,9 +83,15 @@ * A field may also be associated with an analyzer, could have a type and may be stored or not. * * @author Wolfgang Meier + * @author Adam Retter */ public class LuceneFieldConfig extends AbstractFieldConfig { + private static final BigDecimal BD_DOUBLE_MAX_VALUE = BigDecimal.valueOf(Double.MAX_VALUE); + private static final BigDecimal BD_DOUBLE_MIN_VALUE = BigDecimal.valueOf(Double.MIN_VALUE); + private static final BigInteger BI_LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE); + private static final BigInteger BI_LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE); + private static final String ATTR_FIELD_NAME = "name"; private static final String ATTR_TYPE = "type"; private static final String ATTR_BINARY = "binary"; @@ -91,7 +102,7 @@ public class LuceneFieldConfig extends AbstractFieldConfig { protected String fieldName; protected int type = Type.STRING; protected boolean binary = false; - protected boolean store = true; + protected Field.Store store = Field.Store.YES; protected Analyzer analyzer= null; protected Optional condition = Optional.empty(); protected CompiledXQuery compiledCondition = null; @@ -115,7 +126,7 @@ public class LuceneFieldConfig extends AbstractFieldConfig { final String storeStr = configElement.getAttribute(ATTR_STORE); if (!storeStr.isEmpty()) { - this.store = storeStr.equalsIgnoreCase("yes") || storeStr.equalsIgnoreCase("true"); + this.store = (storeStr.equalsIgnoreCase("yes") || storeStr.equalsIgnoreCase("true")) ? Field.Store.YES : Field.Store.NO; } final String analyzerOpt = configElement.getAttribute(ATTR_ANALYZER); @@ -147,10 +158,10 @@ public Analyzer getAnalyzer() { } @Override - protected void build(DBBroker broker, DocumentImpl document, NodeId nodeId, Document luceneDoc, CharSequence text) { + protected void build(final DBBroker broker, final DocumentImpl document, final NodeId nodeId, final Document luceneDoc, final CharSequence text, final Map prefixToNamespaceMappings) { try { if (checkCondition(broker, document, nodeId)) { - doBuild(broker, document, nodeId, luceneDoc, text); + doBuild(broker, document, nodeId, luceneDoc, text, prefixToNamespaceMappings); } } catch (XPathException e) { LOG.warn("XPath error while evaluating expression for field named '{}': {}: {}", fieldName, expression, e.getMessage(), e); @@ -186,10 +197,11 @@ private boolean checkCondition(DBBroker broker, DocumentImpl document, NodeId no } @Override - protected void processResult(Sequence result, Document luceneDoc) throws XPathException { - for (SequenceIterator i = result.unorderedIterator(); i.hasNext(); ) { - final String text = i.nextItem().getStringValue(); - final Field field = binary ? convertToDocValue(text) : convertToField(text); + protected void processResult(final Sequence result, final Map prefixToNamespaceMappings, final Document luceneDoc) throws XPathException { + for (final SequenceIterator i = result.unorderedIterator(); i.hasNext(); ) { + final Item item = i.nextItem(); + final String content = item.getStringValue(); + final Field field = binary ? convertToDocValue(content, prefixToNamespaceMappings) : convertToField(content, prefixToNamespaceMappings); if (field != null) { luceneDoc.add(field); } @@ -197,141 +209,327 @@ protected void processResult(Sequence result, Document luceneDoc) throws XPathEx } @Override - protected void processText(CharSequence text, Document luceneDoc) { + protected void processText(final NodeId nodeId, final CharSequence text, final Map prefixToNamespaceMappings, final Document luceneDoc) { final Field field; if (binary) { - field = convertToDocValue(text.toString()); + field = convertToDocValue(text.toString(), prefixToNamespaceMappings); } else { - field = convertToField(text.toString()); + field = convertToField(text.toString(), prefixToNamespaceMappings); } if (field != null) { luceneDoc.add(field); } } - private Field convertToField(String content) { + private @Nullable Field convertToField(final String content, final Map prefixToNamespaceMappings) { try { switch (type) { + case Type.INTEGER: + case Type.NON_POSITIVE_INTEGER: + case Type.NEGATIVE_INTEGER: + case Type.NON_NEGATIVE_INTEGER: + case Type.POSITIVE_INTEGER: + final BigInteger integerValue = new IntegerValue(content, type).toJavaObject(BigInteger.class); + // NOTE(AR) we can only store this as a Java `long` type in Lucene, so we have to check if it will fit! + if (integerValue.subtract(BI_LONG_MAX_VALUE).compareTo(BigInteger.ZERO) > 0 || integerValue.subtract(BI_LONG_MIN_VALUE).compareTo(BigInteger.ZERO) < 0) { + LOG.warn("Field {} has an xs:integer value outside of the range {} to {}, this is unsupported due to limitations with Lucene 4.10.4. Content was: {}", fieldName, Long.MIN_VALUE, Long.MAX_VALUE, content); + return null; + } + return new LongField(fieldName, integerValue.longValue(), store == Field.Store.YES ? LongField.TYPE_STORED : LongField.TYPE_NOT_STORED); + case Type.LONG: case Type.UNSIGNED_LONG: - long lvalue = Long.parseLong(content); - return new LongField(fieldName, lvalue, LongField.TYPE_STORED); + final long longValue = new IntegerValue(content, type).getLong(); + return new LongField(fieldName, longValue, store == Field.Store.YES ? LongField.TYPE_STORED : LongField.TYPE_NOT_STORED); + case Type.INT: case Type.UNSIGNED_INT: case Type.SHORT: case Type.UNSIGNED_SHORT: - int ivalue = Integer.parseInt(content); - return new IntField(fieldName, ivalue, IntField.TYPE_STORED); + case Type.BYTE: + case Type.UNSIGNED_BYTE: + final int intValue = new IntegerValue(content, type).getInt(); + return new IntField(fieldName, intValue, store == Field.Store.YES ? IntField.TYPE_STORED : IntField.TYPE_NOT_STORED); + case Type.DECIMAL: + final BigDecimal bigDecimal = new DecimalValue(content).toJavaObject(BigDecimal.class); + // NOTE(AR) we can only store this as a Java `double` type in Lucene, so we have to check if it will fit! + if (bigDecimal.subtract(BD_DOUBLE_MAX_VALUE).compareTo(BigDecimal.ZERO) > 0 || bigDecimal.subtract(BD_DOUBLE_MIN_VALUE).compareTo(BigDecimal.ZERO) < 0) { + LOG.warn("Field {} has an xs:decimal value outside of the range {} to {}, this is unsupported due to limitations with Lucene 4.10.4. Content was: {}", fieldName, Double.MIN_VALUE, Double.MAX_VALUE, content); + return null; + } + return new DoubleField(fieldName, bigDecimal.doubleValue(), store == Field.Store.YES ? DoubleField.TYPE_STORED : DoubleField.TYPE_NOT_STORED); + case Type.DOUBLE: - double dvalue = Double.parseDouble(content); - return new DoubleField(fieldName, dvalue, DoubleField.TYPE_STORED); + final double doubleValue = new DoubleValue(content).getDouble(); + return new DoubleField(fieldName, doubleValue, store == Field.Store.YES ? DoubleField.TYPE_STORED : DoubleField.TYPE_NOT_STORED); + case Type.FLOAT: - float fvalue = Float.parseFloat(content); - return new FloatField(fieldName, fvalue, FloatField.TYPE_STORED); + final float floatValue = new FloatValue(content).getFloat(); + return new FloatField(fieldName, floatValue, store == Field.Store.YES ? FloatField.TYPE_STORED : FloatField.TYPE_NOT_STORED); + case Type.DATE: - DateValue dv = new DateValue(content); - long dl = dateToLong(dv); - return new LongField(fieldName, dl, LongField.TYPE_STORED); + final DateValue dateValue = new DateValue(content); + final long longDateValue = dateValue.toJavaObject(long.class); + return new LongField(fieldName, longDateValue, store == Field.Store.YES ? LongField.TYPE_STORED : LongField.TYPE_NOT_STORED); + case Type.TIME: - TimeValue tv = new TimeValue(content); - long tl = timeToLong(tv); - return new LongField(fieldName, tl, LongField.TYPE_STORED); + final TimeValue timeValue = new TimeValue(content); + final long longTimeValue = timeValue.toJavaObject(long.class); + return new LongField(fieldName, longTimeValue, store == Field.Store.YES ? LongField.TYPE_STORED : LongField.TYPE_NOT_STORED); + case Type.DATE_TIME: - DateTimeValue dtv = new DateTimeValue(content); - String dateStr = dateTimeToString(dtv); - return new TextField(fieldName, dateStr, Field.Store.YES); + final DateTimeValue dateTimeValue = new DateTimeValue(content); + return new TextField(fieldName, dateTimeValue.toString(), store); + + case Type.DATE_TIME_STAMP: + final DateTimeStampValue dateTimeStampValue = new DateTimeStampValue(content); + return new TextField(fieldName, dateTimeStampValue.toString(), store); + + case Type.DURATION: + final DurationValue durationValue = new DurationValue(content); + return new TextField(fieldName, durationValue.toString(), store); + + case Type.YEAR_MONTH_DURATION: + final YearMonthDurationValue yearMonthDurationValue = new YearMonthDurationValue(content); + return new TextField(fieldName, yearMonthDurationValue.toString(), store); + + case Type.DAY_TIME_DURATION: + final DayTimeDurationValue dayTimeDurationValue = new DayTimeDurationValue(content); + return new TextField(fieldName, dayTimeDurationValue.toString(), store); + + case Type.G_YEAR_MONTH: + final GYearMonthValue gYearMonthValue = new GYearMonthValue(content); + return new TextField(fieldName, gYearMonthValue.toString(), store); + + case Type.G_YEAR: + final GYearValue gYearValue = new GYearValue(content); + return new TextField(fieldName, gYearValue.toString(), store); + + case Type.G_MONTH_DAY: + final GMonthDayValue gMonthDayValue = new GMonthDayValue(content); + return new TextField(fieldName, gMonthDayValue.toString(), store); + + case Type.G_MONTH: + final GMonthValue gMonthValue = new GMonthValue(content); + return new TextField(fieldName, gMonthValue.toString(), store); + + case Type.G_DAY: + final GDayValue gDayValue = new GDayValue(content); + return new TextField(fieldName, gDayValue.toString(), store); + + case Type.BOOLEAN: + final BooleanValue booleanValue = BooleanValue.valueOf(null, content); + return new IntField(fieldName, booleanValue.getValue() ? 1 : 0, store); + + case Type.BASE64_BINARY: + final BinaryValue base64Binary = new BinaryValueFromBinaryString(new Base64BinaryValueType(), content); + return new TextField(fieldName, base64Binary.getStringValue(), store); + + case Type.HEX_BINARY: + final BinaryValue hexBinary = new BinaryValueFromBinaryString(new HexBinaryValueType(), content); + return new TextField(fieldName, hexBinary.getStringValue(), store); + + case Type.ANY_URI: + final AnyURIValue anyURIValue = new AnyURIValue(content); + return new TextField(fieldName, anyURIValue.getStringValue(), store); + + case Type.QNAME: + final QNameValue qnameValue = getQNameValue(content, prefixToNamespaceMappings); + return new TextField(fieldName, qnameValue.getQName().getExtendedStringValue(), store); + + case Type.STRING: + case Type.NORMALIZED_STRING: + case Type.TOKEN: + case Type.LANGUAGE: + case Type.NMTOKEN: + case Type.NAME: + case Type.NCNAME: + case Type.ID: + case Type.IDREF: + case Type.ENTITY: + final StringValue stringValue = new StringValue(content, type); + return new TextField(fieldName, stringValue.getStringValue(), store); + + case Type.NOTATION: default: - return new TextField(fieldName, content, store ? Field.Store.YES : Field.Store.NO); + // NOTE(AR) report inability to index value + LOG.warn("Cannot convert field {} to type {}. Content was: {}", fieldName, Type.getTypeName(type), content); } - } catch (NumberFormatException | XPathException e) { - // wrong type: ignore - LOG.trace("Cannot convert field {} to type {}. Content was: {}", fieldName, Type.getTypeName(type), content); + } catch (final NumberFormatException | XPathException | QName.IllegalQNameException e) { + // NOTE(AR) report inability to index value + LOG.warn("Cannot convert field {} to type {}. Content was: {}. Error was: {}", fieldName, Type.getTypeName(type), content, e.getMessage()); } return null; } - private Field convertToDocValue(final String content) { + private @Nullable Field convertToDocValue(final String content, final Map prefixToNamespaceMappings) { try { - return switch (type) { - case Type.TIME -> { - final TimeValue timeValue = new TimeValue(content); - yield new BinaryDocValuesField(fieldName, new BytesRef(timeValue.toJavaObject(byte[].class))); - } - case Type.DATE_TIME -> { - final DateTimeValue dateTimeValue = new DateTimeValue(content); - yield new BinaryDocValuesField(fieldName, new BytesRef(dateTimeValue.toJavaObject(byte[].class))); - } - case Type.DATE -> { - final DateValue dateValue = new DateValue(content); - yield new BinaryDocValuesField(fieldName, new BytesRef(dateValue.toJavaObject(byte[].class))); - } - case Type.INTEGER, Type.LONG, Type.UNSIGNED_LONG, Type.INT, Type.UNSIGNED_INT, Type.SHORT, Type.UNSIGNED_SHORT -> { - final IntegerValue iv = new IntegerValue(content, Type.INTEGER); - yield new BinaryDocValuesField(fieldName, new BytesRef(iv.serialize())); - } - case Type.DOUBLE -> { + final BytesRef bytesRef; + switch (type) { + + case Type.INTEGER: + case Type.NON_POSITIVE_INTEGER: + case Type.NEGATIVE_INTEGER: + case Type.LONG: + case Type.INT: + case Type.SHORT: + case Type.BYTE: + case Type.NON_NEGATIVE_INTEGER: + case Type.UNSIGNED_LONG: + case Type.UNSIGNED_INT: + case Type.UNSIGNED_SHORT: + case Type.UNSIGNED_BYTE: + case Type.POSITIVE_INTEGER: + final IntegerValue iv = new IntegerValue(content, type); + bytesRef = new BytesRef(iv.toJavaObject(byte[].class)); + break; + + case Type.DECIMAL: + final DecimalValue dv = new DecimalValue(content); + bytesRef = new BytesRef(dv.toJavaObject(byte[].class)); + break; + + case Type.DOUBLE: final DoubleValue dbv = new DoubleValue(content); - yield new BinaryDocValuesField(fieldName, new BytesRef(dbv.toJavaObject(byte[].class))); - } - case Type.FLOAT -> { + bytesRef = new BytesRef(dbv.toJavaObject(byte[].class)); + break; + + case Type.FLOAT: final FloatValue fv = new FloatValue(content); - yield new BinaryDocValuesField(fieldName, new BytesRef(fv.toJavaObject(byte[].class))); - } - case Type.DECIMAL -> { - final DecimalValue dv = new DecimalValue(content); - yield new BinaryDocValuesField(fieldName, new BytesRef(dv.toJavaObject(byte[].class))); - } - - // everything else treated as string - default -> new BinaryDocValuesField(fieldName, new BytesRef(content)); - }; - } catch (final NumberFormatException | XPathException e) { - // wrong type: ignore - LOG.error("Cannot convert field {} to type {}. Content was: {}", fieldName, Type.getTypeName(type), content); - return null; - } - } + bytesRef = new BytesRef(fv.toJavaObject(byte[].class)); + break; - private static long dateToLong(DateValue date) { - final XMLGregorianCalendar utccal = date.calendar.normalize(); - return ((long)utccal.getYear() << 16) + ((long)utccal.getMonth() << 8) + ((long)utccal.getDay()); - } + case Type.DATE: + final DateValue dateValue = new DateValue(content); + bytesRef = new BytesRef(dateValue.toJavaObject(byte[].class)); + break; - private static long timeToLong(TimeValue time) { - return time.getTimeInMillis(); - } + case Type.TIME: + final TimeValue timeValue = new TimeValue(content); + bytesRef = new BytesRef(timeValue.toJavaObject(byte[].class)); + break; - private static String dateTimeToString(DateTimeValue dtv) { - final XMLGregorianCalendar utccal = dtv.calendar.normalize(); - final StringBuilder sb = new StringBuilder(); - formatNumber(utccal.getMillisecond(), 3, sb); - formatNumber(utccal.getSecond(), 2, sb); - formatNumber(utccal.getMinute(), 2, sb); - formatNumber(utccal.getHour(), 2, sb); - formatNumber(utccal.getDay(), 2, sb); - formatNumber(utccal.getMonth(), 2, sb); - formatNumber(utccal.getYear(), 4, sb); - return sb.toString(); - } + case Type.DATE_TIME: + final DateTimeValue dateTimeValue = new DateTimeValue(content); + bytesRef = new BytesRef(dateTimeValue.toJavaObject(byte[].class)); + break; + + case Type.DATE_TIME_STAMP: + final DateTimeStampValue dateTimeStampValue = new DateTimeStampValue(content); + bytesRef = new BytesRef(dateTimeStampValue.toJavaObject(byte[].class)); + break; + + case Type.DURATION: + final DurationValue durationValue = new DurationValue(content); + bytesRef = new BytesRef(durationValue.toJavaObject(byte[].class)); + break; - private static void formatNumber(int number, int digits, StringBuilder sb) { - int count = 0; - long n = number; - while (n > 0) { - final int digit = '0' + (int)n % 10; - sb.insert(0, (char)digit); - count++; - if (count == digits) { - break; + case Type.YEAR_MONTH_DURATION: + final YearMonthDurationValue yearMonthDurationValue = new YearMonthDurationValue(content); + bytesRef = new BytesRef(yearMonthDurationValue.toJavaObject(byte[].class)); + break; + + case Type.DAY_TIME_DURATION: + final DayTimeDurationValue dayTimeDurationValue = new DayTimeDurationValue(content); + bytesRef = new BytesRef(dayTimeDurationValue.toJavaObject(byte[].class)); + break; + + case Type.G_YEAR_MONTH: + final GYearMonthValue gYearMonthValue = new GYearMonthValue(content); + bytesRef = new BytesRef(gYearMonthValue.toJavaObject(byte[].class)); + break; + + case Type.G_YEAR: + final GYearValue gYearValue = new GYearValue(content); + bytesRef = new BytesRef(gYearValue.toJavaObject(byte[].class)); + break; + + case Type.G_MONTH_DAY: + final GMonthDayValue gMonthDayValue = new GMonthDayValue(content); + bytesRef = new BytesRef(gMonthDayValue.toJavaObject(byte[].class)); + break; + + case Type.G_MONTH: + final GMonthValue gMonthValue = new GMonthValue(content); + bytesRef = new BytesRef(gMonthValue.toJavaObject(byte[].class)); + break; + + case Type.G_DAY: + final GDayValue gDayValue = new GDayValue(content); + bytesRef = new BytesRef(gDayValue.toJavaObject(byte[].class)); + break; + + case Type.BOOLEAN: + final BooleanValue booleanValue = BooleanValue.valueOf(null, content); + bytesRef = new BytesRef(booleanValue.toJavaObject(byte[].class)); + break; + + case Type.BASE64_BINARY: + final BinaryValue base64BinaryValue = new BinaryValueFromBinaryString(new Base64BinaryValueType(), content); + bytesRef = new BytesRef(base64BinaryValue.serialize()); + break; + + case Type.HEX_BINARY: + final BinaryValue hexBinaryValue = new BinaryValueFromBinaryString(new HexBinaryValueType(), content); + bytesRef = new BytesRef(hexBinaryValue.serialize()); + break; + + case Type.ANY_URI: + final AnyURIValue anyURIValue = new AnyURIValue(content); + bytesRef = new BytesRef(anyURIValue.toJavaObject(byte[].class)); + break; + + case Type.QNAME: + final QNameValue qnameValue = getQNameValue(content, prefixToNamespaceMappings); + bytesRef = new BytesRef(qnameValue.toJavaObject(byte[].class)); + break; + + case Type.STRING: + case Type.NORMALIZED_STRING: + case Type.TOKEN: + case Type.LANGUAGE: + case Type.NMTOKEN: + case Type.NAME: + case Type.NCNAME: + case Type.ID: + case Type.IDREF: + case Type.ENTITY: + final StringValue stringValue = new StringValue(content, type); + bytesRef = new BytesRef(stringValue.toJavaObject(byte[].class)); + break; + + case Type.NOTATION: + default: + // NOTE(AR) report inability to index value + LOG.warn("Cannot convert field {} to type {}. Content was: {}", fieldName, Type.getTypeName(type), content); + return null; } - n = n / 10; + + return new BinaryDocValuesField(fieldName, bytesRef); + + } catch (final NumberFormatException | XPathException | QName.IllegalQNameException | IOException e) { + // NOTE(AR) report inability to index value + LOG.warn("Cannot convert field {} to type {}. Content was: {}. Error was: {}", fieldName, Type.getTypeName(type), content, e.getMessage()); + return null; } - if (count < digits) { - for (int i = count; i < digits; i++) { - sb.insert(0, '0'); + } + + private static QNameValue getQNameValue(final String content, final Map prefixToNamespaceMappings) throws XPathException, QName.IllegalQNameException { + final QName qname; + final String qnameLocalName = QName.extractLocalName(content); + @Nullable final String qnamePrefix = QName.extractPrefix(content); + if (qnamePrefix != null) { + @Nullable final String qnameNamespace = prefixToNamespaceMappings.get(qnamePrefix); + if (qnameNamespace == null) { + throw new XPathException("Lucene index module: Missing namespace declaration for qname value in field config"); } + qname = new QName(qnameLocalName, qnameNamespace, qnamePrefix, ElementValue.ATTRIBUTE); + } else { + qname = new QName(qnameLocalName, XMLConstants.NULL_NS_URI, null, ElementValue.ATTRIBUTE); } + + return new QNameValue(null, qname); } } diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneIndexWorker.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneIndexWorker.java index d6ed42ae98..0a963bba73 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneIndexWorker.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneIndexWorker.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -85,6 +109,7 @@ * @author Wolfgang Meier * @author Dannes Wessels * @author Leif-Jöran Olsson + * @author Adam Retter */ public class LuceneIndexWorker implements OrderedValuesIndex, QNamedKeysIndex { @@ -128,6 +153,8 @@ public class LuceneIndexWorker implements OrderedValuesIndex, QNamedKeysIndex { private final StreamListener listener = new LuceneStreamListener(); + private final InScopeNamespaces inScopeNamespaces = new InScopeNamespaces(); + public LuceneIndexWorker(LuceneIndex parent, DBBroker broker) { this.index = parent; this.broker = broker; @@ -881,11 +908,15 @@ public String getFieldContent(int docId, String field) throws IOException { }); } - public IndexableField[] getField(final int docId, final String field) throws IOException { - final Set fields = ObjectArraySet.of(field); + public @Nullable IndexableField[] getField(final int docId, final String field) throws IOException { return index.withReader(reader -> { + final Set fields = ObjectArraySet.of(field); final Document doc = reader.document(docId, fields); - return doc.getFields(field); + final IndexableField[] indexableFields = doc.getFields(field); + if (indexableFields.length > 0) { + return indexableFields; + } + return null; }); } @@ -1316,9 +1347,10 @@ private void addOccurrence(TreeMap map, String term, int fr * @param path the node path * @param config the lucene index config * @param content the content of the node + * @param prefixToNamespaceMappings any prefix to namespace mappings that are in-scope */ - protected void indexText(final NodeId nodeId, final QName qname, final NodePath path, final LuceneIndexConfig config, final CharSequence content) { - final PendingDoc pending = new PendingDoc(nodeId, qname, path, content, config.getBoost(), config); + protected void indexText(final NodeId nodeId, final QName qname, final NodePath path, final LuceneIndexConfig config, final CharSequence content, final Map prefixToNamespaceMappings) { + final PendingDoc pending = new PendingDoc(nodeId, qname, path, content, prefixToNamespaceMappings, config.getBoost(), config); addPending(pending); } @@ -1332,9 +1364,10 @@ protected void indexText(final NodeId nodeId, final QName qname, final NodePath * @param path the path of the node * @param config the lucene index config * @param content the content of the node + * @param prefixToNamespaceMappings any prefix to namespace mappings that are in-scope */ - protected void indexText(final java.util.Collection attribs, final NodeId nodeId, final QName qname, final NodePath path, final LuceneIndexConfig config, final CharSequence content) { - final PendingDoc pending = new PendingDoc(nodeId, qname, path, content, config.getAttrBoost(attribs), config); + protected void indexText(final java.util.Collection attribs, final NodeId nodeId, final QName qname, final NodePath path, final LuceneIndexConfig config, final CharSequence content, final Map prefixToNamespaceMappings) { + final PendingDoc pending = new PendingDoc(nodeId, qname, path, content, prefixToNamespaceMappings, config.getAttrBoost(attribs), config); addPending(pending); } @@ -1346,11 +1379,36 @@ private void addPending(final PendingDoc pending) { } } - private record PendingDoc(NodeId nodeId, QName qname, NodePath path, CharSequence text, float boost, - LuceneIndexConfig idxConf) { + private static class PendingDoc { + final NodeId nodeId; + final QName qname; + final NodePath path; + final CharSequence text; + final Map prefixToNamespaceMappings; + final float boost; + final LuceneIndexConfig idxConf; + + public PendingDoc(final NodeId nodeId, final QName qname, final NodePath path, final CharSequence text, final Map prefixToNamespaceMappings, final float boost, final LuceneIndexConfig idxConf) { + this.nodeId = nodeId; + this.qname = qname; + this.path = path; + this.text = text; + this.prefixToNamespaceMappings = prefixToNamespaceMappings; + this.boost = boost; + this.idxConf = idxConf; + } } - private record PendingAttr(AttrImpl attr, NodePath path, LuceneIndexConfig conf) { + private static class PendingAttr { + final AttrImpl attr; + final NodePath path; + final LuceneIndexConfig conf; + + private PendingAttr(final AttrImpl attr, final NodePath path, final LuceneIndexConfig conf) { + this.attr = attr; + this.path = path; + this.conf = conf; + } } private void write() { @@ -1376,13 +1434,13 @@ private void write() { // docId also needs to be indexed IntField fDocIdIdx = new IntField(FIELD_DOC_ID, 0, IntField.TYPE_NOT_STORED); - for (PendingDoc pending : nodesToWrite) { + for (final PendingDoc pending : nodesToWrite) { final Document doc = new Document(); - List facetConfigs = pending.idxConf.getFacetsAndFields(); - facetConfigs.forEach(config -> - config.build(broker, currentDoc, pending.nodeId, doc, pending.text) + final List fieldAndFacetConfigs = pending.idxConf.getFacetsAndFields(); + fieldAndFacetConfigs.forEach(config -> + config.build(broker, currentDoc, pending.nodeId, doc, pending.text, pending.prefixToNamespaceMappings) ); fDocId.setLongValue(currentDoc.getDocId()); @@ -1490,10 +1548,28 @@ public void startElement(Txn transaction, ElementImpl element, NodePath path) { } currentElement = element; + // push in-scope namespaces + @Nullable Map nsMappings = element.getNamespaceMappings(); + if (nsMappings != null) { + nsMappings = new HashMap<>(nsMappings); // clone the map + } + final QName qname = element.getQName(); + if (qname.hasNamespace()) { + if (nsMappings != null) { + nsMappings.put(qname.getPrefix(), qname.getNamespaceURI()); + } else { + nsMappings = Collections.singletonMap(qname.getPrefix(), qname.getNamespaceURI()); + } + } + if (nsMappings == null) { + nsMappings = Collections.emptyMap(); + } + inScopeNamespaces.push(nsMappings); + if (mode == ReindexMode.STORE && config != null) { if (contentStack != null) { for (final TextExtractor extractor : contentStack) { - extractor.startElement(element.getQName()); + extractor.startElement(qname); } } @@ -1505,8 +1581,7 @@ public void startElement(Txn transaction, ElementImpl element, NodePath path) { while (configIter.hasNext()) { LuceneIndexConfig configuration = configIter.next(); if (configuration.match(path)) { - TextExtractor extractor = new DefaultTextExtractor(); - extractor.configure(config, configuration); + final TextExtractor extractor = new DefaultTextExtractor(config, configuration, inScopeNamespaces.getPrefixToNamespaceMappings()); contentStack.push(extractor); } } @@ -1546,13 +1621,13 @@ public void endElement(Txn transaction, ElementImpl element, NodePath path) { attributes.add((AttrImpl) attributes1.item(i)); } } - indexText(attributes, element.getNodeId(), element.getQName(), path, extractor.getIndexConfig(), extractor.getText()); + indexText(attributes, element.getNodeId(), element.getQName(), path, extractor.getIndexConfig(), extractor.getText(), extractor.getPrefixToNamespaceMappings()); if (wasEmpty) { attributes.clear(); } } else { // no attribute matching, index normally - indexText(element.getNodeId(), element.getQName(), path, extractor.getIndexConfig(), extractor.getText()); + indexText(element.getNodeId(), element.getQName(), path, extractor.getIndexConfig(), extractor.getText(), extractor.getPrefixToNamespaceMappings()); } } } @@ -1561,6 +1636,8 @@ public void endElement(Txn transaction, ElementImpl element, NodePath path) { } indexPendingAttrs(); + // pop in-scope namespaces + inScopeNamespaces.pop(); currentElement = null; super.endElement(transaction, element, path); @@ -1592,7 +1669,7 @@ public void attribute(Txn transaction, AttrImpl attrib, NodePath path) { if (configuration.shouldReindexOnAttributeChange()) { appendAttrToBeIndexedLater(attribCopy, new NodePath(path), configuration); } else { - indexText(attrib.getNodeId(), attrib.getQName(), path, configuration, attrib.getValue()); + indexText(attrib.getNodeId(), attrib.getQName(), path, configuration, attrib.getValue(), inScopeNamespaces.getPrefixToNamespaceMappings()); } } } @@ -1643,7 +1720,7 @@ private void indexPendingAttrs() { if (mode == ReindexMode.STORE && config != null) { for (PendingAttr pending : pendingAttrs) { AttrImpl attr = pending.attr; - indexText(attributes, attr.getNodeId(), attr.getQName(), pending.path, pending.conf, attr.getValue()); + indexText(attributes, attr.getNodeId(), attr.getQName(), pending.path, pending.conf, attr.getValue(), inScopeNamespaces.getPrefixToNamespaceMappings()); } } } finally { @@ -1663,5 +1740,39 @@ private void releaseAttributes() { } } + private static class InScopeNamespaces { + private final Deque> prefixToNamespace; + + public InScopeNamespaces() { + this.prefixToNamespace = new ArrayDeque<>(); + } + + public void push(final Map prefixToNamespaceMappings) { + this.prefixToNamespace.push(prefixToNamespaceMappings); + } + + public void pop() { + this.prefixToNamespace.pop(); + } + + public Map getPrefixToNamespaceMappings() { + if (prefixToNamespace.isEmpty()) { + return Collections.emptyMap(); + } + + final Map mappings = new HashMap<>(); + final Iterator> it = prefixToNamespace.descendingIterator(); + while (it.hasNext()) { + final Map map = it.next(); + mappings.putAll(map); + } + return mappings; + } + + public void clear() { + prefixToNamespace.clear(); + } + } + } diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneMatchListener.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneMatchListener.java index 0f08a79222..04aa77f63a 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneMatchListener.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneMatchListener.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -57,6 +81,10 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.util.AttributeSource.State; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class LuceneMatchListener extends AbstractMatchListener { private static final Logger LOG = LogManager.getLogger(LuceneMatchListener.class); @@ -179,8 +207,7 @@ private void scanMatches(final NodeProxy p) { if(idxConf == null) { return; // there is no index config so there can not be any matches } - final TextExtractor extractor = new DefaultTextExtractor(); - extractor.configure(config, idxConf); + final TextExtractor extractor = new DefaultTextExtractor(config, idxConf, null); final OffsetList offsets = new OffsetList(); int level = 0; diff --git a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/TextExtractor.java b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/TextExtractor.java index feb068ec1a..3c43e3602a 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/TextExtractor.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/TextExtractor.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,24 +48,29 @@ import org.exist.dom.QName; import org.exist.util.XMLString; +import java.util.Map; + /** * Extract text from an XML fragment to be indexed with Lucene. * This interface provides an additional abstraction to handle whitespace * between elements or ignore certain elements. + * + * @author Wolfgang Meier + * @author Adam Retter */ public interface TextExtractor { - public void configure(LuceneConfig config, LuceneIndexConfig idxConfig); + int startElement(QName name); - public int startElement(QName name); + int endElement(QName name); - public int endElement(QName name); - - public int beforeCharacters(); + int beforeCharacters(); - public int characters(XMLString value); + int characters(XMLString value); - public LuceneIndexConfig getIndexConfig(); + LuceneIndexConfig getIndexConfig(); - public XMLString getText(); + XMLString getText(); + + Map getPrefixToNamespaceMappings(); } diff --git a/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/Field.java b/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/Field.java index 5fa259d0a3..1b49f68a9b 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/Field.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/Field.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -31,6 +55,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.exist.Namespaces; +import org.exist.dom.QName; import org.exist.dom.memtree.InMemoryNodeSet; import org.exist.dom.memtree.MemTreeBuilder; import org.exist.dom.persistent.Match; @@ -41,19 +66,20 @@ import org.exist.xquery.value.*; import javax.annotation.Nullable; -import javax.xml.datatype.XMLGregorianCalendar; import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.nio.ByteBuffer; -import java.util.Date; -import java.util.GregorianCalendar; import java.util.Map; import static org.exist.xquery.FunctionDSL.*; import static org.exist.xquery.modules.lucene.LuceneModule.functionSignature; import static org.exist.xquery.modules.lucene.LuceneModule.functionSignatures; +/** + * @author Wolfgang Meier + * @author Adam Retter + */ public class Field extends BasicFunction { private static final FunctionParameterSequenceType FS_PARAM_NODE = param("node", Type.NODE, "the context node to check for attached fields"); @@ -82,16 +108,24 @@ public class Field extends BasicFunction { ); private static final String FS_BINARY_FIELD_NAME = "binary-field"; - static final FunctionSignature FS_BINARY_FIELD = functionSignature( + static final FunctionSignature[] FS_BINARY_FIELD = functionSignatures( FS_BINARY_FIELD_NAME, "Returns the value of a binary field attached to a particular node obtained via a full text search." + "Accepts an additional parameter to name the target type into which the field " + "value should be cast. This is mainly relevant for fields having a different type than xs:string. " + "As lucene does not record type information, numbers or dates would be returned as strings by default.", returnsOptMany(Type.ITEM, "Sequence corresponding to the values of the field attached, cast to the desired target type"), - FS_PARAM_NODE, - FS_PARAM_FIELD, - TYPE_PARAMETER + arities( + arity( + FS_PARAM_NODE, + FS_PARAM_FIELD + ), + arity( + FS_PARAM_NODE, + FS_PARAM_FIELD, + TYPE_PARAMETER + ) + ) ); private static final String FS_HIGHLIGHT_FIELD_MATCHES_NAME = "highlight-field-matches"; @@ -133,30 +167,47 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro final LuceneIndexWorker index = (LuceneIndexWorker) context.getBroker().getIndexController().getWorkerByIndexId(LuceneIndex.ID); try { - return switch (called) { - case FS_FIELD_NAME -> getFieldValues(fieldName, type, match, index); - case FS_HIGHLIGHT_FIELD_MATCHES_NAME -> { - final Sequence result = getFieldValues(fieldName, type, match, index); - yield highlightMatches(fieldName, proxy, match, result); - } - case FS_BINARY_FIELD_NAME -> getBinaryFieldValue(fieldName, type, match, index); - default -> throw new XPathException(this, ErrorCodes.FOER0000, "Unknown function: " + getName()); - }; + Sequence result; + switch (called) { + case FS_FIELD_NAME: + result = getFieldValues(fieldName, type, match, index); + break; + + case FS_HIGHLIGHT_FIELD_MATCHES_NAME: + result = getFieldValues(fieldName, type, match, index); + result = highlightMatches(fieldName, proxy, match, result); + break; + + case FS_BINARY_FIELD_NAME: + result = getBinaryFieldValue(fieldName, type, match, index); + break; + + default: + throw new XPathException(this, ErrorCodes.FOER0000, "Unknown function: " + getName()); + } + + return result; + } catch (final IOException e) { - throw new XPathException(this, LuceneModule.EXXQDYFT0002, "Error retrieving field: " + e.getMessage()); + throw new XPathException(this, LuceneModule.EXXQDYFT0002, "Error retrieving field: " + e.getMessage(), e); } } - private Sequence getBinaryFieldValue(final String fieldName, final int type, final LuceneMatch match, final LuceneIndexWorker index) throws IOException { + private Sequence getBinaryFieldValue(final String fieldName, final int type, final LuceneMatch match, final LuceneIndexWorker index) throws IOException, XPathException { final BytesRef fieldValue = index.getBinaryField(match.getLuceneDocId(), fieldName); if (fieldValue == null) { return Sequence.EMPTY_SEQUENCE; } + return bytesToAtomic(fieldValue, type); } private Sequence getFieldValues(final String fieldName, final int type, final LuceneMatch match, final LuceneIndexWorker index) throws IOException, XPathException { - final IndexableField[] fields = index.getField(match.getLuceneDocId(), fieldName); + @Nullable final IndexableField[] fields = index.getField(match.getLuceneDocId(), fieldName); + if (fields == null) { + return Sequence.EMPTY_SEQUENCE; + } + final Sequence result = new ValueSequence(fields.length); for (final IndexableField field : fields) { if (field.numericValue() != null) { @@ -285,70 +336,267 @@ private Sequence highlightMatches(final String fieldName, final NodeProxy proxy, return null; } - static AtomicValue bytesToAtomic(final BytesRef field, final int type) { - return switch (type) { - case Type.TIME -> TimeValue.deserialize(ByteBuffer.wrap(field.bytes)); - case Type.DATE_TIME -> DateTimeValue.deserialize(ByteBuffer.wrap(field.bytes)); - case Type.DATE -> DateValue.deserialize(ByteBuffer.wrap(field.bytes)); - case Type.INTEGER, Type.LONG, Type.UNSIGNED_LONG, Type.INT, Type.UNSIGNED_INT, Type.SHORT, Type.UNSIGNED_SHORT -> - IntegerValue.deserialize(ByteBuffer.wrap(field.bytes)); - case Type.DOUBLE -> DoubleValue.deserialize(ByteBuffer.wrap(field.bytes)); - case Type.FLOAT -> FloatValue.deserialize(ByteBuffer.wrap(field.bytes)); - case Type.DECIMAL -> DecimalValue.deserialize(ByteBuffer.wrap(field.bytes)); - default -> new StringValue(field.utf8ToString()); - }; - } + AtomicValue bytesToAtomic(final BytesRef field, final int type) throws XPathException, IOException { + try { + switch (type) { + + case Type.INTEGER: + case Type.NON_POSITIVE_INTEGER: + case Type.NEGATIVE_INTEGER: + case Type.LONG: + case Type.INT: + case Type.SHORT: + case Type.BYTE: + case Type.NON_NEGATIVE_INTEGER: + case Type.UNSIGNED_LONG: + case Type.UNSIGNED_INT: + case Type.UNSIGNED_SHORT: + case Type.UNSIGNED_BYTE: + case Type.POSITIVE_INTEGER: + return IntegerValue.deserialize(this, ByteBuffer.wrap(field.bytes), type); + + case Type.DECIMAL: + return DecimalValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.DOUBLE: + return DoubleValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.FLOAT: + return FloatValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.DATE: + return DateValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.TIME: + return TimeValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.DATE_TIME: + return DateTimeValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.DATE_TIME_STAMP: + return DateTimeStampValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.DURATION: + return DurationValue.deserialize(this, ByteBuffer.wrap(field.bytes)); - static AtomicValue stringToAtomic(final int type, final String value) throws XPathException { - return switch (type) { - case Type.TIME -> new TimeValue(value); - case Type.DATE_TIME -> new DateTimeValue(value); - case Type.DATE -> new DateValue(value); - case Type.INTEGER, Type.LONG, Type.UNSIGNED_LONG, Type.INT, Type.UNSIGNED_INT, Type.SHORT, Type.UNSIGNED_SHORT -> - new IntegerValue(value); - case Type.DOUBLE -> new DoubleValue(value); - case Type.FLOAT -> new FloatValue(value); - case Type.DECIMAL -> new DecimalValue(value); - default -> new StringValue(value); - }; + case Type.YEAR_MONTH_DURATION: + return YearMonthDurationValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.DAY_TIME_DURATION: + return DayTimeDurationValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.G_YEAR_MONTH: + return GYearMonthValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.G_YEAR: + return GYearValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.G_MONTH_DAY: + return GMonthDayValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.G_MONTH: + return GMonthValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.G_DAY: + return GDayValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.BOOLEAN: + return BooleanValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.BASE64_BINARY: + return BinaryValue.deserialize(this, new Base64BinaryValueType(), ByteBuffer.wrap(field.bytes)); + + case Type.HEX_BINARY: + return BinaryValue.deserialize(this, new HexBinaryValueType(), ByteBuffer.wrap(field.bytes)); + + case Type.ANY_URI: + return AnyURIValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.QNAME: + return QNameValue.deserialize(this, ByteBuffer.wrap(field.bytes)); + + case Type.STRING: + case Type.NORMALIZED_STRING: + case Type.TOKEN: + case Type.LANGUAGE: + case Type.NMTOKEN: + case Type.NAME: + case Type.NCNAME: + case Type.ID: + case Type.IDREF: + case Type.ENTITY: + return StringValue.deserialize(this, ByteBuffer.wrap(field.bytes), type); + + case Type.NOTATION: + default: + throw new XPathException(this, LuceneModule.EXXQDYFT0005, "Cannot convert binary field to " + Type.getTypeName(type)); + } + } catch (final NumberFormatException e) { + throw new XPathException(this, e.getMessage(), e); + } } - static AtomicValue numberToAtomic(final int type, final Number value) throws XPathException { - switch(type) { + private AtomicValue stringToAtomic(final int type, final String value) throws XPathException { + switch (type) { + case Type.INTEGER: + case Type.NON_POSITIVE_INTEGER: + case Type.NEGATIVE_INTEGER: + case Type.LONG: + case Type.INT: + case Type.SHORT: + case Type.BYTE: + case Type.NON_NEGATIVE_INTEGER: + case Type.UNSIGNED_LONG: + case Type.UNSIGNED_INT: + case Type.UNSIGNED_SHORT: + case Type.UNSIGNED_BYTE: + case Type.POSITIVE_INTEGER: + return new IntegerValue(this, value, type); + + case Type.DECIMAL: + return new DecimalValue(this, value); + + case Type.DOUBLE: + return new DoubleValue(this, value); + + case Type.FLOAT: + return new FloatValue(this, value); + + case Type.DATE: + return new DateValue(this, value); + case Type.TIME: - final Date time = new Date(value.longValue()); - final GregorianCalendar gregorianCalendar = new GregorianCalendar(); - gregorianCalendar.setTime(time); - final XMLGregorianCalendar calendar = TimeUtils.getInstance().newXMLGregorianCalendar(gregorianCalendar); - return new TimeValue(calendar); + return new TimeValue(this, value); + case Type.DATE_TIME: - throw new XPathException(LuceneModule.EXXQDYFT0004, "Cannot convert numeric field to xs:dateTime"); - case Type.DATE: - final long dl = value.longValue(); - final int year = (int)(dl >> 16) & 0xFFFF; - final int month = (int)(dl >> 8) & 0xFF; - final int day = (int)(dl & 0xFF); - final DateValue date = new DateValue(); - date.calendar.setYear(year); - date.calendar.setMonth(month); - date.calendar.setDay(day); - return date; + return new DateTimeValue(this, value); + + case Type.DATE_TIME_STAMP: + return new DateTimeStampValue(this, value); + + case Type.DURATION: + return new DurationValue(this, value); + + case Type.YEAR_MONTH_DURATION: + return new YearMonthDurationValue(this, value); + + case Type.DAY_TIME_DURATION: + return new DayTimeDurationValue(this, value); + + case Type.G_YEAR_MONTH: + return new GYearMonthValue(this, value); + + case Type.G_YEAR: + return new GYearValue(this, value); + + case Type.G_MONTH_DAY: + return new GMonthDayValue(this, value); + + case Type.G_MONTH: + return new GMonthValue(this, value); + + case Type.G_DAY: + return new GDayValue(this, value); + + case Type.BOOLEAN: + return BooleanValue.valueOf(this, value); + + case Type.BASE64_BINARY: + return new BinaryValueFromBinaryString(this, new Base64BinaryValueType(), value); + + case Type.HEX_BINARY: + return new BinaryValueFromBinaryString(this, new HexBinaryValueType(), value); + + case Type.ANY_URI: + return new AnyURIValue(value); + + case Type.QNAME: + try { + return new QNameValue(null, QName.parse(value)); + } catch (final QName.IllegalQNameException e) { + throw new XPathException(this, LuceneModule.EXXQDYFT0006, "Cannot convert string field to " + Type.getTypeName(type), e); + } + + case Type.STRING: + case Type.NORMALIZED_STRING: + case Type.TOKEN: + case Type.LANGUAGE: + case Type.NMTOKEN: + case Type.NAME: + case Type.NCNAME: + case Type.ID: + case Type.IDREF: + case Type.ENTITY: + return new StringValue(this, value, type); + + case Type.NOTATION: + default: + throw new XPathException(this, LuceneModule.EXXQDYFT0006, "Cannot convert string field to " + Type.getTypeName(type)); + } + } + + private AtomicValue numberToAtomic(final int type, final Number value) throws XPathException { + switch (type) { case Type.INTEGER: + case Type.NON_POSITIVE_INTEGER: + case Type.NEGATIVE_INTEGER: case Type.LONG: - case Type.UNSIGNED_LONG: case Type.INT: - case Type.UNSIGNED_INT: case Type.SHORT: + case Type.BYTE: + case Type.NON_NEGATIVE_INTEGER: + case Type.UNSIGNED_LONG: + case Type.UNSIGNED_INT: case Type.UNSIGNED_SHORT: - return new IntegerValue(value.longValue()); + case Type.UNSIGNED_BYTE: + case Type.POSITIVE_INTEGER: + return new IntegerValue(this, value.longValue(), type); + + case Type.DECIMAL: + return new DecimalValue(this, value.doubleValue()); + case Type.DOUBLE: - return new DoubleValue(value.doubleValue()); + return new DoubleValue(this, value.doubleValue()); + case Type.FLOAT: - return new FloatValue(value.floatValue()); - case Type.DECIMAL: - return new DecimalValue(value.doubleValue()); + return new FloatValue(this, value.floatValue()); + + case Type.DATE: + return DateValue.deserialize(this, value.longValue()); + + case Type.TIME: + return TimeValue.deserialize(this, value.longValue()); + + case Type.BOOLEAN: + return value.intValue() == 1 ? BooleanValue.TRUE : BooleanValue.FALSE; + + case Type.DATE_TIME: + case Type.DATE_TIME_STAMP: + case Type.DURATION: + case Type.YEAR_MONTH_DURATION: + case Type.DAY_TIME_DURATION: + case Type.G_YEAR_MONTH: + case Type.G_YEAR: + case Type.G_MONTH_DAY: + case Type.G_MONTH: + case Type.G_DAY: + case Type.BASE64_BINARY: + case Type.HEX_BINARY: + case Type.ANY_URI: + case Type.QNAME: + case Type.NOTATION: + case Type.STRING: + case Type.NORMALIZED_STRING: + case Type.TOKEN: + case Type.LANGUAGE: + case Type.NMTOKEN: + case Type.NAME: + case Type.NCNAME: + case Type.ID: + case Type.IDREF: + case Type.ENTITY: default: - return new StringValue(value.toString()); + throw new XPathException(this, LuceneModule.EXXQDYFT0007, "Cannot convert numeric field to " + Type.getTypeName(type)); } } } diff --git a/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/LuceneModule.java b/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/LuceneModule.java index 284cd17b88..1aefa65b1c 100644 --- a/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/LuceneModule.java +++ b/extensions/indexes/lucene/src/main/java/org/exist/xquery/modules/lucene/LuceneModule.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -38,7 +62,7 @@ * * @author wolf * @author ljo - * + * @author Adam Retter */ public class LuceneModule extends AbstractInternalModule { @@ -52,6 +76,9 @@ public class LuceneModule extends AbstractInternalModule { public final static ErrorCode EXXQDYFT0002 = new LuceneErrorCode("EXXQDYFT0002", "IO Exception in lucene index."); public final static ErrorCode EXXQDYFT0003 = new LuceneErrorCode("EXXQDYFT0003", "Document not found."); public final static ErrorCode EXXQDYFT0004 = new LuceneErrorCode("EXXQDYFT0004", "Wrong configuration passed to ft:query"); + public final static ErrorCode EXXQDYFT0005 = new LuceneErrorCode("EXXQDYFT0005", "Unable to deserialize binary value in call to ft:field"); + public final static ErrorCode EXXQDYFT0006 = new LuceneErrorCode("EXXQDYFT0006", "Unable to deserialize string value in call to ft:field"); + public final static ErrorCode EXXQDYFT0007 = new LuceneErrorCode("EXXQDYFT0007", "Unable to deserialize numeric value in call to ft:field"); public static final FunctionDef[] functions = { new FunctionDef(Query.signatures[0], Query.class), @@ -74,7 +101,8 @@ public class LuceneModule extends AbstractInternalModule { new FunctionDef(Facets.signatures[2], Facets.class), new FunctionDef(Field.FS_FIELD[0], Field.class), new FunctionDef(Field.FS_FIELD[1], Field.class), - new FunctionDef(Field.FS_BINARY_FIELD, Field.class), + new FunctionDef(Field.FS_BINARY_FIELD[0], Field.class), + new FunctionDef(Field.FS_BINARY_FIELD[1], Field.class), new FunctionDef(Field.FS_HIGHLIGHT_FIELD_MATCHES, Field.class), new FunctionDef(LuceneIndexKeys.signatures[0], LuceneIndexKeys.class) }; diff --git a/extensions/indexes/lucene/src/test/xquery/lucene/fields.xqm b/extensions/indexes/lucene/src/test/xquery/lucene/fields.xqm new file mode 100644 index 0000000000..b16a85a12c --- /dev/null +++ b/extensions/indexes/lucene/src/test/xquery/lucene/fields.xqm @@ -0,0 +1,1630 @@ +(: + : Elemental + : Copyright (C) 2024, Evolved Binary Ltd + : + : admin@evolvedbinary.com + : https://www.evolvedbinary.com | https://www.elemental.xyz + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; version 2.1. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace fields = "http://exist-db.org/xquery/lucene/test/fields"; + +declare namespace test = "http://exist-db.org/xquery/xqsuite"; +declare namespace util = "http://exist-db.org/xquery/util"; + +import module namespace ft = "http://exist-db.org/xquery/lucene"; + +declare variable $fields:COLLECTION_CONF := document { + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +}; + +declare variable $fields:TEST_COLLECTION_NAME := "ft-field-test"; +declare variable $fields:TEST_COLLECTION_PATH := "/db/" || $fields:TEST_COLLECTION_NAME; + +declare variable $fields:TYPES_DOC_NAME := "types.xml"; +declare variable $fields:TYPES_DOC_PATH := $fields:TEST_COLLECTION_PATH || "/" || $fields:TYPES_DOC_NAME; + +declare variable $fields:TYPES_DOC := document { + + 20250706 + 2025-07-06-04:55 + + + 20250706085058265 + 2025-07-06T08:50:58.265-04:55 + 2025-07-06T08:51:11.932 + 2025-07-06T08:51:11.932-04:55 + P15WT7H + P2025Y07M15DT7H00M00.000S + PT7H00M00S + P2025Y07M + P2025Y + P15DT7H000.000S + 1.23E2 + -19223372036854775808.23 + 199769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.000000 + 1.23 + -9223372036854775809 + 9223372036854775808 + 9876543210 + 01 + 0 + -0 + -0223372036854775808 + 01 + 0 + -0 + -0223372036854775808 + 9.876 + -9223372036854775809 + 9223372036854775808 + 9876543210 + 9.876 + -92147483648 + 92147483647 + -98765 + 98765 + 9.876 + -932768 + 932767 + -9767 + 9767 + 9.876 + -987 + 987 + -99 + 99 + -01 + -0 + 0 + 0223372036854775808 + 9.876 + -9 + 918446744073709551616 + 9876543210 + 9.876 + -9 + 94294967296 + 98765 + 9.876 + -9 + 965536 + 9536 + 9.876 + -9 + 9255 + 9 + -01 + -0 + 0 + 0223372036854775808 + 9.2EE1 + -9.4028235E38 + 9.4028235E38 + 9.2E1 + 9.2EE1 + -9.7976931348623157E308 + 9.7976931348623157E308 + 9.2E1 + 1999-00 + 1978-09-05:00 + --1978 + 1978 + 09-12 + --09-12 + 09 + --09 + 12 + ---12 + tru + true + dmFsaWQ + dmFsaWQ= + 76616C696 + 76616C6964 + http://evolvedbinary.com#f1#f2 + http://evolvedbinary.com + xs:in:valid + xs:valid + xs:in:valid + xs:valid + AB C + ABC + ABC A + ABC + AB C + ABC + englishing + en-GB + %ABC + ABC + -ABC + ABC + A:BC + ABC + A:BC + ABC + A:BC + ABC + A:BC + ABC + +}; + +declare + %test:setUp +function fields:setup() { + let $conf-collection := xmldb:create-collection("/db/system/config/db", $fields:TEST_COLLECTION_NAME) + return + let $_ := xmldb:store($conf-collection, "collection.xconf", $fields:COLLECTION_CONF) + return + let $test-collection := xmldb:create-collection("/db", $fields:TEST_COLLECTION_NAME) + return + xmldb:store($test-collection, $fields:TYPES_DOC_NAME, $fields:TYPES_DOC) +}; + +declare + %test:tearDown +function fields:cleanup() { + xmldb:remove($fields:TEST_COLLECTION_PATH), + xmldb:remove("/db/system/config" || $fields:TEST_COLLECTION_PATH) +}; + +declare + %private +function fields:assert-field-types($search as xs:string, $type-name as xs:string, $type-check as function(item()*) as xs:boolean) as element()+ { + for $element in doc($fields:TYPES_DOC_PATH)/types/element()[local-name() eq $type-name][ft:query(., $search)] + return + ( + {ft:field($element, $type-name) instance of xs:string}, + {ft:field($element, $type-name, "xs:string") instance of xs:string}, + {$type-check(ft:field($element, $type-name || "-typed-" || $type-name, "xs:" || $type-name))}, + {ft:binary-field($element, $type-name || "-binary") instance of xs:string}, + {ft:binary-field($element, $type-name || "-binary", "xs:string") instance of xs:string}, + {$type-check(ft:binary-field($element, $type-name || "-typed-" || $type-name || "-binary", "xs:" || $type-name))} + ) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:date-type-fields() { + fields:assert-field-types("2025*", "date", function($field-value) { $field-value instance of xs:date }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:time-type-fields() { + fields:assert-field-types("08*", "time", function($field-value) { $field-value instance of xs:time }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:dateTime-type-fields() { + fields:assert-field-types("2025*", "dateTime", function($field-value) { $field-value instance of xs:dateTime }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:dateTimeStamp-type-fields() { + fields:assert-field-types("2025*", "dateTimeStamp", function($field-value) { $field-value instance of xs:dateTimeStamp }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:duration-type-fields() { + fields:assert-field-types("P*", "duration", function($field-value) { $field-value instance of xs:duration }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:yearMonthDuration-type-fields() { + fields:assert-field-types("P*", "yearMonthDuration", function($field-value) { $field-value instance of xs:yearMonthDuration }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:dayTimeDuration-type-fields() { + fields:assert-field-types("P*", "dayTimeDuration", function($field-value) { $field-value instance of xs:dayTimeDuration }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:decimal-type-fields() { + fields:assert-field-types("1*", "decimal", function($field-value) { $field-value instance of xs:decimal }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:integer-type-fields() { + fields:assert-field-types("9*", "integer", function($field-value) { $field-value instance of xs:integer }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:nonPositiveInteger-type-fields() { + fields:assert-field-types("0*", "nonPositiveInteger", function($field-value) { $field-value instance of xs:nonPositiveInteger }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:negativeInteger-type-fields() { + fields:assert-field-types("0*", "negativeInteger", function($field-value) { $field-value instance of xs:negativeInteger }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:long-type-fields() { + fields:assert-field-types("9*", "long", function($field-value) { $field-value instance of xs:long }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:int-type-fields() { + fields:assert-field-types("9*", "int", function($field-value) { $field-value instance of xs:int }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:short-type-fields() { + fields:assert-field-types("9*", "short", function($field-value) { $field-value instance of xs:short }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:byte-type-fields() { + fields:assert-field-types("9*", "byte", function($field-value) { $field-value instance of xs:byte }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:nonNegativeInteger-type-fields() { + fields:assert-field-types("0*", "nonNegativeInteger", function($field-value) { $field-value instance of xs:nonNegativeInteger }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:unsignedLong-type-fields() { + fields:assert-field-types("9*", "unsignedLong", function($field-value) { $field-value instance of xs:unsignedLong }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:unsignedInt-type-fields() { + fields:assert-field-types("9*", "unsignedInt", function($field-value) { $field-value instance of xs:unsignedInt }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:unsignedShort-type-fields() { + fields:assert-field-types("9*", "unsignedShort", function($field-value) { $field-value instance of xs:unsignedShort }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:unsignedByte-type-fields() { + fields:assert-field-types("9*", "unsignedByte", function($field-value) { $field-value instance of xs:unsignedByte }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:positiveInteger-type-fields() { + fields:assert-field-types("0*", "positiveInteger", function($field-value) { $field-value instance of xs:positiveInteger }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:float-type-fields() { + fields:assert-field-types("9*", "float", function($field-value) { $field-value instance of xs:float }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:double-type-fields() { + fields:assert-field-types("9*", "double", function($field-value) { $field-value instance of xs:double }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:gYearMonth-type-fields() { + fields:assert-field-types("1*", "gYearMonth", function($field-value) { $field-value instance of xs:gYearMonth }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:gYear-type-fields() { + fields:assert-field-types("1*", "gYear", function($field-value) { $field-value instance of xs:gYear }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:gMonthDay-type-fields() { + fields:assert-field-types("1*", "gMonthDay", function($field-value) { $field-value instance of xs:gMonthDay }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:gMonth-type-fields() { + fields:assert-field-types("09*", "gMonth", function($field-value) { $field-value instance of xs:gMonth }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:gDay-type-fields() { + fields:assert-field-types("1*", "gDay", function($field-value) { $field-value instance of xs:gDay }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:boolean-type-fields() { + fields:assert-field-types("t*", "boolean", function($field-value) { $field-value instance of xs:boolean }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:base64Binary-type-fields() { + fields:assert-field-types("d*", "base64Binary", function($field-value) { $field-value instance of xs:base64Binary }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:hexBinary-type-fields() { + fields:assert-field-types("7*", "hexBinary", function($field-value) { $field-value instance of xs:hexBinary }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:anyURI-type-fields() { + fields:assert-field-types("h*", "anyURI", function($field-value) { $field-value instance of xs:anyURI }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:QName-type-fields() { + fields:assert-field-types("x*", "QName", function($field-value) { $field-value instance of xs:QName }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'false', + 'true', + 'true', + 'false' + ) +function fields:NOTATION-type-fields() { + fields:assert-field-types("x*", "NOTATION", function($field-value) { $field-value instance of xs:NOTATION }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:string-type-fields() { + fields:assert-field-types("A*", "string", function($field-value) { $field-value instance of xs:string }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:normalizedString-type-fields() { + fields:assert-field-types("A*", "normalizedString", function($field-value) { $field-value instance of xs:normalizedString }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'true', + 'true', + 'true', + 'true', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:token-type-fields() { + fields:assert-field-types("A*", "token", function($field-value) { $field-value instance of xs:token }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:language-type-fields() { + fields:assert-field-types("en*", "language", function($field-value) { $field-value instance of xs:language }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:NMTOKEN-type-fields() { + fields:assert-field-types("A*", "NMTOKEN", function($field-value) { $field-value instance of xs:NMTOKEN }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:Name-type-fields() { + fields:assert-field-types("A*", "Name", function($field-value) { $field-value instance of xs:Name }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:NCName-type-fields() { + fields:assert-field-types("A*", "NCName", function($field-value) { $field-value instance of xs:NCName }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:ID-type-fields() { + fields:assert-field-types("A*", "ID", function($field-value) { $field-value instance of xs:ID }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:IDREF-type-fields() { + fields:assert-field-types("A*", "IDREF", function($field-value) { $field-value instance of xs:IDREF }) +}; + +declare + %test:assertEquals( + 'true', + 'true', + 'false', + 'true', + 'true', + 'false', + + 'true', + 'true', + 'true', + 'true', + 'true', + 'true' + ) +function fields:ENTITY-type-fields() { + fields:assert-field-types("A*", "ENTITY", function($field-value) { $field-value instance of xs:ENTITY }) +}; + +declare + %test:assertEquals("2025-07-06-04:55", "2025-07-06-04:55") +function fields:date-typed-field-preserves-timezone() { + let $date-elem := doc($fields:TYPES_DOC_PATH)/types/element()[local-name() eq "date"][empty(@description)][ft:query(., "2025*")] + return + ( + ft:field($date-elem, "date-typed-date", "xs:date"), + ft:binary-field($date-elem, "date-typed-date-binary", "xs:date") + ) +}; + +declare + %test:assertEquals("08:50:49.831-04:55", "08:50:49.831-04:55") +function fields:time-typed-field-preserves-timezone() { + let $time-elem := doc($fields:TYPES_DOC_PATH)/types/element()[local-name() eq "time"][empty(@description)][ft:query(., "08*")] + return + ( + ft:field($time-elem, "time-typed-time", "xs:time"), + ft:binary-field($time-elem, "time-typed-time-binary", "xs:time") + ) +}; + +declare + %test:assertEquals("2025-07-06T08:50:58.265-04:55", "2025-07-06T08:50:58.265-04:55") +function fields:dateTime-typed-field-preserves-timezone() { + let $dateTime-elem := doc($fields:TYPES_DOC_PATH)/types/element()[local-name() eq "dateTime"][empty(@description)][ft:query(., "2025*")] + return + ( + ft:field($dateTime-elem, "dateTime-typed-dateTime", "xs:dateTime"), + ft:binary-field($dateTime-elem, "dateTime-typed-dateTime-binary", "xs:dateTime") + ) +}; + +declare + %test:assertEquals("2025-07-06T08:51:11.932-04:55", "2025-07-06T08:51:11.932-04:55") +function fields:dateTimeStamp-typed-field-preserves-timezone() { + let $dateTimeStamp-elem := doc($fields:TYPES_DOC_PATH)/types/element()[local-name() eq "dateTimeStamp"][empty(@description)][ft:query(., "2025*")] + return + ( + ft:field($dateTimeStamp-elem, "dateTimeStamp-typed-dateTimeStamp", "xs:dateTimeStamp"), + ft:binary-field($dateTimeStamp-elem, "dateTimeStamp-typed-dateTimeStamp-binary", "xs:dateTimeStamp") + ) +}; diff --git a/extensions/indexes/ngram/src/main/java/org/exist/indexing/ngram/NGramIndexWorker.java b/extensions/indexes/ngram/src/main/java/org/exist/indexing/ngram/NGramIndexWorker.java index 5f0ebb65e8..40b9dee9b8 100644 --- a/extensions/indexes/ngram/src/main/java/org/exist/indexing/ngram/NGramIndexWorker.java +++ b/extensions/indexes/ngram/src/main/java/org/exist/indexing/ngram/NGramIndexWorker.java @@ -90,8 +90,8 @@ import org.exist.storage.btree.IndexQuery; import org.exist.storage.btree.Value; import org.exist.storage.index.BFile; +import org.exist.storage.io.VariableByteArrayOutputStream; import org.exist.storage.io.VariableByteInput; -import org.exist.storage.io.VariableByteOutputStream; import org.exist.storage.lock.LockManager; import org.exist.storage.lock.ManagedLock; import org.exist.storage.txn.Txn; @@ -135,7 +135,7 @@ public class NGramIndexWorker implements OrderedValuesIndex, QNamedKeysIndex { @SuppressWarnings("unused") private IndexController controller; private final Map ngrams = new TreeMap<>(); - private final VariableByteOutputStream os = new VariableByteOutputStream(128); + private final VariableByteArrayOutputStream os = new VariableByteArrayOutputStream(128); private NGramMatchListener matchListener = null; @@ -206,64 +206,73 @@ private void saveIndex() { return; } - final VariableByteOutputStream buf = new VariableByteOutputStream(); - for (final Map.Entry entry : ngrams.entrySet()) { - final QNameTerm key = entry.getKey(); - final OccurrenceList occurences = entry.getValue(); - occurences.sort(); - os.clear(); - os.writeInt(currentDoc.getDocId()); - os.writeByte(key.qname.getNameType()); - os.writeInt(occurences.getTermCount()); - - // write nodeids, freq, and offsets to a `temp` buf - try { - NodeId previous = null; - for (int m = 0; m < occurences.getSize(); ) { - previous = occurences.getNode(m).write(previous, buf); - - final int freq = occurences.getOccurrences(m); - buf.writeInt(freq); - for (int n = 0; n < freq; n++) { - buf.writeInt(occurences.getOffset(m + n)); - } - m += freq; + try (final VariableByteArrayOutputStream buf = new VariableByteArrayOutputStream()) { + for (final Map.Entry entry : ngrams.entrySet()) { + final QNameTerm key = entry.getKey(); + final OccurrenceList occurences = entry.getValue(); + occurences.sort(); + os.clear(); + try { + os.writeInt(currentDoc.getDocId()); + os.writeByte(key.qname.getNameType()); + os.writeInt(occurences.getTermCount()); + } catch (final IOException e) { + LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); + os.clear(); + continue; } - final byte[] bufData = buf.toByteArray(); + // write nodeids, freq, and offsets to a `temp` buf + try { + NodeId previous = null; + for (int m = 0; m < occurences.getSize(); ) { + previous = occurences.getNode(m).write(previous, buf); + + final int freq = occurences.getOccurrences(m); + buf.writeInt(freq); + for (int n = 0; n < freq; n++) { + buf.writeInt(occurences.getOffset(m + n)); + } + m += freq; + } - // clear the buf for the next iteration - buf.clear(); + final byte[] bufData = buf.toByteArray(); - // Write length of node IDs + frequency + offsets (bytes) - os.writeFixedInt(bufData.length); + // clear the buf for the next iteration + buf.clear(); - // Write the node IDs + frequency + offset - os.write(bufData); - } catch (final IOException e) { - LOG.error("IOException while writing nGram index: {}", e.getMessage(), e); - } + // Write length of node IDs + frequency + offsets (bytes) + os.writeFixedInt(bufData.length); - final ByteArray data = os.data(); - if (data.size() == 0) { - continue; - } + // Write the node IDs + frequency + offset + os.write(bufData); + } catch (final IOException e) { + LOG.error("IOException while writing nGram index: {}", e.getMessage(), e); + } - try (final ManagedLock dbLock = lockManager.acquireBtreeWriteLock(index.db.getLockName())) { - final NGramQNameKey value = new NGramQNameKey(currentDoc.getCollection().getId(), key.qname, + final ByteArray data = os.data(); + if (data.size() == 0) { + continue; + } + + try (final ManagedLock dbLock = lockManager.acquireBtreeWriteLock(index.db.getLockName())) { + final NGramQNameKey value = new NGramQNameKey(currentDoc.getCollection().getId(), key.qname, index.getBrokerPool().getSymbols(), key.term); - index.db.append(value, data); - } catch (final LockException e) { - LOG.warn("Failed to acquire lock for file {}", FileUtils.fileName(index.db.getFile()), e); - } catch (final IOException e) { - LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); - } catch (final ReadOnlyException e) { - LOG.warn("Read-only error for file {}", FileUtils.fileName(index.db.getFile()), e); - } finally { - os.clear(); + index.db.append(value, data); + } catch (final LockException e) { + LOG.warn("Failed to acquire lock for file {}", FileUtils.fileName(index.db.getFile()), e); + } catch (final IOException e) { + LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); + } catch (final ReadOnlyException e) { + LOG.warn("Read-only error for file {}", FileUtils.fileName(index.db.getFile()), e); + } finally { + os.clear(); + } } + ngrams.clear(); + } catch (final IOException e) { + LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); } - ngrams.clear(); } private void dropIndex(final ReindexMode mode) { @@ -271,117 +280,120 @@ private void dropIndex(final ReindexMode mode) { return; } - final VariableByteOutputStream buf = new VariableByteOutputStream(); + try (final VariableByteArrayOutputStream buf = new VariableByteArrayOutputStream()) { - for (final Map.Entry entry : ngrams.entrySet()) { - final QNameTerm key = entry.getKey(); - final OccurrenceList occurencesList = entry.getValue(); - occurencesList.sort(); - os.clear(); + for (final Map.Entry entry : ngrams.entrySet()) { + final QNameTerm key = entry.getKey(); + final OccurrenceList occurencesList = entry.getValue(); + occurencesList.sort(); + os.clear(); - try (final ManagedLock dbLock = lockManager.acquireBtreeWriteLock(index.db.getLockName())) { - final NGramQNameKey value = new NGramQNameKey(currentDoc.getCollection().getId(), key.qname, + try (final ManagedLock dbLock = lockManager.acquireBtreeWriteLock(index.db.getLockName())) { + final NGramQNameKey value = new NGramQNameKey(currentDoc.getCollection().getId(), key.qname, index.getBrokerPool().getSymbols(), key.term); - boolean changed = false; - os.clear(); - final VariableByteInput is = index.db.getAsStream(value); - if (is == null) { - continue; - } - while (is.available() > 0) { - final int storedDocId = is.readInt(); - final byte nameType = is.readByte(); - final int occurrences = is.readInt(); - //Read (variable) length of node IDs + frequency + offsets - final int length = is.readFixedInt(); - if (storedDocId != currentDoc.getDocId()) { - // data are related to another document: - // copy them to any existing data - os.writeInt(storedDocId); - os.writeByte(nameType); - os.writeInt(occurrences); - os.writeFixedInt(length); - is.copyRaw(os, length); - } else { - // data are related to our document: - if (mode == ReindexMode.REMOVE_ALL_NODES) { - // skip them - is.skipBytes(length); + boolean changed = false; + os.clear(); + final VariableByteInput is = index.db.getAsStream(value); + if (is == null) { + continue; + } + while (is.available() > 0) { + final int storedDocId = is.readInt(); + final byte nameType = is.readByte(); + final int occurrences = is.readInt(); + //Read (variable) length of node IDs + frequency + offsets + final int length = is.readFixedInt(); + if (storedDocId != currentDoc.getDocId()) { + // data are related to another document: + // copy them to any existing data + os.writeInt(storedDocId); + os.writeByte(nameType); + os.writeInt(occurrences); + os.writeFixedInt(length); + is.copyRaw(os, length); } else { - // removing nodes: need to filter out the node ids to be removed - // feed the new list with the GIDs - - final OccurrenceList newOccurrences = new OccurrenceList(); - NodeId previous = null; - for (int m = 0; m < occurrences; m++) { - final NodeId nodeId = index.getBrokerPool().getNodeFactory().createFromStream(previous, is); - previous = nodeId; - final int freq = is.readInt(); - // add the node to the new list if it is not - // in the list of removed nodes - if (!occurencesList.contains(nodeId)) { - for (int n = 0; n < freq; n++) { - newOccurrences.add(nodeId, is.readInt()); + // data are related to our document: + if (mode == ReindexMode.REMOVE_ALL_NODES) { + // skip them + is.skipBytes(length); + } else { + // removing nodes: need to filter out the node ids to be removed + // feed the new list with the GIDs + + final OccurrenceList newOccurrences = new OccurrenceList(); + NodeId previous = null; + for (int m = 0; m < occurrences; m++) { + final NodeId nodeId = index.getBrokerPool().getNodeFactory().createFromStream(previous, is); + previous = nodeId; + final int freq = is.readInt(); + // add the node to the new list if it is not + // in the list of removed nodes + if (!occurencesList.contains(nodeId)) { + for (int n = 0; n < freq; n++) { + newOccurrences.add(nodeId, is.readInt()); + } + } else { + is.skip(freq); } - } else { - is.skip(freq); } - } - // append the data from the new list - if (newOccurrences.getSize() > 0) { - //Don't forget this one - newOccurrences.sort(); - os.writeInt(currentDoc.getDocId()); - os.writeByte(nameType); - os.writeInt(newOccurrences.getTermCount()); - - // write nodeids, freq, and offsets to a `temp` buf - previous = null; - for (int m = 0; m < newOccurrences.getSize(); ) { - previous = newOccurrences.getNode(m).write(previous, buf); - final int freq = newOccurrences.getOccurrences(m); - buf.writeInt(freq); - for (int n = 0; n < freq; n++) { - buf.writeInt(newOccurrences.getOffset(m + n)); + // append the data from the new list + if (newOccurrences.getSize() > 0) { + //Don't forget this one + newOccurrences.sort(); + os.writeInt(currentDoc.getDocId()); + os.writeByte(nameType); + os.writeInt(newOccurrences.getTermCount()); + + // write nodeids, freq, and offsets to a `temp` buf + previous = null; + for (int m = 0; m < newOccurrences.getSize(); ) { + previous = newOccurrences.getNode(m).write(previous, buf); + final int freq = newOccurrences.getOccurrences(m); + buf.writeInt(freq); + for (int n = 0; n < freq; n++) { + buf.writeInt(newOccurrences.getOffset(m + n)); + } + m += freq; } - m += freq; - } - final byte[] bufData = buf.toByteArray(); + final byte[] bufData = buf.toByteArray(); - // clear the buf for the next iteration - buf.clear(); + // clear the buf for the next iteration + buf.clear(); - // Write length of node IDs + frequency + offsets (bytes) - os.writeFixedInt(bufData.length); + // Write length of node IDs + frequency + offsets (bytes) + os.writeFixedInt(bufData.length); - // Write the node IDs + frequency + offset - os.write(bufData); + // Write the node IDs + frequency + offset + os.write(bufData); + } } + changed = true; } - changed = true; } - } - //Store new data, if relevant - if (changed) { - //Well, nothing to store : remove the existing data - if (os.data().size() == 0) { - index.db.remove(value); - } else { - if (index.db.put(value, os.data()) == BFile.UNKNOWN_ADDRESS) { - LOG.error("Could not put index data for token '{}' in '{}'", key.term, FileUtils.fileName(index.db.getFile())); + //Store new data, if relevant + if (changed) { + //Well, nothing to store : remove the existing data + if (os.data().size() == 0) { + index.db.remove(value); + } else { + if (index.db.put(value, os.data()) == BFile.UNKNOWN_ADDRESS) { + LOG.error("Could not put index data for token '{}' in '{}'", key.term, FileUtils.fileName(index.db.getFile())); + } } } + } catch (final LockException e) { + LOG.warn("Failed to acquire lock for file {}", FileUtils.fileName(index.db.getFile()), e); + } catch (final IOException e) { + LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); + } finally { + os.clear(); } - } catch (final LockException e) { - LOG.warn("Failed to acquire lock for file {}", FileUtils.fileName(index.db.getFile()), e); - } catch (final IOException e) { - LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); - } finally { - os.clear(); } + ngrams.clear(); + } catch (final IOException e) { + LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e); } - ngrams.clear(); } @Override