[7.x.x] Correct the data problems with Fields in the Lucene Full Text Index

exist-core/src/main/java/org/exist/collections/Collection.java

@@ -54,7 +54,7 @@
 import org.exist.security.SecurityManager;
 import org.exist.storage.*;
 import org.exist.storage.io.VariableByteInput;
-import org.exist.storage.io.VariableByteOutputStream;
+import org.exist.storage.io.VariableByteOutput;
 import org.exist.storage.lock.*;
 import org.exist.storage.lock.Lock.LockMode;
 import org.exist.storage.txn.Txn;
@@ -1207,7 +1207,7 @@ BinaryDocument addBinaryResource(Txn transaction, DBBroker broker, BinaryDocumen
      * @throws IOException in case of I/O errors
 
      */
-    @EnsureContainerLocked(mode=READ_LOCK) void serialize(final VariableByteOutputStream outputStream) throws IOException, LockException;
+    @EnsureContainerLocked(mode=READ_LOCK) void serialize(final VariableByteOutput outputStream) throws IOException, LockException;
 
     @Override void close();
 

exist-core/src/main/java/org/exist/collections/LockedCollection.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -29,7 +53,7 @@
 import org.exist.security.PermissionDeniedException;
 import org.exist.security.Subject;
 import org.exist.storage.*;
-import org.exist.storage.io.VariableByteOutputStream;
+import org.exist.storage.io.VariableByteOutput;
 import org.exist.storage.lock.Lock;
 import org.exist.storage.lock.LockedDocumentMap;
 import org.exist.storage.lock.ManagedCollectionLock;
@@ -473,7 +497,7 @@ public BinaryDocument addBinaryResource(final Txn transaction, final DBBroker br
     }
 
     @Override
-    public void serialize(final VariableByteOutputStream outputStream) throws IOException, LockException {
+    public void serialize(final VariableByteOutput outputStream) throws IOException, LockException {
         collection.serialize(outputStream);
     }
 

exist-core/src/main/java/org/exist/collections/MutableCollection.java

@@ -72,7 +72,7 @@
 import org.exist.security.Subject;
 import org.exist.storage.*;
 import org.exist.storage.io.VariableByteInput;
-import org.exist.storage.io.VariableByteOutputStream;
+import org.exist.storage.io.VariableByteOutput;
 import org.exist.storage.lock.*;
 import org.exist.storage.lock.Lock.LockMode;
 import org.exist.storage.lock.Lock.LockType;
@@ -195,7 +195,7 @@ private MutableCollection(final DBBroker broker, final int collectionId,
     /**
      * Deserializes a Collection object
      *
-     * Counterpart method to {@link #serialize(VariableByteOutputStream)}
+     * Counterpart method to {@link #serialize(VariableByteOutput)}
      *
      * @param broker The database broker
      * @param path The path of the Collection
@@ -884,7 +884,7 @@ public Iterator<DocumentImpl> iteratorNoLock(final DBBroker broker) throws Permi
      * @param outputStream The output stream to write the collection contents to
      */
     @Override
-    public void serialize(final VariableByteOutputStream outputStream) throws IOException, LockException {
+    public void serialize(final VariableByteOutput outputStream) throws IOException, LockException {
         outputStream.writeInt(collectionId);
 
         final int size;
@@ -914,7 +914,7 @@ public void close() {
     /**
      * Read collection contents from the stream
      *
-     * Counterpart method to {@link #serialize(VariableByteOutputStream)}
+     * Counterpart method to {@link #serialize(VariableByteOutput)}
      *
      * @param broker The database broker
      * @param path The path of the Collection

exist-core/src/main/java/org/exist/dom/QName.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -26,16 +50,19 @@
 import org.exist.util.XMLNames;
 import org.exist.xquery.Constants;
 
+import javax.annotation.Nullable;
 import javax.xml.XMLConstants;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static org.exist.dom.QName.Validity.*;
+import static org.exist.util.StringUtil.isNullOrEmpty;
 
 /**
  * Represents a QName, consisting of a local name, a namespace URI and a prefix.
  *
  * @author <a href="mailto:[email protected]">Wolfgang</a>
+ * @author <a href="mailto:[email protected]">Adam Retter</a>
  */
 public class QName implements Comparable<QName> {
 
@@ -134,7 +161,18 @@ public byte getNameType() {
      * @return the string representation of this qualified name.
      * */
     public String getStringValue() {
-        return getStringRepresentation(false);
+        return getStringRepresentation(false, false);
+    }
+
+    /**
+     * Get an extended string representation of this qualified name.
+     *
+     * Will be of the format `local-name`, `{namespace}local-name`, or `{namespace}prefix:local-name`.
+     *
+     * @return the string representation of this qualified name.
+     */
+    public String getExtendedStringValue() {
+        return getStringRepresentation(false, true);
     }
 
     /**
@@ -146,23 +184,32 @@ public String getStringValue() {
      */
     @Override
     public String toString() {
-        return getStringRepresentation(true);
+        return getStringRepresentation(true, false);
     }
 
     /**
      * Get a string representation of this qualified name.
      *
      * @param showNsWithoutPrefix true if the namespace should be shown even when there is no prefix, false otherwise.
-     *         When shown, it will be output using Clark notation, e.g. `{http://namespace}local-name`.
+     *         When shown, it will be output using Clark notation, e.g. `{namespace}local-name`.
+     *
+     * @param extended true if the namespace and prefix should be shown, requires showNsWithoutPrefix == false.
      *
      * @return the string representation of this qualified name.
      */
-    private String getStringRepresentation(final boolean showNsWithoutPrefix) {
+    private String getStringRepresentation(final boolean showNsWithoutPrefix, final boolean extended) {
         if (prefix != null && !prefix.isEmpty()) {
-            return prefix + COLON + localPart;
-        } else if (showNsWithoutPrefix && namespaceURI != null && !XMLConstants.NULL_NS_URI.equals(namespaceURI)) {
+            if (extended) {
+                return LEFT_BRACE + namespaceURI + RIGHT_BRACE + prefix + COLON + localPart;
+            } else {
+                return prefix + COLON + localPart;
+            }
+        }
+
+        if (showNsWithoutPrefix && namespaceURI != null && !XMLConstants.NULL_NS_URI.equals(namespaceURI)) {
             return LEFT_BRACE + namespaceURI + RIGHT_BRACE + localPart;
         }
+
         return localPart;
     }
 
@@ -330,6 +377,52 @@ public static QName parse(final String namespaceURI, final String qname) throws
         return new QName(qname.substring(p + 1), namespaceURI, qname.substring(0, p));
     }
 
+    /**
+     * Extract a QName from a namespace and qualified name string.
+     *
+     * @param extendedStringValue a string representation as produced by {@link #getExtendedStringValue()}, i.e.: `local-name`, `{namespace}local-name`, or `{namespace}prefix:local-name`.
+     * @return The QName
+     * @throws IllegalQNameException if the qname component is invalid
+     */
+    public static QName parse(String extendedStringValue) throws IllegalQNameException {
+        if (isNullOrEmpty(extendedStringValue)) {
+            throw new IllegalQNameException(ILLEGAL_FORMAT.val, "Illegal extended string QName is empty");
+        }
+
+        final String namespaceUri;
+        if (extendedStringValue.charAt(0) == LEFT_BRACE) {
+            final int idxNsEnd = extendedStringValue.indexOf(RIGHT_BRACE);
+            if (idxNsEnd == Constants.STRING_NOT_FOUND) {
+                throw new IllegalQNameException(ILLEGAL_FORMAT.val, "Illegal extended string QName, missing right brace: '" + extendedStringValue + "'");
+            }
+            namespaceUri = extendedStringValue.substring(1, idxNsEnd);
+            extendedStringValue = extendedStringValue.substring(idxNsEnd + 1);
+        } else if (extendedStringValue.indexOf(RIGHT_BRACE) != Constants.STRING_NOT_FOUND) {
+            throw new IllegalQNameException(ILLEGAL_FORMAT.val, "Illegal extended string QName, missing left brace: '" + extendedStringValue + "'");
+        } else {
+            namespaceUri = XMLConstants.NULL_NS_URI;
+        }
+
+        @Nullable final String prefix;
+        final int idxColon = extendedStringValue.indexOf(COLON);
+        if (idxColon == Constants.STRING_NOT_FOUND) {
+            prefix = null;
+        } else {
+            prefix = extendedStringValue.substring(0, idxColon);
+            if (!XMLNames.isNCName(prefix)) {
+                throw new IllegalQNameException(INVALID_PREFIX.val, "Illegal extended string QName, invalid prefix: '" + extendedStringValue + "'");
+            }
+            extendedStringValue = extendedStringValue.substring(idxColon + 1);
+        }
+
+        final String localPart = extendedStringValue;
+        if (!XMLNames.isNCName(localPart)) {
+            throw new IllegalQNameException(INVALID_LOCAL_PART.val, "Illegal extended string QName, invalid prefix: '" + extendedStringValue + "'");
+        }
+
+        return new QName(localPart, namespaceUri, prefix);
+    }
+
     /**
      * Parses the given string into a QName. The method uses context to look up
      * a namespace URI for an existing prefix.

exist-core/src/main/java/org/exist/dom/persistent/BinaryDocument.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -27,7 +51,7 @@
 import org.exist.storage.BrokerPool;
 import org.exist.storage.blob.BlobId;
 import org.exist.storage.io.VariableByteInput;
-import org.exist.storage.io.VariableByteOutputStream;
+import org.exist.storage.io.VariableByteOutput;
 import org.exist.xmldb.XmldbURI;
 import org.exist.xquery.Expression;
 import org.w3c.dom.DocumentType;
@@ -167,7 +191,7 @@ public void setBlobId(final BlobId blobId) {
     }
 
     @Override
-    public void write(final VariableByteOutputStream ostream) throws IOException {
+    public void write(final VariableByteOutput ostream) throws IOException {
         ostream.writeInt(getDocId());
         ostream.writeUTF(getFileURI().toString());
 

exist-core/src/main/java/org/exist/dom/persistent/DocumentImpl.java

@@ -59,7 +59,7 @@
 import org.exist.security.*;
 import org.exist.storage.*;
 import org.exist.storage.io.VariableByteInput;
-import org.exist.storage.io.VariableByteOutputStream;
+import org.exist.storage.io.VariableByteOutput;
 import org.exist.storage.lock.EnsureContainerLocked;
 import org.exist.storage.lock.EnsureLocked;
 import org.exist.storage.txn.Txn;
@@ -859,11 +859,11 @@ public void appendChild(final NodeHandle child) throws DOMException {
     /**
      * The method <code>write</code>
      *
-     * @param ostream a <code>VariableByteOutputStream</code> value
+     * @param ostream a <code>VariableByteOutput</code> value
      * @throws IOException if an error occurs
      */
     @EnsureContainerLocked(mode=READ_LOCK)
-    public void write(final VariableByteOutputStream ostream) throws IOException {
+    public void write(final VariableByteOutput ostream) throws IOException {
         try {
             ostream.writeInt(docId);
             ostream.writeUTF(fileURI.toString());
@@ -885,7 +885,7 @@ public void write(final VariableByteOutputStream ostream) throws IOException {
         }
     }
 
-    void writeDocumentAttributes(final SymbolTable symbolTable, final VariableByteOutputStream ostream) throws IOException {
+    void writeDocumentAttributes(final SymbolTable symbolTable, final VariableByteOutput ostream) throws IOException {
         ostream.writeLong(created);
         ostream.writeLong(lastModified);
         ostream.writeInt(symbolTable.getMimeTypeId(mimeType));

exist-core/src/main/java/org/exist/dom/persistent/DocumentMetadata.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -23,7 +47,7 @@
 
 import org.exist.ResourceMetadata;
 import org.exist.storage.io.VariableByteInput;
-import org.exist.storage.io.VariableByteOutputStream;
+import org.exist.storage.io.VariableByteOutput;
 import org.w3c.dom.DocumentType;
 
 import java.io.IOException;
@@ -100,7 +124,7 @@ public void decPageCount() {
     }
 
     @Deprecated
-    public void write(final SymbolTable symbolTable, final VariableByteOutputStream ostream) throws IOException {
+    public void write(final SymbolTable symbolTable, final VariableByteOutput ostream) throws IOException {
         doc.writeDocumentAttributes(symbolTable, ostream);
     }