diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 3b36d3777a712..a8f5cb0e94b98 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -17,6 +17,9 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.UUIDs; +import org.opensearch.index.store.metadata.RemoteSegmentMetadata; +import org.opensearch.index.store.metadata.RemoteSegmentMetadataManager; +import org.opensearch.index.store.metadata.SegmentMetadataParser; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -75,6 +78,8 @@ public final class RemoteSegmentStoreDirectory extends FilterDirectory { */ private Map segmentsUploadedToRemoteStore; + private RemoteSegmentMetadataManager remoteMetadataManager; + private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectory.class); public RemoteSegmentStoreDirectory(RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory) throws IOException { @@ -93,6 +98,7 @@ public RemoteSegmentStoreDirectory(RemoteDirectory remoteDataDirectory, RemoteDi */ public void init() throws IOException { this.commonFilenameSuffix = UUIDs.base64UUID(); + this.remoteMetadataManager = new RemoteSegmentMetadataManager(new SegmentMetadataParser()); this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(readLatestMetadataFile()); } @@ -126,10 +132,8 @@ private Map readLatestMetadataFile() throws IOE private Map readMetadataFile(String metadataFilename) throws IOException { try (IndexInput indexInput = remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)) { - Map segmentMetadata = indexInput.readMapOfStrings(); - return segmentMetadata.entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> UploadedSegmentMetadata.fromString(entry.getValue()))); + RemoteSegmentMetadata metadata = this.remoteMetadataManager.readMetadata(indexInput); + return metadata.getMap(); } } @@ -138,7 +142,10 @@ private Map readMetadataFile(String metadataFil */ public static class UploadedSegmentMetadata { // Visible for testing + static final int CURRENT_VERSION = 1; + static final String METADATA_CODEC = "segment_md"; static final String SEPARATOR = "::"; + private final String originalFilename; private final String uploadedFilename; private final String checksum; @@ -353,7 +360,7 @@ public void uploadMetadata(Collection segmentFiles, Directory storeDirec throw new NoSuchFileException(file); } } - indexOutput.writeMapOfStrings(uploadedSegments); + this.remoteMetadataManager.writeMetadata(indexOutput, uploadedSegments); indexOutput.close(); storeDirectory.sync(Collections.singleton(metadataFilename)); remoteMetadataDirectory.copyFrom(storeDirectory, metadataFilename, metadataFilename, IOContext.DEFAULT); diff --git a/server/src/main/java/org/opensearch/index/store/metadata/MetadataParser.java b/server/src/main/java/org/opensearch/index/store/metadata/MetadataParser.java new file mode 100644 index 0000000000000..11a9eb7edfa03 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/metadata/MetadataParser.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.metadata; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; +import java.util.Map; + +public interface MetadataParser { + T readContent(IndexInput indexInput) throws IOException; + + void writeContent(IndexOutput indexOutput, T content) throws IOException; + + /** + * This method is to be removed in future and above method is supposed to be used + * @param indexOutput + * @param content + * @throws IOException + */ + @Deprecated + void writeContent(IndexOutput indexOutput, Map content) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/store/metadata/RemoteSegmentMetadata.java b/server/src/main/java/org/opensearch/index/store/metadata/RemoteSegmentMetadata.java new file mode 100644 index 0000000000000..c6d7be0e9a589 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/metadata/RemoteSegmentMetadata.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.metadata; + +import java.util.Map; +import java.util.stream.Collectors; + +import org.opensearch.index.store.RemoteSegmentStoreDirectory; + +public class RemoteSegmentMetadata { + private final Map metadata; + + public RemoteSegmentMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getMetadata() { + return this.metadata; + } + + public static RemoteSegmentMetadata fromMapOfStrings(Map segmentMetadata) { + return new RemoteSegmentMetadata( + segmentMetadata.entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString(entry.getValue()) + ) + ) + ); + } + + /** + * Ideally we shouldn't need expose internal data structures. all operations should be added into this class + * @return + */ + public Map getMap() { + return this.metadata; + } +} diff --git a/server/src/main/java/org/opensearch/index/store/metadata/RemoteSegmentMetadataManager.java b/server/src/main/java/org/opensearch/index/store/metadata/RemoteSegmentMetadataManager.java new file mode 100644 index 0000000000000..0d983c1c897d8 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/metadata/RemoteSegmentMetadataManager.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.metadata; + +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.BufferedChecksumIndexInput; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +public class RemoteSegmentMetadataManager { + static final int CURRENT_VERSION = 1; + static final String METADATA_CODEC = "segment_md"; + + // replace with a parser factory to support multiple versions of metadata in future if needed + private final MetadataParser parser; + + public RemoteSegmentMetadataManager(MetadataParser parser) { + this.parser = parser; + } + + public RemoteSegmentMetadata readMetadata(IndexInput indexInput) throws IOException { + ChecksumIndexInput checksumIndexInput = new BufferedChecksumIndexInput(indexInput); + checkHeader(checksumIndexInput); + RemoteSegmentMetadata metadata = this.parser.readContent(checksumIndexInput); + checkFooter(checksumIndexInput); + return metadata; + } + + /** + * this method should only accept RemoteSegmentMetadata and not Map + * @param indexOutput + * @param metadata + * @throws IOException + */ + public void writeMetadata(IndexOutput indexOutput, Map metadata) throws IOException { + this.writeHeader(indexOutput); + this.parser.writeContent(indexOutput, metadata); + this.writeFooter(indexOutput); + } + + private int checkHeader(IndexInput indexInput) throws IOException { + return CodecUtil.checkHeader(indexInput, METADATA_CODEC, CURRENT_VERSION, CURRENT_VERSION); + } + + private void checkFooter(ChecksumIndexInput indexInput) throws IOException { + CodecUtil.checkFooter(indexInput); + } + + private void writeHeader(IndexOutput indexOutput) throws IOException { + CodecUtil.writeHeader(indexOutput, METADATA_CODEC, CURRENT_VERSION); + } + + private void writeFooter(IndexOutput indexOutput) throws IOException { + CodecUtil.writeFooter(indexOutput); + } +} diff --git a/server/src/main/java/org/opensearch/index/store/metadata/SegmentMetadataParser.java b/server/src/main/java/org/opensearch/index/store/metadata/SegmentMetadataParser.java new file mode 100644 index 0000000000000..3f28873e2305c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/metadata/SegmentMetadataParser.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.metadata; + +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +public class SegmentMetadataParser implements MetadataParser { + @Override + public RemoteSegmentMetadata readContent(IndexInput indexInput) throws IOException { + return RemoteSegmentMetadata.fromMapOfStrings(indexInput.readMapOfStrings()); + } + + @Override + public void writeContent(IndexOutput indexOutput, RemoteSegmentMetadata content) throws IOException {} + + @Override + public void writeContent(IndexOutput indexOutput, Map content) throws IOException { + indexOutput.writeMapOfStrings(content); + } +} diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 3f1c20f0d88a5..514f86948ebef 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -9,14 +9,21 @@ package org.opensearch.index.store; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.OutputStreamIndexOutput; import org.apache.lucene.tests.util.LuceneTestCase; import org.junit.Before; import org.opensearch.common.UUIDs; +import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.collect.Set; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -28,9 +35,11 @@ import java.util.Map; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.startsWith; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -148,6 +157,26 @@ private Map getDummyMetadata(String prefix, int commitGeneration return metadata; } + /** + * Prepares metadata file bytes with header and footer + * @param segmentFilesMap: actual metadata content + * @return ByteArrayIndexInput: metadata file bytes with header and footer + * @throws IOException + */ + private ByteArrayIndexInput createMetadataFileBytes(Map segmentFilesMap) throws IOException { + BytesStreamOutput output = new BytesStreamOutput(); + OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + CodecUtil.writeHeader( + indexOutput, + RemoteSegmentStoreDirectory.UploadedSegmentMetadata.METADATA_CODEC, + RemoteSegmentStoreDirectory.UploadedSegmentMetadata.CURRENT_VERSION + ); + indexOutput.writeMapOfStrings(segmentFilesMap); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + return new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + } + private Map> populateMetadata() throws IOException { List metadataFiles = List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv"); when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( @@ -163,17 +192,16 @@ private Map> populateMetadata() throws IOException { getDummyMetadata("_0", 1) ); - IndexInput indexInput1 = mock(IndexInput.class); - when(indexInput1.readMapOfStrings()).thenReturn(metadataFilenameContentMapping.get("metadata__1__5__abc")); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(indexInput1); - - IndexInput indexInput2 = mock(IndexInput.class); - when(indexInput2.readMapOfStrings()).thenReturn(metadataFilenameContentMapping.get("metadata__1__6__pqr")); - when(remoteMetadataDirectory.openInput("metadata__1__6__pqr", IOContext.DEFAULT)).thenReturn(indexInput2); - - IndexInput indexInput3 = mock(IndexInput.class); - when(indexInput3.readMapOfStrings()).thenReturn(metadataFilenameContentMapping.get("metadata__2__1__zxv")); - when(remoteMetadataDirectory.openInput("metadata__2__1__zxv", IOContext.DEFAULT)).thenReturn(indexInput3); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn( + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__5__abc")) + ); + when(remoteMetadataDirectory.openInput("metadata__1__6__pqr", IOContext.DEFAULT)).thenReturn( + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__1__6__pqr")) + ); + when(remoteMetadataDirectory.openInput("metadata__2__1__zxv", IOContext.DEFAULT)).thenReturn( + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv")), + createMetadataFileBytes(metadataFilenameContentMapping.get("metadata__2__1__zxv")) + ); return metadataFilenameContentMapping; } @@ -354,11 +382,7 @@ public void testContainsFile() throws IOException { metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); - Map> metadataFilenameContentMapping = Map.of("metadata__1__5__abc", metadata); - - IndexInput indexInput1 = mock(IndexInput.class); - when(indexInput1.readMapOfStrings()).thenReturn(metadataFilenameContentMapping.get("metadata__1__5__abc")); - when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(indexInput1); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(createMetadataFileBytes(metadata)); remoteSegmentStoreDirectory.init(); @@ -394,7 +418,8 @@ public void testUploadMetadataNonEmpty() throws IOException { remoteSegmentStoreDirectory.init(); Directory storeDirectory = mock(Directory.class); - IndexOutput indexOutput = mock(IndexOutput.class); + BytesStreamOutput output = new BytesStreamOutput(); + IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); Collection segmentFiles = List.of("_0.si"); @@ -407,7 +432,129 @@ public void testUploadMetadataNonEmpty() throws IOException { eq(IOContext.DEFAULT) ); String metadataString = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().get("_0.si").toString(); - verify(indexOutput).writeMapOfStrings(Map.of("_0.si", metadataString)); + + ByteArrayIndexInput expectedMetadataFileContent = createMetadataFileBytes(Map.of("_0.si", metadataString)); + int expectedBytesLength = (int) expectedMetadataFileContent.length(); + byte[] expectedBytes = new byte[expectedBytesLength]; + expectedMetadataFileContent.readBytes(expectedBytes, 0, expectedBytesLength); + + assertArrayEquals(expectedBytes, BytesReference.toBytes(output.bytes())); + } + + public void testNoMetadataHeaderCorruptIndexException() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + Map metadata = new HashMap<>(); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + + BytesStreamOutput output = new BytesStreamOutput(); + OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + indexOutput.writeMapOfStrings(metadata); + indexOutput.close(); + ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + + assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); + } + + public void testInvalidCodecHeaderCorruptIndexException() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + Map metadata = new HashMap<>(); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + + BytesStreamOutput output = new BytesStreamOutput(); + OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + CodecUtil.writeHeader(indexOutput, "invalidCodec", RemoteSegmentStoreDirectory.UploadedSegmentMetadata.CURRENT_VERSION); + indexOutput.writeMapOfStrings(metadata); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + + assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); + } + + public void testHeaderMinVersionCorruptIndexException() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + Map metadata = new HashMap<>(); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + + BytesStreamOutput output = new BytesStreamOutput(); + OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + CodecUtil.writeHeader(indexOutput, RemoteSegmentStoreDirectory.UploadedSegmentMetadata.METADATA_CODEC, -1); + indexOutput.writeMapOfStrings(metadata); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + + assertThrows(IndexFormatTooOldException.class, () -> remoteSegmentStoreDirectory.init()); + } + + public void testHeaderMaxVersionCorruptIndexException() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + Map metadata = new HashMap<>(); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + + BytesStreamOutput output = new BytesStreamOutput(); + OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + CodecUtil.writeHeader(indexOutput, RemoteSegmentStoreDirectory.UploadedSegmentMetadata.METADATA_CODEC, 2); + indexOutput.writeMapOfStrings(metadata); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + + assertThrows(IndexFormatTooNewException.class, () -> remoteSegmentStoreDirectory.init()); + } + + public void testIncorrectChecksumCorruptIndexException() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + Map metadata = new HashMap<>(); + metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234"); + metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345"); + + BytesStreamOutput output = new BytesStreamOutput(); + IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096); + IndexOutput wrappedIndexOutput = new WrapperIndexOutput(indexOutput); + IndexOutput indexOutputSpy = spy(wrappedIndexOutput); + CodecUtil.writeHeader( + indexOutputSpy, + RemoteSegmentStoreDirectory.UploadedSegmentMetadata.METADATA_CODEC, + RemoteSegmentStoreDirectory.UploadedSegmentMetadata.CURRENT_VERSION + ); + indexOutputSpy.writeMapOfStrings(metadata); + doReturn(12345L).when(indexOutputSpy).getChecksum(); + CodecUtil.writeFooter(indexOutputSpy); + indexOutputSpy.close(); + + ByteArrayIndexInput byteArrayIndexInput = new ByteArrayIndexInput("segment metadata", BytesReference.toBytes(output.bytes())); + when(remoteMetadataDirectory.openInput("metadata__1__5__abc", IOContext.DEFAULT)).thenReturn(byteArrayIndexInput); + + assertThrows(CorruptIndexException.class, () -> remoteSegmentStoreDirectory.init()); } public void testDeleteStaleCommitsException() throws IOException { @@ -489,4 +636,65 @@ public void testDeleteStaleCommitsActualDeleteNoSuchFileException() throws IOExc ; verify(remoteMetadataDirectory).deleteFile("metadata__1__5__abc"); } + + public void testSegmentMetadataCurrentVersion() { + /* + This is a fake test which will fail whenever the CURRENT_VERSION is incremented. + This is to bring attention of the author towards backward compatibility of metadata files. + If there is any breaking change the author needs to specify how old metadata file will be supported after + this change + If author doesn't want to support old metadata files. Then this can be ignored. + After taking appropriate action, fix this test by setting the correct version here + */ + assertEquals(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.CURRENT_VERSION, 1); + } + + private static class WrapperIndexOutput extends IndexOutput { + public IndexOutput indexOutput; + + public WrapperIndexOutput(IndexOutput indexOutput) { + super(indexOutput.toString(), indexOutput.getName()); + this.indexOutput = indexOutput; + } + + @Override + public final void writeByte(byte b) throws IOException { + this.indexOutput.writeByte(b); + } + + @Override + public final void writeBytes(byte[] b, int offset, int length) throws IOException { + this.indexOutput.writeBytes(b, offset, length); + } + + @Override + public void writeShort(short i) throws IOException { + this.indexOutput.writeShort(i); + } + + @Override + public void writeInt(int i) throws IOException { + this.indexOutput.writeInt(i); + } + + @Override + public void writeLong(long i) throws IOException { + this.indexOutput.writeLong(i); + } + + @Override + public void close() throws IOException { + this.indexOutput.close(); + } + + @Override + public final long getFilePointer() { + return this.indexOutput.getFilePointer(); + } + + @Override + public long getChecksum() throws IOException { + return this.indexOutput.getChecksum(); + } + } }