Skip to content

Commit 8566559

Browse files
martijnvgmartijnvg
committed
Store ignored source entries into stored fields with different name using _ignored_source. as prefix.
1 parent c376155 commit 8566559

File tree

12 files changed

+69
-48
lines changed

12 files changed

+69
-48
lines changed

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import org.apache.lucene.index.FieldInfo;
1212
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
13+
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
1314

1415
import java.util.HashSet;
1516
import java.util.Set;
@@ -50,6 +51,10 @@ public Status needsField(FieldInfo fieldInfo) {
5051
if (fields.contains(fieldInfo.name)) {
5152
return Status.YES;
5253
}
54+
55+
if (fieldInfo.name.startsWith(IgnoredSourceFieldMapper.NAME)) {
56+
return Status.YES;
57+
}
5358
return Status.NO;
5459
}
5560
}

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import org.elasticsearch.common.CheckedBiConsumer;
1717
import org.elasticsearch.common.bytes.BytesReference;
1818
import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
19-
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
2019
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
2120
import org.elasticsearch.search.fetch.StoredFieldsSpec;
2221

@@ -29,23 +28,36 @@
2928

3029
class IgnoredSourceFieldLoader extends StoredFieldLoader {
3130

32-
final Set<String> potentialFieldsInIgnoreSource;
31+
final Set<String> potentialFieldsToLoad;
3332

3433
IgnoredSourceFieldLoader(StoredFieldsSpec spec) {
35-
Set<String> potentialFieldsInIgnoreSource = new HashSet<>();
34+
Set<String> potentialFieldsToLoad = new HashSet<>();
3635
for (String requiredStoredField : spec.requiredStoredFields()) {
3736
if (requiredStoredField.startsWith(IgnoredSourceFieldMapper.NAME)) {
3837
String fieldName = requiredStoredField.substring(IgnoredSourceFieldMapper.NAME.length());
39-
potentialFieldsInIgnoreSource.addAll(FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(fieldName));
38+
potentialFieldsToLoad.addAll(splitIntoFieldPaths(fieldName));
4039
}
4140
}
42-
this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource;
41+
this.potentialFieldsToLoad = potentialFieldsToLoad;
42+
}
43+
44+
static Set<String> splitIntoFieldPaths(String fieldName) {
45+
var paths = new HashSet<String>();
46+
var current = new StringBuilder();
47+
for (var part : fieldName.split("\\.")) {
48+
if (current.isEmpty() == false) {
49+
current.append('.');
50+
}
51+
current.append(part);
52+
paths.add(IgnoredSourceFieldMapper.NAME + "." + current);
53+
}
54+
return paths;
4355
}
4456

4557
@Override
4658
public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException {
4759
var reader = sequentialReader(ctx);
48-
var visitor = new SFV(potentialFieldsInIgnoreSource);
60+
var visitor = new SFV(potentialFieldsToLoad);
4961
return new LeafStoredFieldLoader() {
5062

5163
private int doc = -1;
@@ -83,42 +95,42 @@ public Map<String, List<Object>> storedFields() {
8395

8496
@Override
8597
public List<String> fieldsToLoad() {
86-
return List.of(IgnoredSourceFieldMapper.NAME);
98+
return List.of(potentialFieldsToLoad.toArray(new String[0]));
8799
}
88100

89101
static class SFV extends StoredFieldVisitor {
90102

91-
boolean done;
103+
boolean found;
92104
final List<Object> values = new ArrayList<>();
93-
final Set<String> potentialFieldsInIgnoreSource;
105+
final Set<String> potentialFieldsToLoad;
94106

95-
SFV(Set<String> potentialFieldsInIgnoreSource) {
96-
this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource;
107+
SFV(Set<String> potentialFieldsToLoad) {
108+
this.potentialFieldsToLoad = potentialFieldsToLoad;
97109
}
98110

99111
@Override
100112
public Status needsField(FieldInfo fieldInfo) throws IOException {
101-
if (done) {
102-
return Status.STOP;
103-
} else if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) {
113+
if (potentialFieldsToLoad.contains(fieldInfo.name)) {
114+
found = true;
104115
return Status.YES;
105116
} else {
106-
return Status.NO;
117+
if (found) {
118+
return Status.STOP;
119+
} else {
120+
return Status.NO;
121+
}
107122
}
108123
}
109124

110125
@Override
111126
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
112-
var result = IgnoredSourceFieldMapper.decodeIfMatch(value, potentialFieldsInIgnoreSource);
113-
if (result != null) {
114-
done = true;
115-
values.add(result);
116-
}
127+
var result = IgnoredSourceFieldMapper.decode(value);
128+
values.add(result);
117129
}
118130

119131
void reset() {
120132
values.clear();
121-
done = false;
133+
found = false;
122134
}
123135

124136
}

‎server/src/main/java/org/elasticsearch/index/get/GetResult.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
244244

245245
for (DocumentField field : metaFields.values()) {
246246
// TODO: can we avoid having an exception here?
247-
if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) {
247+
if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().startsWith(IgnoredSourceFieldMapper.NAME)) {
248248
builder.field(field.getName(), field.getValues());
249249
} else {
250250
builder.field(field.getName(), field.<Object>getValue());

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException
7272
throw new UnsupportedOperationException();
7373
}
7474

75-
public static Set<String> splitIntoFieldPaths(String fieldName) {
75+
static Set<String> splitIntoFieldPaths(String fieldName) {
7676
var paths = new HashSet<String>();
7777
paths.add("_doc");
7878
var current = new StringBuilder();

‎server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,8 @@ public void postParse(DocumentParserContext context) {
160160
}
161161

162162
for (NameValue nameValue : context.getIgnoredFieldValues()) {
163-
nameValue.doc().add(new StoredField(NAME, encode(nameValue)));
163+
String fieldName = NAME + "." + nameValue.name;
164+
nameValue.doc().add(new StoredField(fieldName, encode(nameValue)));
164165
}
165166
}
166167

@@ -176,8 +177,11 @@ static byte[] encode(NameValue values) {
176177
return bytes;
177178
}
178179

179-
static NameValue decode(Object field) {
180+
public static NameValue decode(Object field) {
180181
byte[] bytes = ((BytesRef) field).bytes;
182+
return decode(bytes);
183+
}
184+
public static NameValue decode(byte[] bytes) {
181185
int encodedSize = ByteUtils.readIntLE(bytes, 0);
182186
int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET;
183187
int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET;
@@ -186,20 +190,6 @@ static NameValue decode(Object field) {
186190
return new NameValue(name, parentOffset, value, null);
187191
}
188192

189-
public static NameValue decodeIfMatch(byte[] bytes, Set<String> potentialFieldsInIgnoreSource) {
190-
int encodedSize = ByteUtils.readIntLE(bytes, 0);
191-
int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET;
192-
int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET;
193-
194-
String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8);
195-
if (potentialFieldsInIgnoreSource.contains(name)) {
196-
BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4);
197-
return new NameValue(name, parentOffset, value, null);
198-
} else {
199-
return null;
200-
}
201-
}
202-
203193
// In rare cases decoding values stored in this field can fail leading to entire source
204194
// not being available.
205195
// We would like to have an option to lose some values in synthetic source

‎server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ public void write(LeafStoredFieldLoader storedFieldLoader, int docId, XContentBu
211211
if (loader != null) {
212212
loader.load(e.getValue());
213213
}
214-
if (IgnoredSourceFieldMapper.NAME.equals(e.getKey())) {
214+
if (e.getKey().startsWith(IgnoredSourceFieldMapper.NAME)) {
215215
for (Object value : e.getValue()) {
216216
if (objectsWithIgnoredFields == null) {
217217
objectsWithIgnoredFields = new HashMap<>();

‎server/src/main/java/org/elasticsearch/search/SearchHit.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -878,7 +878,7 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
878878
}
879879
// _ignored is the only multi-valued meta field
880880
// TODO: can we avoid having an exception here?
881-
if (IgnoredFieldMapper.NAME.equals(field.getName()) || IgnoredSourceFieldMapper.NAME.equals(field.getName())) {
881+
if (IgnoredFieldMapper.NAME.equals(field.getName()) || field.getName().startsWith(IgnoredSourceFieldMapper.NAME)) {
882882
builder.field(field.getName(), field.getValues());
883883
} else {
884884
builder.field(field.getName(), field.<Object>getValue());

‎server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.apache.lucene.index.LeafReaderContext;
1313
import org.apache.lucene.util.SetOnce;
1414
import org.elasticsearch.index.mapper.IdFieldMapper;
15+
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
1516
import org.elasticsearch.search.lookup.FieldLookup;
1617
import org.elasticsearch.search.lookup.LeafFieldLookupProvider;
1718

@@ -21,6 +22,7 @@
2122
import java.util.Map;
2223
import java.util.Set;
2324
import java.util.function.Supplier;
25+
import java.util.stream.Collectors;
2426

2527
/**
2628
* Makes pre-loaded stored fields available via a LeafSearchLookup.
@@ -61,7 +63,13 @@ void setPreloadedStoredFieldNames(Set<String> preloadedStoredFieldNames) {
6163
}
6264

6365
void setPreloadedStoredFieldValues(String id, Map<String, List<Object>> preloadedStoredFieldValues) {
64-
assert preloadedStoredFieldNames.get().containsAll(preloadedStoredFieldValues.keySet())
66+
assert preloadedStoredFieldNames.get()
67+
.containsAll(
68+
preloadedStoredFieldValues.keySet()
69+
.stream()
70+
.filter(s -> s.startsWith(IgnoredSourceFieldMapper.NAME) == false)
71+
.collect(Collectors.toSet())
72+
)
6573
: "Provided stored field that was not expected to be preloaded? "
6674
+ preloadedStoredFieldValues.keySet()
6775
+ " - "

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
106106
}
107107
final MappedFieldType fieldType = searchExecutionContext.getFieldType(matchingFieldName);
108108
// NOTE: Exclude _ignored_source when requested via wildcard '*'
109-
if (matchingFieldName.equals(IgnoredSourceFieldMapper.NAME) && Regex.isSimpleMatchPattern(storedField)) {
109+
if (matchingFieldName.startsWith(IgnoredSourceFieldMapper.NAME) && Regex.isSimpleMatchPattern(storedField)) {
110110
continue;
111111
}
112112
// NOTE: checking if the field is stored is required for backward compatibility reasons and to make

‎server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ public void testDisableIgnoredSourceRead() throws IOException {
5252

5353
var doc = mapperService.documentMapper().parse(source(inputDocument));
5454
// Field was written.
55-
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
55+
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME + ".fallback_field"));
56+
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME + ".disabled_object"));
5657

5758
String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
5859
// Values are not loaded.
@@ -64,7 +65,8 @@ public void testDisableIgnoredSourceRead() throws IOException {
6465

6566
doc = mapperService.documentMapper().parse(source(inputDocument));
6667
// Field was written.
67-
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
68+
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME + ".fallback_field"));
69+
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME + ".disabled_object"));
6870

6971
syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
7072
// Values are loaded.
@@ -116,7 +118,8 @@ public void testDisableIgnoredSourceWrite() throws IOException {
116118

117119
doc = mapperService.documentMapper().parse(source(inputDocument));
118120
// Field was written.
119-
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
121+
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME + ".fallback_field"));
122+
assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME + ".disabled_object"));
120123

121124
syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
122125
// Values are loaded.

0 commit comments

Comments
 (0)