Skip to content

Commit 76e4aad

Browse files
Rich-T-kidRich-T-kid
committed
fix lint
1 parent d90e9df commit 76e4aad

1 file changed

Lines changed: 67 additions & 68 deletions

File tree

  • datafusion/datasource-parquet/src/opener

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 67 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -998,81 +998,80 @@ impl MetadataLoadedParquetOpen {
998998
&& let Some(rg) = reader_metadata.metadata().row_groups().first()
999999
{
10001000
let rewrite_field = |field: &FieldRef, col_idx: usize| -> FieldRef {
1001-
// dictionary_page_offset is only set when the column was written with
1002-
// dictionary encoding. Plain-encoded string/binary columns will have
1003-
// no offset and pass through unchanged.
1004-
let is_dict = rg.column(col_idx).dictionary_page_offset().is_some();
1005-
let dict_value_type = match field.data_type() {
1006-
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
1007-
Some(DataType::Utf8)
1008-
}
1009-
DataType::Binary
1010-
| DataType::LargeBinary
1011-
| DataType::BinaryView => Some(DataType::Binary),
1012-
_ => None,
1013-
};
1014-
if is_dict && let Some(value_type) = dict_value_type {
1015-
return Arc::new(Field::new(
1016-
field.name(),
1017-
DataType::Dictionary(
1018-
Box::new(DataType::Int32),
1019-
Box::new(value_type),
1020-
),
1021-
field.is_nullable(),
1022-
));
1001+
// dictionary_page_offset is only set when the column was written with
1002+
// dictionary encoding. Plain-encoded string/binary columns will have
1003+
// no offset and pass through unchanged.
1004+
let is_dict = rg.column(col_idx).dictionary_page_offset().is_some();
1005+
let dict_value_type = match field.data_type() {
1006+
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
1007+
Some(DataType::Utf8)
1008+
}
1009+
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
1010+
Some(DataType::Binary)
10231011
}
1024-
Arc::clone(field)
1012+
_ => None,
10251013
};
1014+
if is_dict && let Some(value_type) = dict_value_type {
1015+
return Arc::new(Field::new(
1016+
field.name(),
1017+
DataType::Dictionary(
1018+
Box::new(DataType::Int32),
1019+
Box::new(value_type),
1020+
),
1021+
field.is_nullable(),
1022+
));
1023+
}
1024+
Arc::clone(field)
1025+
};
1026+
1027+
let new_physical: Vec<FieldRef> = physical_file_schema
1028+
.fields()
1029+
.iter()
1030+
.enumerate()
1031+
.map(|(col_idx, field)| rewrite_field(field, col_idx))
1032+
.collect();
1033+
let new_physical_schema = Arc::new(Schema::new_with_metadata(
1034+
new_physical,
1035+
physical_file_schema.metadata().clone(),
1036+
));
10261037

1027-
let new_physical: Vec<FieldRef> = physical_file_schema
1038+
// True when at least one string/binary column had a dictionary page,
1039+
// meaning we need to update the schemas and reader options.
1040+
if new_physical_schema != physical_file_schema {
1041+
let dict_type_overrides: HashMap<&str, &DataType> = new_physical_schema
10281042
.fields()
10291043
.iter()
1030-
.enumerate()
1031-
.map(|(col_idx, field)| rewrite_field(field, col_idx))
1044+
.filter_map(|field| {
1045+
matches!(field.data_type(), DataType::Dictionary(_, _))
1046+
.then_some((field.name().as_str(), field.data_type()))
1047+
})
10321048
.collect();
1033-
let new_physical_schema = Arc::new(Schema::new_with_metadata(
1034-
new_physical,
1035-
physical_file_schema.metadata().clone(),
1036-
));
10371049

1038-
// True when at least one string/binary column had a dictionary page,
1039-
// meaning we need to update the schemas and reader options.
1040-
if new_physical_schema != physical_file_schema {
1041-
let dict_type_overrides: HashMap<&str, &DataType> =
1042-
new_physical_schema
1043-
.fields()
1044-
.iter()
1045-
.filter_map(|field| {
1046-
matches!(field.data_type(), DataType::Dictionary(_, _))
1047-
.then_some((field.name().as_str(), field.data_type()))
1048-
})
1049-
.collect();
1050-
1051-
let new_output_fields: Vec<FieldRef> = prepared
1052-
.output_schema
1053-
.fields()
1054-
.iter()
1055-
.map(|output_field| {
1056-
match dict_type_overrides.get(output_field.name().as_str()) {
1057-
Some(&dict_ty) => Arc::new(Field::new(
1058-
output_field.name(),
1059-
dict_ty.clone(),
1060-
output_field.is_nullable(),
1061-
)),
1062-
None => Arc::clone(output_field),
1063-
}
1064-
})
1065-
.collect();
1066-
1067-
physical_file_schema = new_physical_schema;
1068-
prepared.logical_file_schema = Arc::clone(&physical_file_schema);
1069-
prepared.output_schema = Arc::new(Schema::new_with_metadata(
1070-
new_output_fields,
1071-
prepared.output_schema.metadata().clone(),
1072-
));
1073-
options = options.with_schema(Arc::clone(&physical_file_schema));
1074-
metadata_dirty = true;
1075-
}
1050+
let new_output_fields: Vec<FieldRef> = prepared
1051+
.output_schema
1052+
.fields()
1053+
.iter()
1054+
.map(|output_field| {
1055+
match dict_type_overrides.get(output_field.name().as_str()) {
1056+
Some(&dict_ty) => Arc::new(Field::new(
1057+
output_field.name(),
1058+
dict_ty.clone(),
1059+
output_field.is_nullable(),
1060+
)),
1061+
None => Arc::clone(output_field),
1062+
}
1063+
})
1064+
.collect();
1065+
1066+
physical_file_schema = new_physical_schema;
1067+
prepared.logical_file_schema = Arc::clone(&physical_file_schema);
1068+
prepared.output_schema = Arc::new(Schema::new_with_metadata(
1069+
new_output_fields,
1070+
prepared.output_schema.metadata().clone(),
1071+
));
1072+
options = options.with_schema(Arc::clone(&physical_file_schema));
1073+
metadata_dirty = true;
1074+
}
10761075
}
10771076

10781077
// Arrow-rs appends virtual columns to the supplied schema internally,

0 commit comments

Comments
 (0)