From 5d0af018811fdfaf0702509a0b2b9787a62ecc67 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 9 Oct 2025 14:56:15 +0200 Subject: [PATCH 1/6] Add FastCSV dependency to pom.xml for CSV processing Signed-off-by: Nicolas Rol --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 3b8cf817ba1..0c546693a2e 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,7 @@ 3.6.1 1.13.1 0.44.0 + 4.0.0 3.0.1 3.0.2 @@ -745,6 +746,11 @@ commons-io ${commonsio.version} + + de.siegmar + fastcsv + ${fastcsv.version} + gov.nist.math jama From de76762466fdac6327f1b296a1f69660e2b6057c Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Thu, 9 Oct 2025 15:03:19 +0200 Subject: [PATCH 2/6] Refactor `StringAnonymizer` to replace Univocity with FastCSV for CSV processing and improve resource management with try-with-resources. Signed-off-by: Nicolas Rol --- commons/pom.xml | 8 +-- .../commons/util/StringAnonymizer.java | 54 +++++++++---------- .../commons/util/StringAnonymizerTest.java | 17 +++--- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index c727f5c7549..3180af3902b 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -60,10 +60,6 @@ com.google.re2j re2j - - com.univocity - univocity-parsers - commons-cli commons-cli @@ -72,6 +68,10 @@ commons-io commons-io + + de.siegmar + fastcsv + net.java.dev.stax-utils stax-utils diff --git a/commons/src/main/java/com/powsybl/commons/util/StringAnonymizer.java b/commons/src/main/java/com/powsybl/commons/util/StringAnonymizer.java index 3bea2fee680..082b583d1d7 100644 --- a/commons/src/main/java/com/powsybl/commons/util/StringAnonymizer.java +++ b/commons/src/main/java/com/powsybl/commons/util/StringAnonymizer.java @@ -10,12 +10,14 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.powsybl.commons.PowsyblException; -import com.univocity.parsers.csv.*; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.CsvRecord; +import de.siegmar.fastcsv.writer.CsvWriter; +import de.siegmar.fastcsv.writer.LineDelimiter; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.util.Arrays; -import java.util.Map; +import java.io.IOException; /** * @author Geoffroy Jamgotchian {@literal } @@ -66,21 +68,19 @@ public void readCsv(BufferedReader reader) { readCsv(reader, DEFAULT_SEPARATOR); } - private static void setFormat(CsvFormat format, char separator) { - format.setLineSeparator(System.lineSeparator()); - format.setDelimiter(separator); - format.setQuoteEscape('"'); - } - public void readCsv(BufferedReader reader, char separator) { - CsvParserSettings settings = new CsvParserSettings(); - setFormat(settings.getFormat(), separator); - CsvParser csvParser = new CsvParser(settings); - for (String[] nextLine : csvParser.iterate(reader)) { - if (nextLine.length != 2) { - throw new PowsyblException("Invalid line '" + Arrays.toString(nextLine) + "'"); - } - mapping.put(nextLine[0], nextLine[1]); + try (CsvReader csvReader = CsvReader.builder() + .fieldSeparator(separator) + .quoteCharacter('"') + .ofCsvRecord(reader)) { + csvReader.forEach(csvRecord -> { + if (csvRecord.getFieldCount() != 2) { + throw new PowsyblException("Invalid line '" + csvRecord + "'"); + } + mapping.put(csvRecord.getField(0), csvRecord.getField(1)); + }); + } catch (IOException e) { + throw new PowsyblException("Failed to read the CSV", e); } } @@ -89,18 +89,14 @@ public void writeCsv(BufferedWriter writer) { } public void writeCsv(BufferedWriter writer, char separator) { - CsvWriterSettings settings = new CsvWriterSettings(); - setFormat(settings.getFormat(), separator); - CsvWriter csvWriter = new CsvWriter(writer, settings); - try { - String[] nextLine = new String[2]; - for (Map.Entry e : mapping.entrySet()) { - nextLine[0] = e.getKey(); - nextLine[1] = e.getValue(); - csvWriter.writeRow(nextLine); - } - } finally { - csvWriter.close(); + try (CsvWriter csvWriter1 = CsvWriter.builder() + .fieldSeparator(separator) + .quoteCharacter('"') + .lineDelimiter(LineDelimiter.PLATFORM) + .build(writer)) { + mapping.forEach(csvWriter1::writeRecord); + } catch (IOException e) { + throw new PowsyblException("Failed to write the CSV", e); } } } diff --git a/commons/src/test/java/com/powsybl/commons/util/StringAnonymizerTest.java b/commons/src/test/java/com/powsybl/commons/util/StringAnonymizerTest.java index c96cbb4991e..e8f5600b402 100644 --- a/commons/src/test/java/com/powsybl/commons/util/StringAnonymizerTest.java +++ b/commons/src/test/java/com/powsybl/commons/util/StringAnonymizerTest.java @@ -7,6 +7,7 @@ */ package com.powsybl.commons.util; +import com.powsybl.commons.PowsyblException; import org.junit.jupiter.api.Test; import java.io.*; @@ -20,11 +21,8 @@ class StringAnonymizerTest { private static String toCsv(StringAnonymizer anonymizer) throws IOException { StringWriter stringWriter = new StringWriter(); - BufferedWriter writer = new BufferedWriter(stringWriter); - try { + try (BufferedWriter writer = new BufferedWriter(stringWriter)) { anonymizer.writeCsv(writer); - } finally { - writer.close(); } return stringWriter.toString(); } @@ -50,14 +48,11 @@ void test() throws IOException { assertEquals("bar", anonymizer.deanonymize(anonymizedBar)); assertNull(anonymizer.anonymize(null)); assertNull(anonymizer.deanonymize(null)); - try { - anonymizer.deanonymize("baz"); - fail(); - } catch (Exception ignored) { - } + PowsyblException exception = assertThrows(PowsyblException.class, () -> anonymizer.deanonymize("baz")); + assertEquals("Mapping not found for anonymized string 'baz'", exception.getMessage()); String csv = toCsv(anonymizer); assertEquals(String.join(System.lineSeparator(), "foo;A", "bar;B") + System.lineSeparator(), - csv.toString()); + csv); StringAnonymizer anonymizer2 = fromCsv(csv); assertEquals("foo", anonymizer2.deanonymize(anonymizedFoo)); assertEquals("bar", anonymizer2.deanonymize(anonymizedBar)); @@ -73,7 +68,7 @@ void invalidFileTest() throws IOException { } @Test - void invalidFileTest2() throws IOException { + void invalidFileTest2() { String csv = String.join(System.lineSeparator(), "C"); assertThrows(RuntimeException.class, () -> fromCsv(csv)); From c7bbbbcacfe847f01549439b011db0015a7300d9 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Mon, 13 Oct 2025 11:08:18 +0200 Subject: [PATCH 3/6] Replace Univocity with FastCSV in IEEE CDF network parsing Signed-off-by: Nicolas Rol --- .../converter/IeeeCdfNetworkFactory.java | 89 ++++++++++--------- 1 file changed, 48 insertions(+), 41 deletions(-) diff --git a/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfNetworkFactory.java b/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfNetworkFactory.java index 3664ba9aec6..60d1a55cedf 100644 --- a/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfNetworkFactory.java +++ b/ieee-cdf/ieee-cdf-converter/src/main/java/com/powsybl/ieeecdf/converter/IeeeCdfNetworkFactory.java @@ -16,10 +16,12 @@ import com.powsybl.ieeecdf.model.IeeeCdfTitle; import com.powsybl.iidm.network.Network; import com.powsybl.iidm.network.NetworkFactory; -import com.univocity.parsers.csv.CsvParser; -import com.univocity.parsers.csv.CsvParserSettings; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.CsvRecord; +import java.io.IOException; import java.time.LocalDate; +import java.util.Objects; import java.util.Properties; import java.util.function.ToDoubleFunction; @@ -136,42 +138,48 @@ public static Network create9zeroimpedance() { return create9zeroimpedance(NetworkFactory.findDefault()); } - private static void parseBuses(IeeeCdfModel model, CsvParserSettings settings, String fileName, double baseKv) { - CsvParser csvParser = new CsvParser(settings); - for (String[] nextLine : csvParser.iterate(IeeeCdfNetworkFactory.class.getResourceAsStream("/" + fileName))) { - int busNo = Integer.parseInt(nextLine[0]); - int busCode = Integer.parseInt(nextLine[1]); - double loadP = Double.parseDouble(nextLine[2]); - double loadQ = Double.parseDouble(nextLine[3]); - IeeeCdfBus bus = new IeeeCdfBus(); - bus.setNumber(busNo); - bus.setName("bus-" + busNo); - bus.setBaseVoltage(baseKv); - bus.setActiveLoad(loadP / 1000); - bus.setReactiveLoad(loadQ / 1000); - if (busCode == 1) { - bus.setType(IeeeCdfBus.Type.HOLD_VOLTAGE_AND_ANGLE); - bus.setDesiredVoltage(1); - } else { - bus.setType(IeeeCdfBus.Type.UNREGULATED); - } - model.getBuses().add(bus); + private static void parseBuses(IeeeCdfModel model, CsvReader.CsvReaderBuilder builder, String fileName, double baseKv) { + try (CsvReader csvReader = builder.ofCsvRecord(Objects.requireNonNull(IeeeCdfNetworkFactory.class.getResourceAsStream("/" + fileName)))) { + csvReader.forEach(csvRecord -> { + int busNo = Integer.parseInt(csvRecord.getField(0)); + int busCode = Integer.parseInt(csvRecord.getField(1)); + double loadP = Double.parseDouble(csvRecord.getField(2)); + double loadQ = Double.parseDouble(csvRecord.getField(3)); + IeeeCdfBus bus = new IeeeCdfBus(); + bus.setNumber(busNo); + bus.setName("bus-" + busNo); + bus.setBaseVoltage(baseKv); + bus.setActiveLoad(loadP / 1000); + bus.setReactiveLoad(loadQ / 1000); + if (busCode == 1) { + bus.setType(IeeeCdfBus.Type.HOLD_VOLTAGE_AND_ANGLE); + bus.setDesiredVoltage(1); + } else { + bus.setType(IeeeCdfBus.Type.UNREGULATED); + } + model.getBuses().add(bus); + }); + } catch (IOException exception) { + throw new PowsyblException("Failed to read the CSV", exception); } } - private static void parseLines(IeeeCdfModel model, CsvParserSettings settings, String fileName) { - CsvParser csvParser = new CsvParser(settings); - for (String[] nextLine : csvParser.iterate(IeeeCdfNetworkFactory.class.getResourceAsStream("/" + fileName))) { - int sendingBus = Integer.parseInt(nextLine[0]); - int receivingBus = Integer.parseInt(nextLine[1]); - double r = Double.parseDouble(nextLine[2]); - double x = Double.parseDouble(nextLine[3]); - IeeeCdfBranch branch = new IeeeCdfBranch(); - branch.setTapBusNumber(sendingBus); - branch.setzBusNumber(receivingBus); - branch.setResistance(r); - branch.setReactance(x); - model.getBranches().add(branch); + private static void parseLines(IeeeCdfModel model, CsvReader.CsvReaderBuilder builder, String fileName) { + try (CsvReader csvReader = builder.ofCsvRecord(Objects.requireNonNull(IeeeCdfNetworkFactory.class.getResourceAsStream("/" + fileName)))) { + csvReader.forEach(csvRecord -> { + int sendingBus = Integer.parseInt(csvRecord.getField(0)); + int receivingBus = Integer.parseInt(csvRecord.getField(1)); + double r = Double.parseDouble(csvRecord.getField(2)); + double x = Double.parseDouble(csvRecord.getField(3)); + IeeeCdfBranch branch = new IeeeCdfBranch(); + branch.setTapBusNumber(sendingBus); + branch.setzBusNumber(receivingBus); + branch.setResistance(r); + branch.setReactance(x); + model.getBranches().add(branch); + }); + } catch (IOException exception) { + throw new PowsyblException("Failed to read the CSV", exception); } } @@ -183,13 +191,12 @@ private static Network createFromCsv(String name, NetworkFactory networkFactory, title.setMvaBase(100); title.setDate(LocalDate.parse("2022-09-23")); IeeeCdfModel model = new IeeeCdfModel(title); - CsvParserSettings settings = new CsvParserSettings(); - settings.getFormat().setLineSeparator(System.lineSeparator()); - settings.getFormat().setDelimiter(" "); - parseBuses(model, settings, name + "-bus.csv", baseKv); - parseLines(model, settings, name + "-line.csv"); + CsvReader.CsvReaderBuilder builder = CsvReader.builder() + .fieldSeparator(" "); + parseBuses(model, builder, name + "-bus.csv", baseKv); + parseLines(model, builder, name + "-line.csv"); if (meshed) { - parseLines(model, settings, name + "-mesh.csv"); + parseLines(model, builder, name + "-mesh.csv"); } return new IeeeCdfImporter().convert(model, networkFactory, name, false); } From 87fbb129a5e90aede942da0c0f248f49e58af8f8 Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Mon, 13 Oct 2025 16:46:50 +0200 Subject: [PATCH 4/6] Refactor EuropeanLvTestFeederFactory to replace Univocity with FastCSV Signed-off-by: Nicolas Rol --- .../test/EuropeanLvTestFeederFactory.java | 253 ++++++++---------- 1 file changed, 118 insertions(+), 135 deletions(-) diff --git a/iidm/iidm-test/src/main/java/com/powsybl/iidm/network/test/EuropeanLvTestFeederFactory.java b/iidm/iidm-test/src/main/java/com/powsybl/iidm/network/test/EuropeanLvTestFeederFactory.java index c32a129d01f..9d5492ae3ae 100644 --- a/iidm/iidm-test/src/main/java/com/powsybl/iidm/network/test/EuropeanLvTestFeederFactory.java +++ b/iidm/iidm-test/src/main/java/com/powsybl/iidm/network/test/EuropeanLvTestFeederFactory.java @@ -8,22 +8,35 @@ package com.powsybl.iidm.network.test; import com.powsybl.commons.PowsyblException; -import com.powsybl.iidm.network.*; -import com.powsybl.iidm.network.extensions.*; -import com.univocity.parsers.annotations.Parsed; -import com.univocity.parsers.common.processor.BeanListProcessor; -import com.univocity.parsers.csv.CsvParser; -import com.univocity.parsers.csv.CsvParserSettings; +import com.powsybl.iidm.network.Bus; +import com.powsybl.iidm.network.Generator; +import com.powsybl.iidm.network.Network; +import com.powsybl.iidm.network.NetworkFactory; +import com.powsybl.iidm.network.Substation; +import com.powsybl.iidm.network.TopologyKind; +import com.powsybl.iidm.network.VoltageLevel; +import com.powsybl.iidm.network.extensions.GeneratorFortescueAdder; +import com.powsybl.iidm.network.extensions.LineFortescueAdder; +import com.powsybl.iidm.network.extensions.LoadAsymmetricalAdder; +import com.powsybl.iidm.network.extensions.LoadConnectionType; +import com.powsybl.iidm.network.extensions.SlackTerminalAdder; +import com.powsybl.iidm.network.extensions.TwoWindingsTransformerFortescueAdder; +import com.powsybl.iidm.network.extensions.WindingConnectionType; +import de.siegmar.fastcsv.reader.CommentStrategy; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.FieldModifiers; +import de.siegmar.fastcsv.reader.NamedCsvRecord; +import de.siegmar.fastcsv.reader.NamedCsvRecordHandler; import org.apache.commons.configuration2.INIConfiguration; import org.apache.commons.configuration2.SubnodeConfiguration; import org.apache.commons.configuration2.ex.ConfigurationException; -import java.time.ZonedDateTime; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.time.ZonedDateTime; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,6 +50,14 @@ */ public final class EuropeanLvTestFeederFactory { + private static final Map, Mapper> MAPPERS = Map.of( + BusCoord.class, (Mapper) EuropeanLvTestFeederFactory::mapBusCoord, + Line.class, (Mapper) EuropeanLvTestFeederFactory::mapLine, + LineCode.class, (Mapper) EuropeanLvTestFeederFactory::mapLineCode, + Load.class, (Mapper) EuropeanLvTestFeederFactory::mapLoad, + Transformer.class, (Mapper) EuropeanLvTestFeederFactory::mapTransformer + ); + private EuropeanLvTestFeederFactory() { } @@ -91,145 +112,84 @@ private static void createSource(Transformer transformer, Network network) { .add(); } - public static class BusCoord { - @Parsed(field = "Busname") - int busName; - - @Parsed - double x; - - @Parsed - double y; + private static BusCoord mapBusCoord(NamedCsvRecord rec) { + return new BusCoord( + Integer.parseInt(rec.getField("Busname")), + Double.parseDouble(rec.getField("x")), + Double.parseDouble(rec.getField("y")) + ); } - public static class Line { - @Parsed(field = "Name") - String name; - - @Parsed(field = "Bus1") - int bus1; - - @Parsed(field = "Bus2") - int bus2; - - @Parsed(field = "Phases") - String phases; - - @Parsed(field = "Length") - double length; - - @Parsed(field = "Units") - String units; - - @Parsed(field = "LineCode") - String code; + private static Line mapLine(NamedCsvRecord rec) { + return new Line( + rec.getField("Name"), + Integer.parseInt(rec.getField("Bus1")), + Integer.parseInt(rec.getField("Bus2")), + rec.getField("Phases"), + Double.parseDouble(rec.getField("Length")), + rec.getField("Units"), + rec.getField("LineCode") + ); } - public static class LineCode { - @Parsed(field = "Name") - String name; - - @Parsed - int nphases; - - @Parsed(field = "R1") - double r1; - - @Parsed(field = "X1") - double x1; - - @Parsed(field = "R0") - double r0; - - @Parsed(field = "X0") - double x0; - - @Parsed(field = "C1") - double c1; - - @Parsed(field = "C0") - double c0; - - @Parsed(field = "Units") - String units; + private static LineCode mapLineCode(final NamedCsvRecord rec) { + return new LineCode( + rec.getField("Name"), + Integer.parseInt(rec.getField("nphases")), + Double.parseDouble(rec.getField("R1")), + Double.parseDouble(rec.getField("X1")), + Double.parseDouble(rec.getField("R0")), + Double.parseDouble(rec.getField("X0")), + Double.parseDouble(rec.getField("C1")), + Double.parseDouble(rec.getField("C0")), + rec.getField("Units") + ); } - public static class Load { - @Parsed(field = "Name") - String name; - - @Parsed - int numPhases; - - @Parsed(field = "Bus") - int bus; - - @Parsed - char phases; - - @Parsed - double kV; - - @Parsed(field = "Model") - int model; - - @Parsed(field = "Connection") - String connection; - - @Parsed - double kW; - - @Parsed(field = "PF") - double pf; - - @Parsed(field = "Yearly") - String yearly; + private static Load mapLoad(final NamedCsvRecord rec) { + return new Load( + rec.getField("Name"), + Integer.parseInt(rec.getField("numPhases")), + Integer.parseInt(rec.getField("Bus")), + rec.getField("phases").charAt(0), + Double.parseDouble(rec.getField("kV")), + Integer.parseInt(rec.getField("Model")), + rec.getField("Connection"), + Double.parseDouble(rec.getField("kW")), + Double.parseDouble(rec.getField("PF")), + rec.getField("Yearly") + ); } - public static class Transformer { - @Parsed(field = "Name") - String name; - - @Parsed - int phases; - - @Parsed - String bus1; - - @Parsed - int bus2; - - @Parsed(field = "kV_pri") - double kvPri; - - @Parsed(field = "kV_sec") - double kvSec; - - @Parsed(field = "MVA") - double mva; - - @Parsed(field = "Conn_pri") - String connPri; - - @Parsed(field = "Conn_sec") - String connSec; - - @Parsed(field = "%XHL") - double xhl; - - @Parsed(field = "% resistance") - double resistance; + private static Transformer mapTransformer(final NamedCsvRecord rec) { + return new Transformer( + rec.getField("Name"), + Integer.parseInt(rec.getField("phases")), + rec.getField("bus1"), + Integer.parseInt(rec.getField("bus2")), + Double.parseDouble(rec.getField("kV_pri")), + Double.parseDouble(rec.getField("kV_sec")), + Double.parseDouble(rec.getField("MVA")), + rec.getField("Conn_pri"), + rec.getField("Conn_sec"), + Double.parseDouble(rec.getField("%XHL")), + Double.parseDouble(rec.getField("% resistance")) + ); } + // Casting is safe here + @SuppressWarnings("unchecked") private static List parseCsv(String resourceName, Class clazz) { - try (Reader inputReader = new InputStreamReader(Objects.requireNonNull(EuropeanLvTestFeederFactory.class.getResourceAsStream(resourceName)), StandardCharsets.UTF_8)) { - BeanListProcessor rowProcessor = new BeanListProcessor<>(clazz); - CsvParserSettings settings = new CsvParserSettings(); - settings.setHeaderExtractionEnabled(true); - settings.setProcessor(rowProcessor); - CsvParser parser = new CsvParser(settings); - parser.parse(inputReader); - return rowProcessor.getBeans(); + try (Reader inputReader = new InputStreamReader(Objects.requireNonNull(EuropeanLvTestFeederFactory.class.getResourceAsStream(resourceName)), StandardCharsets.UTF_8); + CsvReader csvReader = CsvReader.builder() + .commentStrategy(CommentStrategy.SKIP) + .build(NamedCsvRecordHandler.of(c -> c.fieldModifier(FieldModifiers.TRIM)), inputReader)) { + Mapper mapper = (Mapper) MAPPERS.get(clazz); + if (mapper == null) { + throw new IllegalArgumentException("Unsupported class: " + clazz); + } + + return csvReader.stream().map(mapper::apply).toList(); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -402,4 +362,27 @@ public static Network create(NetworkFactory networkFactory) { createTransformer(transformer, network); return network; } + + private interface Mapper { + T apply(NamedCsvRecord rec); + } + + public record BusCoord(int busName, double x, double y) { + } + + public record Line(String name, int bus1, int bus2, String phases, double length, String units, String code) { + } + + public record LineCode(String name, int nphases, + double r1, double x1, double r0, double x0, double c1, double c0, String units) { + } + + public record Load(String name, int numPhases, int bus, char phases, double kV, int model, + String connection, double kW, double pf, String yearly) { + } + + public record Transformer(String name, int phases, String bus1, int bus2, + double kvPri, double kvSec, double mva, + String connPri, String connSec, double xhl, double resistance) { + } } From 3fd92e71d3aec0ac750a5fc6d3d5d2af25a0a32f Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Mon, 13 Oct 2025 16:27:40 +0200 Subject: [PATCH 5/6] Refactor `TimeSeries` to replace Univocity with FastCSV for CSV processing. Signed-off-by: Nicolas Rol --- .../com/powsybl/timeseries/TimeSeries.java | 111 ++++++++++-------- .../timeseries/TimeSeriesException.java | 4 + .../powsybl/timeseries/TimeSeriesTest.java | 2 +- 3 files changed, 65 insertions(+), 52 deletions(-) diff --git a/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeries.java b/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeries.java index b051a4928d7..3e7d3b9533a 100644 --- a/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeries.java +++ b/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeries.java @@ -12,13 +12,13 @@ import com.fasterxml.jackson.core.JsonToken; import com.google.common.base.Stopwatch; import com.google.common.primitives.Doubles; +import com.google.re2j.Pattern; import com.powsybl.commons.json.JsonUtil; import com.powsybl.commons.report.ReportNode; import com.powsybl.timeseries.ast.NodeCalc; -import com.univocity.parsers.common.ParsingContext; -import com.univocity.parsers.common.ResultIterator; -import com.univocity.parsers.csv.CsvParser; -import com.univocity.parsers.csv.CsvParserSettings; +import de.siegmar.fastcsv.reader.CsvParseException; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.CsvRecord; import gnu.trove.list.array.TDoubleArrayList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,6 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -57,6 +56,7 @@ public interface TimeSeries

> extends Iterable

{ Logger LOGGER = LoggerFactory.getLogger(TimeSeries.class); + Pattern WRONG_NUMBER_OF_FIELDS_IN_RECORD_PATTERN = Pattern.compile("Record (\\d+) has (\\d+) fields, but first record had (\\d+) fields"); int DEFAULT_VERSION_NUMBER_FOR_UNVERSIONED_TIMESERIES = -1; @@ -245,17 +245,17 @@ private List createStringValues() { return stringValues; } - int getVersion(String[] tokens, ReportNode reportNode) { + int getVersion(CsvRecord rec, ReportNode reportNode) { // Initialisation at the default unversioned value int version = DEFAULT_VERSION_NUMBER_FOR_UNVERSIONED_TIMESERIES; // Change the value if it is versioned if (timeSeriesCsvConfig.versioned()) { - version = Integer.parseInt(tokens[1]); + version = Integer.parseInt(rec.getField(1)); // If the version is equals to the default version, either log a warning or throw an exception if (version == DEFAULT_VERSION_NUMBER_FOR_UNVERSIONED_TIMESERIES) { - String line = String.join(";", tokens); + String line = String.join(";", rec.getFields().stream().map(str -> str.isEmpty() ? "null" : str).toList()); if (timeSeriesCsvConfig.withStrictVersioningImport()) { throw new TimeSeriesException(String.format("The version number for a versioned TimeSeries cannot be equals to the default version number (%s) at line \"%s\"", DEFAULT_VERSION_NUMBER_FOR_UNVERSIONED_TIMESERIES, @@ -304,24 +304,24 @@ void parseToken(int i, String token) { } } - void parseLineDuplicate(String[] tokens) { - Instant time = parseTokenTime(tokens[0]); - if (instants.isEmpty() || !instants.get(instants.size() - 1).equals(time)) { - parseTokenData(tokens); + void parseLineDuplicate(CsvRecord rec) { + Instant time = parseTokenTime(rec.getField(0)); + if (instants.isEmpty() || !instants.getLast().equals(time)) { + parseTokenData(rec); instants.add(time); } else { LOGGER.warn("Row with the same time have already been read, the row will be skipped"); } } - void parseLine(String[] tokens) { - parseTokenData(tokens); - instants.add(parseTokenTime(tokens[0])); + void parseLine(CsvRecord rec) { + parseTokenData(rec); + instants.add(parseTokenTime(rec.getField(0))); } - void parseTokenData(String[] tokens) { - for (int i = fixedColumns; i < tokens.length; i++) { - String token = tokens[i] != null ? tokens[i].trim() : ""; + void parseTokenData(CsvRecord rec) { + for (int i = fixedColumns; i < rec.getFieldCount(); i++) { + String token = rec.getField(i) != null ? rec.getField(i).trim() : ""; parseToken(i, token); } } @@ -364,7 +364,7 @@ List createTimeSeries() { List timeSeriesList = new ArrayList<>(names.size()); for (int i = 0; i < names.size(); i++) { - if (Objects.isNull(names.get(i))) { + if (Objects.isNull(names.get(i)) || names.get(i).isEmpty() || names.get(i).isBlank()) { LOGGER.warn("Timeseries without name"); continue; } @@ -414,43 +414,50 @@ private Duration checkRegularSpacing() { } } - static void readCsvValues(ResultIterator iterator, CsvParsingContext context, + static void readCsvValues(Iterator iterator, CsvParsingContext context, Map> timeSeriesPerVersion, ReportNode reportNode) { int currentVersion = Integer.MIN_VALUE; - Consumer lineparser = context.timeSeriesCsvConfig.isSkipDuplicateTimeEntry() + Consumer lineparser = context.timeSeriesCsvConfig.isSkipDuplicateTimeEntry() ? context::parseLineDuplicate : context::parseLine; - while (iterator.hasNext()) { - String[] tokens = iterator.next(); + try { + while (iterator.hasNext()) { + CsvRecord rec = iterator.next(); - if (tokens.length != context.expectedTokens()) { - throw new TimeSeriesException("Columns of line " + context.timesSize() + " are inconsistent with header"); - } + if (rec.getFieldCount() != context.expectedTokens()) { + throw new TimeSeriesException("Columns of line " + context.timesSize() + " are inconsistent with header"); + } - int version = context.getVersion(tokens, reportNode); - if (currentVersion == Integer.MIN_VALUE) { - currentVersion = version; - } else if (version != currentVersion) { - timeSeriesPerVersion.put(currentVersion, context.createTimeSeries()); - context.reInit(); - currentVersion = version; + int version = context.getVersion(rec, reportNode); + if (currentVersion == Integer.MIN_VALUE) { + currentVersion = version; + } else if (version != currentVersion) { + timeSeriesPerVersion.put(currentVersion, context.createTimeSeries()); + context.reInit(); + currentVersion = version; + } + lineparser.accept(rec); + } + } catch (CsvParseException e) { + if (WRONG_NUMBER_OF_FIELDS_IN_RECORD_PATTERN.matcher(e.getCause().getMessage()).find()) { + throw new TimeSeriesException("Columns of line " + context.timesSize() + " are inconsistent with header", e); } - lineparser.accept(tokens); + throw e; } timeSeriesPerVersion.put(currentVersion, context.createTimeSeries()); } - static CsvParsingContext readCsvHeader(ResultIterator iterator, TimeSeriesCsvConfig timeSeriesCsvConfig) { + static CsvParsingContext readCsvHeader(Iterator iterator, TimeSeriesCsvConfig timeSeriesCsvConfig) { if (!iterator.hasNext()) { throw new TimeSeriesException("CSV header is missing"); } - String[] tokens = iterator.next(); + CsvRecord headerRecord = iterator.next(); - checkCsvHeader(timeSeriesCsvConfig, tokens); + checkCsvHeader(timeSeriesCsvConfig, headerRecord); List duplicates = new ArrayList<>(); Set namesWithoutDuplicates = new HashSet<>(); - for (String token : tokens) { + for (String token : headerRecord.getFields()) { if (!namesWithoutDuplicates.add(token)) { duplicates.add(token); } @@ -458,15 +465,15 @@ static CsvParsingContext readCsvHeader(ResultIterator if (!duplicates.isEmpty()) { throw new TimeSeriesException("Bad CSV header, there are duplicates in time series names " + duplicates); } - List names = Arrays.asList(tokens).subList(timeSeriesCsvConfig.versioned() ? 2 : 1, tokens.length); + List names = headerRecord.getFields().subList(timeSeriesCsvConfig.versioned() ? 2 : 1, headerRecord.getFieldCount()); return new CsvParsingContext(names, timeSeriesCsvConfig); } - static void checkCsvHeader(TimeSeriesCsvConfig timeSeriesCsvConfig, String[] tokens) { + static void checkCsvHeader(TimeSeriesCsvConfig timeSeriesCsvConfig, CsvRecord headerRecord) { String separatorStr = Character.toString(timeSeriesCsvConfig.separator()); - if (timeSeriesCsvConfig.versioned() && (tokens.length < 3 || !"time".equalsIgnoreCase(tokens[0]) || !"version".equalsIgnoreCase(tokens[1]))) { + if (timeSeriesCsvConfig.versioned() && (headerRecord.getFieldCount() < 3 || !"time".equalsIgnoreCase(headerRecord.getField(0)) || !"version".equalsIgnoreCase(headerRecord.getField(1)))) { throw new TimeSeriesException("Bad CSV header, should be \ntime" + separatorStr + "version" + separatorStr + "..."); - } else if (tokens.length < 2 || !"time".equalsIgnoreCase(tokens[0])) { + } else if (headerRecord.getFieldCount() < 2 || !"time".equalsIgnoreCase(headerRecord.getField(0))) { throw new TimeSeriesException("Bad CSV header, should be \ntime" + separatorStr + "..."); } } @@ -483,16 +490,18 @@ static Map> parseCsv(BufferedReader reader, TimeSeries Map> timeSeriesPerVersion = new HashMap<>(); - CsvParserSettings settings = new CsvParserSettings(); - settings.getFormat().setDelimiter(timeSeriesCsvConfig.separator()); - settings.getFormat().setQuoteEscape('"'); - settings.getFormat().setLineSeparator(System.lineSeparator()); - settings.setMaxColumns(timeSeriesCsvConfig.getMaxColumns()); - CsvParser csvParser = new CsvParser(settings); - ResultIterator iterator = csvParser.iterate(reader).iterator(); - CsvParsingContext context = readCsvHeader(iterator, timeSeriesCsvConfig); - readCsvValues(iterator, context, timeSeriesPerVersion, reportNode); + try (CsvReader csvReader = CsvReader.builder() + .fieldSeparator(timeSeriesCsvConfig.separator()) + .quoteCharacter('"') + .ofCsvRecord(reader)) { + Iterator iterator = csvReader.iterator(); + CsvParsingContext context = readCsvHeader(iterator, timeSeriesCsvConfig); + readCsvValues(iterator, context, timeSeriesPerVersion, reportNode); + + } catch (IOException e) { + throw new UncheckedIOException(e); + } long timing = stopwatch.elapsed(TimeUnit.MILLISECONDS); LOGGER.info("{} time series loaded from CSV in {} ms", timeSeriesPerVersion.values().stream().mapToInt(List::size).sum(), diff --git a/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeriesException.java b/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeriesException.java index 41fb2940d6c..f017ddec847 100644 --- a/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeriesException.java +++ b/time-series/time-series-api/src/main/java/com/powsybl/timeseries/TimeSeriesException.java @@ -17,4 +17,8 @@ public class TimeSeriesException extends PowsyblException { public TimeSeriesException(String msg) { super(msg); } + + public TimeSeriesException(String msg, Throwable cause) { + super(msg, cause); + } } diff --git a/time-series/time-series-api/src/test/java/com/powsybl/timeseries/TimeSeriesTest.java b/time-series/time-series-api/src/test/java/com/powsybl/timeseries/TimeSeriesTest.java index 3d169217a95..cae27b5e8d0 100644 --- a/time-series/time-series-api/src/test/java/com/powsybl/timeseries/TimeSeriesTest.java +++ b/time-series/time-series-api/src/test/java/com/powsybl/timeseries/TimeSeriesTest.java @@ -152,7 +152,7 @@ void testTimeSeriesNameMissing() { Map> timeSeriesPerVersion = TimeSeries.parseCsv(csv); - // Since the name if the first timeseries is missing, only the second is saved + // Since the name of the first timeseries is missing, only the second is saved assertEquals(2, timeSeriesPerVersion.size()); assertEquals(1, timeSeriesPerVersion.get(1).size()); assertEquals(1, timeSeriesPerVersion.get(2).size()); From 1b1dacf04a1e72a850d32361d98eb67590e5855e Mon Sep 17 00:00:00 2001 From: Nicolas Rol Date: Mon, 13 Oct 2025 16:33:03 +0200 Subject: [PATCH 6/6] Refactor `AbstractCgmesAliasNamingStrategy` to replace Univocity with FastCSV for CSV writing Signed-off-by: Nicolas Rol --- .../AbstractCgmesAliasNamingStrategy.java | 67 ++++++++++--------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/naming/AbstractCgmesAliasNamingStrategy.java b/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/naming/AbstractCgmesAliasNamingStrategy.java index 46d5d346a37..f88a763f964 100644 --- a/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/naming/AbstractCgmesAliasNamingStrategy.java +++ b/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/naming/AbstractCgmesAliasNamingStrategy.java @@ -16,15 +16,22 @@ import com.powsybl.commons.datasource.DataSource; import com.powsybl.iidm.network.DanglingLine; import com.powsybl.iidm.network.Identifiable; -import com.univocity.parsers.csv.*; +import de.siegmar.fastcsv.writer.CsvWriter; +import de.siegmar.fastcsv.writer.LineDelimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.util.*; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; -import static com.powsybl.cgmes.conversion.naming.CgmesObjectReference.*; +import static com.powsybl.cgmes.conversion.naming.CgmesObjectReference.combine; import static com.powsybl.cgmes.conversion.naming.CgmesObjectReference.ref; +import static com.powsybl.cgmes.conversion.naming.CgmesObjectReference.refTyped; /** * @author Miora Vedelago {@literal } @@ -117,37 +124,33 @@ public void debug(String baseName, DataSource ds) { return; } String mappingFilename = baseName + "_debug_naming_strategy.csv"; - try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(ds.newOutputStream(mappingFilename, false)))) { - CsvWriterSettings settings = new CsvWriterSettings(); - settings.getFormat().setLineSeparator(System.lineSeparator()); - settings.getFormat().setDelimiter(';'); - settings.getFormat().setQuoteEscape('"'); - CsvWriter csvWriter = new CsvWriter(writer, settings); - try { - String[] nextLine = new String[3]; - nextLine[0] = "CgmesUuid"; - nextLine[1] = "IidmId"; - nextLine[2] = "Seed"; - csvWriter.writeRow(nextLine); - - for (Map.Entry e : idByUuid.entrySet()) { - String uuid = e.getKey(); + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(ds.newOutputStream(mappingFilename, false))); + CsvWriter csvWriter = CsvWriter.builder() + .fieldSeparator(';') + .lineDelimiter(LineDelimiter.PLATFORM) + .quoteCharacter('"') + .build(writer)) { + String[] nextLine = new String[3]; + nextLine[0] = "CgmesUuid"; + nextLine[1] = "IidmId"; + nextLine[2] = "Seed"; + csvWriter.writeRecord(nextLine); + + for (Map.Entry e : idByUuid.entrySet()) { + String uuid = e.getKey(); + nextLine[0] = uuid; + nextLine[1] = e.getValue(); + nextLine[2] = uuidSeed.get(uuid); + csvWriter.writeRecord(nextLine); + } + for (Map.Entry e : uuidSeed.entrySet()) { + String uuid = e.getKey(); + if (!idByUuid.containsKey(uuid)) { nextLine[0] = uuid; - nextLine[1] = e.getValue(); + nextLine[1] = "unknown"; nextLine[2] = uuidSeed.get(uuid); - csvWriter.writeRow(nextLine); - } - for (Map.Entry e : uuidSeed.entrySet()) { - String uuid = e.getKey(); - if (!idByUuid.containsKey(uuid)) { - nextLine[0] = uuid; - nextLine[1] = "unknown"; - nextLine[2] = uuidSeed.get(uuid); - csvWriter.writeRow(nextLine); - } + csvWriter.writeRecord(nextLine); } - } finally { - csvWriter.close(); } } catch (IOException e) { throw new UncheckedIOException(e);