diff --git a/.gitignore b/.gitignore index a2917e1..45464e7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ generated*.md # Test data data +*sqlite # System-specific files and directories generated by the BinaryProvider and BinDeps packages # They contain absolute paths specific to the host computer, and so should not be committed diff --git a/Project.toml b/Project.toml index 8999b29..4803693 100644 --- a/Project.toml +++ b/Project.toml @@ -1,11 +1,28 @@ -name = "SiennaTemplate" +name = "PowerFlowFileParser" uuid = "bed98974-b02a-5e2f-9ee0-a103f5c450dd" -authors = ["YOUR_NAME"] version = "0.1.0" +authors = ["Daniel Thom, José Daniel Lara, Hannah Chubin"] [deps] +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +PowerFlowData = "dd99e9e3-7471-40fc-b48d-a10501125371" +PowerSystems = "bcd98974-b02a-5e2f-9ee0-a103f5c450dd" +SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" +SiennaOpenAPIModels = "6f904ddb-d94b-53ea-97a4-b251110faebf" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" [compat] +DataStructures = "0.19.3" DocStringExtensions = "~0.8, ~0.9" +InfrastructureSystems = "3.2.0" +LinearAlgebra = "1.12.0" +PowerFlowData = "1.6.0" +PowerSystems = "5.4.0" +SQLite = "1.8.0" +Unicode = "1.11.0" +YAML = "0.4.16" julia = "^1.6" diff --git a/src/PowerFlowFileParser.jl b/src/PowerFlowFileParser.jl new file mode 100644 index 0000000..43ec48f --- /dev/null +++ b/src/PowerFlowFileParser.jl @@ -0,0 +1,127 @@ +isdefined(Base, :__precompile__) && __precompile__() + +module PowerFlowFileParser + +################################################################################# +# Exports + +export PowerModelsData +export PowerFlowDataNetwork +export System # this function is tested as PowerFlowFileParser.System to disambiguate from PowerSystems.System +export parse_file +export make_database + +################################################################################# +# Imports + +import PowerFlowData +import LinearAlgebra # in PSY only used in src/pm_io/data.jl +import DataStructures: SortedDict +# import CSV +# import DataFrames +# import JSON3 +import SiennaOpenAPIModels +import SQLite +import Unicode: normalize +import YAML + +import InfrastructureSystems +const IS = InfrastructureSystems + +import PowerSystems +const PSY = PowerSystems + +# should I import entire model library? end user might build a system with any +# object in model library, but at the same time we only want to support the +# current objects we build in this repo + +# importing PSY.System. Previously, when just exporting System as defined in +# this repo I got an error saying there was no method for +# System(PowerFlowDataNetwork). Whenever System is tested, its the method from +# this repo. But that System is defined using methods of System from PSY as +# well. + +import PowerSystems: + ACBus, + ACBusTypes, + TwoWindingTransformer, + ThreeWindingTransformer, + ImpedanceCorrectionData, + Area, + WindingCategory, + WindingGroupNumber, + System, + get_component, + add_component!, + set_ext!, + has_component, + StandardLoad, + get_name, + LoadZone, + set_load_zone!, + PowerLoad, + ThermalStandard, + GeneratorCostModels, + QuadraticFunctionData, + CostCurve, + InputOutputCurve, + UnitSystem, + ThermalGenerationCost, + MinMax, + ThermalFuels, + PrimeMovers, + Line, + DiscreteControlledACBranch, + Transformer2W, + TapTransformer, + PhaseShiftingTransformer, + get_bustype, + Arc, + FixedAdmittance, + check, + HydroDispatch, + HydroTurbine, + RenewableDispatch, + RenewableGenerationCost, + get_slopes, + PiecewiseLinearData, + RenewableNonDispatch, + SynchronousCondenser, + HydroGenerationCost, + EnergyReservoirStorage, + LinearCurve, + TwoTerminalGenericHVDCLine, + StorageTech, + ImpedanceCorrectionTransformerControlMode, + add_supplemental_attribute!, + SwitchedAdmittance, + TwoTerminalLCCLine, + FACTSControlDevice, + Transformer3W, + PhaseShiftingTransformer3W + +import InfrastructureSystems: + DataFormatError + +################################################################################# +# Includes + +include("powerflowdata_data.jl") +include("power_models_data.jl") +include("common.jl") +include("definitions.jl") +include("im_io.jl") +include("pm_io.jl") + +################################################################################# + +using DocStringExtensions + +@template (FUNCTIONS, METHODS) = """ + $(TYPEDSIGNATURES) + $(DOCSTRING) + """ + +################################################################################# + +end diff --git a/src/SiennaTemplate.jl b/src/SiennaTemplate.jl deleted file mode 100644 index bfff18b..0000000 --- a/src/SiennaTemplate.jl +++ /dev/null @@ -1,9 +0,0 @@ -module SiennaTemplate -using DocStringExtensions - -@template (FUNCTIONS, METHODS) = """ - $(TYPEDSIGNATURES) - $(DOCSTRING) - """ - -end diff --git a/src/common.jl b/src/common.jl new file mode 100644 index 0000000..d186570 --- /dev/null +++ b/src/common.jl @@ -0,0 +1,196 @@ +# this function would be exactly the same for both System(PowerModelsData) and +# System(PowerFlowNetworkData) so it feels redundant to put in both power*.jl +# files. Instead I'm putting it here, but this also doesnt feel like the right +# place for it + +""" +Function that creates a database from System. + +""" +function make_database(sys::System, database_name::Union{String, Nothing}) + + # making sure that database_name isn't an existing file + if isfile(database_name) || isfile(string(database_name, ".sqlite")) + error("database with this name already exists") + # creating database file name with .sqlite extension + elseif !isfile(database_name) + if !endswith(database_name, ".sqlite") + database_name = string(database_name, ".sqlite") + elseif endswith(database_name, ".sqlite") + database_name = database_name + end + end + + # making database, with time series, if given + db = SQLite.DB(database_name) + SiennaOpenAPIModels.make_sqlite!(db) + ids = SiennaOpenAPIModels.IDGenerator() + SiennaOpenAPIModels.sys2db!(db, sys, ids) + #TODO (this repo and PowerTableDataParser) this check should already be + #built in to serialize_timeseries!() + if IS.get_num_time_series(sys.data) !== 0 + SiennaOpenAPIModels.serialize_timeseries!(db, sys, ids) + end +end + +const GENERATOR_MAPPING_FILE_PM = + joinpath(dirname(pathof(PowerSystems)), "parsers", "generator_mapping_pm.yaml") + +const SKIP_PM_VALIDATION = false + +const PSSE_PARSER_TAP_RATIO_UBOUND = 1.5 +const PSSE_PARSER_TAP_RATIO_LBOUND = 0.5 +const INFINITE_BOUND = 1e6 + +const STRING2FUEL = + Dict((normalize(string(x); casefold = true) => x) for x in instances(ThermalFuels)) +merge!( + STRING2FUEL, + Dict( + "ng" => ThermalFuels.NATURAL_GAS, + "nuc" => ThermalFuels.NUCLEAR, + "gas" => ThermalFuels.NATURAL_GAS, + "oil" => ThermalFuels.DISTILLATE_FUEL_OIL, + "dfo" => ThermalFuels.DISTILLATE_FUEL_OIL, + "sync_cond" => ThermalFuels.OTHER, + "geothermal" => ThermalFuels.GEOTHERMAL, + "ag_byproduct" => ThermalFuels.AG_BYPRODUCT, + ), +) + +const STRING2PRIMEMOVER = + Dict((normalize(string(x); casefold = true) => x) for x in instances(PrimeMovers)) +merge!( + STRING2PRIMEMOVER, + Dict( + "w2" => PrimeMovers.WT, + "wind" => PrimeMovers.WT, + "pv" => PrimeMovers.PVe, + "solar" => PrimeMovers.PVe, + "rtpv" => PrimeMovers.PVe, + "nb" => PrimeMovers.ST, + "steam" => PrimeMovers.ST, + "hydro" => PrimeMovers.HY, + "ror" => PrimeMovers.HY, + "pump" => PrimeMovers.PS, + "pumped_hydro" => PrimeMovers.PS, + "nuclear" => PrimeMovers.ST, + "sync_cond" => PrimeMovers.OT, + "csp" => PrimeMovers.CP, + "un" => PrimeMovers.OT, + "storage" => PrimeMovers.BA, + "ice" => PrimeMovers.IC, + ), +) + +"""Return a dict where keys are a tuple of input parameters (fuel, unit_type) and values are +generator types.""" +function get_generator_mapping(filename::String) + genmap = open(filename) do file + YAML.load(file) + end + + mappings = Dict{NamedTuple, DataType}() + for (gen_type, vals) in genmap + if gen_type == "GenericBattery" + @warn "GenericBattery type is no longer supported. The new type is EnergyReservoirStorage" + gen = EnergyReservoirStorage + else + gen = getfield(PowerSystems, Symbol(gen_type)) + end + for val in vals + key = (fuel = val["fuel"], unit_type = val["type"]) + if haskey(mappings, key) + error("duplicate generator mappings: $gen $(key.fuel) $(key.unit_type)") + end + mappings[key] = gen + end + end + + return mappings +end + +"""Return the PowerSystems generator type for this fuel and unit_type.""" +function get_generator_type(fuel, unit_type, mappings::Dict{NamedTuple, DataType}) + fuel = isnothing(fuel) ? "" : uppercase(fuel) + unit_type = uppercase(unit_type) + generator = nothing + + # Try to match the unit_type if it's defined. If it's nothing then just match on fuel. + for ut in (unit_type, nothing), fu in (fuel, nothing) + key = (fuel = fu, unit_type = ut) + if haskey(mappings, key) + generator = mappings[key] + break + end + end + + if isnothing(generator) + @error "No mapping for generator fuel=$fuel unit_type=$unit_type" + end + + return generator +end + +function calculate_gen_rating( + active_power_limits::Union{MinMax, Nothing}, + reactive_power_limits::Union{MinMax, Nothing}, + base_conversion::Float64, +) + reactive_power_max = isnothing(reactive_power_limits) ? 0.0 : reactive_power_limits.max + return calculate_gen_rating( + active_power_limits.max, + reactive_power_max, + base_conversion, + ) +end + +function calculate_gen_rating( + active_power_max::Float64, + reactive_power_max::Float64, + base_conversion::Float64, +) + rating = sqrt(active_power_max^2 + reactive_power_max^2) + if rating == 0.0 + @warn "Rating calculation returned 0.0. Changing to 1.0 in the p.u. of the device." + return 1.0 + end + return rating * base_conversion +end + +function calculate_ramp_limit( + d::Dict{String, Any}, + gen_name::Union{SubString{String}, String}, +) + if haskey(d, "ramp_agc") + return (up = d["ramp_agc"], down = d["ramp_agc"]) + end + if haskey(d, "ramp_10") + return (up = d["ramp_10"], down = d["ramp_10"]) + end + if haskey(d, "ramp_30") + return (up = d["ramp_30"], down = d["ramp_30"]) + end + if abs(d["pmax"]) > 0.0 + @debug "No ramp limits found for generator $(gen_name). Using pmax as ramp limit." + return (up = abs(d["pmax"]), down = abs(d["pmax"])) + end + @warn "Not enough information to determine ramp limit for generator $(gen_name). Returning nothing" + return nothing +end + +function parse_enum_mapping(::Type{ThermalFuels}, fuel::AbstractString) + return STRING2FUEL[normalize(fuel; casefold = true)] +end + +function parse_enum_mapping(::Type{ThermalFuels}, fuel::Symbol) + return parse_enum_mapping(ThermalFuels, string(fuel)) +end + +function parse_enum_mapping(::Type{PrimeMovers}, prime_mover::AbstractString) + return STRING2PRIMEMOVER[normalize(prime_mover; casefold = true)] +end + +function parse_enum_mapping(::Type{PrimeMovers}, prime_mover::Symbol) + return parse_enum_mapping(PrimeMovers, string(prime_mover)) +end diff --git a/src/definitions.jl b/src/definitions.jl new file mode 100644 index 0000000..7aca6a4 --- /dev/null +++ b/src/definitions.jl @@ -0,0 +1,16 @@ +# copied from PowerSystems/src/definitions.jl + +const PS_MAX_LOG = parse(Int, get(ENV, "PS_MAX_LOG", "50")) + +const BRANCH_BUS_VOLTAGE_DIFFERENCE_TOL = 0.01 + +const WINDING_NAMES = Dict( + WindingCategory.PRIMARY_WINDING => "primary", + WindingCategory.SECONDARY_WINDING => "secondary", + WindingCategory.TERTIARY_WINDING => "tertiary", +) + +const TRANSFORMER3W_PARAMETER_NAMES = [ + "COD", "CONT", "NOMV", "WINDV", "RMA", "RMI", + "NTP", "VMA", "VMI", "RATA", "RATB", "RATC", +] diff --git a/src/im_io.jl b/src/im_io.jl new file mode 100644 index 0000000..69681e2 --- /dev/null +++ b/src/im_io.jl @@ -0,0 +1,3 @@ +include("im_io/matlab.jl") +include("im_io/common.jl") +include("im_io/data.jl") diff --git a/src/im_io/LICENSE.md b/src/im_io/LICENSE.md new file mode 100644 index 0000000..a371758 --- /dev/null +++ b/src/im_io/LICENSE.md @@ -0,0 +1,9 @@ +Copyright (c) 2016, Los Alamos National Security, LLC +All rights reserved. +Copyright 2016. Los Alamos National Security, LLC. This software was produced under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National Laboratory (LANL), which is operated by Los Alamos National Security, LLC for the U.S. Department of Energy. The U.S. Government has rights to use, reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is modified to produce derivative works, such modified software should be clearly marked, so as not to confuse it with the version available from LANL. + +Additionally, redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of Los Alamos National Security, LLC, Los Alamos National Laboratory, LANL, the U.S. Government, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/im_io/common.jl b/src/im_io/common.jl new file mode 100644 index 0000000..ba3d9f2 --- /dev/null +++ b/src/im_io/common.jl @@ -0,0 +1,54 @@ + +"turns top level arrays into dicts" +function arrays_to_dicts!(data::Dict{String, <:Any}) + # update lookup structure + for (k, v) in data + if isa(v, Array) && length(v) > 0 && isa(v[1], Dict) + #println("updating $(k)") + dict = Dict{Int, Any}() + for (i, item) in enumerate(v) + if haskey(item, "index") + key = item["index"] + else + key = i + end + + if !(haskey(dict, key)) + dict[key] = item + else + @warn "skipping component $(item["index"]) from the $(k) table because a component with the same id already exists" + end + end + data[k] = dict + end + end +end + +"takes a row from a matrix and assigns the values names and types" +function row_to_typed_dict(row_data, columns) + dict_data = Dict{String, Any}() + for (i, v) in enumerate(row_data) + if i <= length(columns) + name, typ = columns[i] + dict_data[name] = check_type(typ, v) + else + dict_data["col_$(i)"] = v + end + end + return dict_data +end + +"takes a row from a matrix and assigns the values names" +function row_to_dict(row_data, columns) + dict_data = Dict{String, Any}() + for (i, v) in enumerate(row_data) + if i <= length(columns) + dict_data[columns[i]] = v + else + dict_data["col_$(i)"] = v + end + end + return dict_data +end + +row_to_dict(row_data) = row_to_dict(row_data, []) diff --git a/src/im_io/data.jl b/src/im_io/data.jl new file mode 100644 index 0000000..a212424 --- /dev/null +++ b/src/im_io/data.jl @@ -0,0 +1,178 @@ +"recursively applies new_data to data, overwriting information" +function update_data!(data::Dict{String, <:Any}, new_data::Dict{String, <:Any}) + if haskey(data, "per_unit") && haskey(new_data, "per_unit") + if data["per_unit"] != new_data["per_unit"] + error( + "update_data requires datasets in the same units, try make_per_unit and make_mixed_units", + ) + end + else + @warn "running update_data with data that does not include per_unit field, units may be incorrect" + end + _update_data!(data, new_data) + return +end + +"recursive call of _update_data" +function _update_data!(data::Dict{String, <:Any}, new_data::Dict{String, <:Any}) + for (key, new_v) in new_data + if haskey(data, key) + v = data[key] + if isa(v, Dict) && isa(new_v, Dict) + _update_data!(v, new_v) + else + data[key] = new_v + end + else + data[key] = new_v + end + end + return +end + +"checks if a given network data is a multinetwork" +ismultinetwork(data::Dict{String, <:Any}) = + (haskey(data, "multinetwork") && data["multinetwork"] == true) + +"Transforms a single network into a multinetwork with several deepcopies of the original network" +function im_replicate(sn_data::Dict{String, <:Any}, count::Int, global_keys::Set{String}) + @assert count > 0 + if ismultinetwork(sn_data) + error("replicate can only be used on single networks") + end + + name = get(sn_data, "name", "anonymous") + + mn_data = Dict{String, Any}("nw" => Dict{String, Any}()) + + mn_data["multinetwork"] = true + + sn_data_tmp = deepcopy(sn_data) + for k in global_keys + if haskey(sn_data_tmp, k) + mn_data[k] = sn_data_tmp[k] + end + + # note this is robust to cases where k is not present in sn_data_tmp + delete!(sn_data_tmp, k) + end + + mn_data["name"] = "$(count) replicates of $(name)" + + for n in 1:count + mn_data["nw"]["$n"] = deepcopy(sn_data_tmp) + end + + return mn_data +end + +#= +"Attempts to determine if the given data is a component dictionary" +function _iscomponentdict(data::Dict) + return all( typeof(comp) <: Dict for (i, comp) in data ) +end +=# + +"Makes a string bold in the terminal" +function _bold(s::String) + return "\033[1m$(s)\033[0m" +end + +""" +Makes a string grey in the terminal, does not seem to work well on Windows terminals +more info can be found at https://en.wikipedia.org/wiki/ANSI_escape_code +""" +function _grey(s::String) + return "\033[38;5;239m$(s)\033[0m" +end + +"converts any value to a string, summarizes arrays and dicts" +function _value2string(v, float_precision::Int) + if typeof(v) <: AbstractFloat + return _float2string(v, float_precision) + end + if typeof(v) <: Array + return "[($(length(v)))]" + end + if typeof(v) <: Dict + return "{($(length(v)))}" + end + + return "$(v)" +end + +""" +converts a float value into a string of fixed precision + +sprintf would do the job but this work around is needed because +sprintf cannot take format strings during runtime +""" +function _float2string(v::AbstractFloat, float_precision::Int) + #str = "$(round(v; digits=float_precision))" + str = "$(round(v; digits=float_precision))" + lhs = length(split(str, '.')[1]) + return rpad(str, lhs + 1 + float_precision, "0") +end + +"tests if two dicts are equal, up to floating point precision" +function compare_dict(d1, d2) + for (k1, v1) in d1 + if !haskey(d2, k1) + #println(k1) + return false + end + v2 = d2[k1] + + if isa(v1, Number) + if !_compare_numbers(v1, v2) + return false + end + elseif isa(v1, Array) + if length(v1) != length(v2) + return false + end + for i in 1:length(v1) + if isa(v1[i], Number) + if !_compare_numbers(v1[i], v2[i]) + return false + end + else + if v1 != v2 + #println(v1, " ", v2) + return false + end + end + end + elseif isa(v1, Dict) + if !compare_dict(v1, v2) + #println(v1, " ", v2) + return false + end + else + #println("2") + if !isapprox(v1, v2) + #println(v1, " ", v2) + return false + end + end + end + return true +end + +"tests if two numbers are equal, up to floating point precision" +function _compare_numbers(v1, v2) + if isnan(v1) + #println("1.1") + if !isnan(v2) + #println(v1, " ", v2) + return false + end + else + #println("1.2") + if !isapprox(v1, v2) + #println(v1, " ", v2) + return false + end + end + return true +end diff --git a/src/im_io/matlab.jl b/src/im_io/matlab.jl new file mode 100644 index 0000000..2812725 --- /dev/null +++ b/src/im_io/matlab.jl @@ -0,0 +1,338 @@ +######################################################################### +# # +# This file provides functions for interfacing with Matlab .m files # +# # +######################################################################### + +# this could benefit from a much more robust parser + +export parse_matlab_file, parse_matlab_string + +function parse_matlab_file(file_string::String; kwargs...) + data_string = read(open(file_string), String) + return parse_matlab_string(data_string; kwargs...) +end + +function parse_matlab_string(data_string::String; extended = false) + data_lines = split(data_string, '\n') + + matlab_dict = Dict{String, Any}() + struct_name = nothing + function_name = nothing + column_names = Dict{String, Any}() + + last_index = length(data_lines) + index = 1 + while index <= last_index + line = strip(data_lines[index]) + line = "$(line)" + + if length(line) <= 0 || strip(line)[1] == '%' + index = index + 1 + continue + end + + if occursin("function", line) + func, value = _extract_matlab_assignment(line) + struct_name = strip(replace(func, "function" => "")) + function_name = value + elseif occursin("=", line) + if struct_name !== nothing && !occursin("$(struct_name).", line) + @warn "assignments are expected to be made to \"$(struct_name)\" but given: $(line)" + end + + if occursin("[", line) + matrix_dict = _parse_matlab_matrix(data_lines, index) + matlab_dict[matrix_dict["name"]] = matrix_dict["data"] + if haskey(matrix_dict, "column_names") + column_names[matrix_dict["name"]] = matrix_dict["column_names"] + end + index = index + matrix_dict["line_count"] - 1 + elseif occursin("{", line) + cell_dict = _parse_matlab_cells(data_lines, index) + matlab_dict[cell_dict["name"]] = cell_dict["data"] + if haskey(cell_dict, "column_names") + column_names[cell_dict["name"]] = cell_dict["column_names"] + end + index = index + cell_dict["line_count"] - 1 + else + name, value = _extract_matlab_assignment(line) + value = _type_value(value) + matlab_dict[name] = value + end + else + @error "Matlab parser skipping line number $(index) consisting of:\n $(line)" + end + + index += 1 + end + + if extended + return matlab_dict, function_name, column_names + else + return matlab_dict + end +end + +"breaks up matlab strings of the form 'name = value;'" +function _extract_matlab_assignment(string::AbstractString) + statement = split(string, ';')[1] + statement_parts = split(statement, '=') + @assert(length(statement_parts) == 2) + name = strip(statement_parts[1]) + value = strip(statement_parts[2]) + return name, value +end + +"Attempts to determine the type of a string extracted from a matlab file" +function _type_value(value_string::AbstractString) + value_string = strip(value_string) + + if occursin("'", value_string) # value is a string + value = strip(value_string, '\'') + else + # if value is a float + if occursin(".", value_string) || occursin("e", value_string) + value = check_type(Float64, value_string) + else # otherwise assume it is an int + value = check_type(Int, value_string) + end + end + + return value +end + +"Attempts to determine the type of an array of strings extracted from a matlab file" +function _type_array(string_array::Vector{T}) where {T <: AbstractString} + value_string = [strip(value_string) for value_string in string_array] + + return if any(occursin("'", value_string) for value_string in string_array) + [strip(value_string, '\'') for value_string in string_array] + elseif any( + occursin(".", value_string) || + occursin("e", value_string) || + occursin("Inf", value_string) || + occursin("NaN", value_string) for value_string in string_array + ) + [check_type(Float64, value_string) for value_string in string_array] + else # otherwise assume it is an int + [check_type(Int, value_string) for value_string in string_array] + end +end + +"" +_parse_matlab_cells(lines, index) = _parse_matlab_data(lines, index, '{', '}') + +"" +_parse_matlab_matrix(lines, index) = _parse_matlab_data(lines, index, '[', ']') + +"" +function _parse_matlab_data(lines, index, start_char, end_char) + last_index = length(lines) + line_count = 0 + columns = -1 + + @assert(occursin("=", lines[index + line_count])) + matrix_assignment = split(lines[index + line_count], '%')[1] + matrix_assignment = strip(matrix_assignment) + + @assert(occursin(".", matrix_assignment)) + matrix_assignment_parts = split(matrix_assignment, '=') + matrix_name = strip(matrix_assignment_parts[1]) + + matrix_assignment_rhs = "" + if length(matrix_assignment_parts) > 1 + matrix_assignment_rhs = strip(matrix_assignment_parts[2]) + end + + line_count = line_count + 1 + matrix_body_lines = [matrix_assignment_rhs] + found_close_bracket = occursin(string(end_char), matrix_assignment_rhs) + + while index + line_count < last_index && !found_close_bracket + line = strip(lines[index + line_count]) + + if length(line) == 0 || line[1] == '%' + line_count += 1 + continue + end + + line = strip(split(line, '%')[1]) + + if occursin(string(end_char), line) + found_close_bracket = true + end + + push!(matrix_body_lines, line) + + line_count = line_count + 1 + end + + #print(matrix_body_lines) + matrix_body_lines = + [_add_line_delimiter(line, start_char, end_char) for line in matrix_body_lines] + #print(matrix_body_lines) + + matrix_body = join(matrix_body_lines, ' ') + matrix_body = + strip(replace(strip(strip(matrix_body), start_char), "$(end_char);" => "")) + matrix_body_rows = split(matrix_body, ';') + matrix_body_rows = matrix_body_rows[1:(length(matrix_body_rows) - 1)] + + matrix = [] + for row in matrix_body_rows + row_items = split_line(strip(row)) + #println(row_items) + push!(matrix, row_items) + if columns < 0 + columns = length(row_items) + elseif columns != length(row_items) + @error "matrix parsing error, inconsistent number of items in each row\n$(row)" + end + end + + rows = length(matrix) + typed_columns = [_type_array([matrix[r][c] for r in 1:rows]) for c in 1:columns] + for r in 1:rows + matrix[r] = [typed_columns[c][r] for c in 1:columns] + end + + matrix_dict = Dict("name" => matrix_name, "data" => matrix, "line_count" => line_count) + + if index > 1 && occursin("%column_names%", lines[index - 1]) + column_names_string = lines[index - 1] + column_names_string = replace(column_names_string, "%column_names%" => "") + column_names = split(column_names_string) + if length(matrix[1]) != length(column_names) + @error "column name parsing error, data rows $(length(matrix[1])), column names $(length(column_names)) \n$(column_names)" + end + for (c, column_name) in enumerate(column_names) + if column_name == "index" + @error "column name parsing error, \"index\" is a reserved column name \n$(column_names)" + + if !(typeof(typed_columns[c][1]) <: Int) + @error "the type of a column named \"index\" must be Int, but given $(typeof(typed_columns[c][1]))" + end + end + end + matrix_dict["column_names"] = column_names + end + + return matrix_dict +end + +"" +function split_line(mp_line::AbstractString) + tokens = [] + curr_token = "" + is_curr_token_quote = false + + isquote(c) = (c == '\'' || c == '"') + + function _push_curr_token() + if curr_pos <= length(mp_line) + curr_token *= mp_line[curr_pos] + end + curr_token = strip(curr_token) + if length(curr_token) > 0 + push!(tokens, curr_token) + end + curr_token = "" + curr_pos += 1 + is_curr_token_quote = false + end + + function _push_curr_char() + curr_token *= mp_line[curr_pos] + curr_pos += 1 + end + + curr_pos = 1 + while curr_pos <= length(mp_line) + if is_curr_token_quote + if mp_line[curr_pos] == curr_token[1] + if mp_line[curr_pos - 1] == '\\' + # If we are inside a quote and we see slash-quote, we should + # treat the quote character as a regular character. + _push_curr_char() + elseif curr_pos < length(mp_line) && mp_line[curr_pos + 1] == curr_token[1] + # If we are inside a quote, and we see two quotes in a row, + # we should append one of the quotes to the current + # token, then skip the other one. + curr_token *= mp_line[curr_pos] + curr_pos += 2 + else + # If we are inside a quote, and we see an unescaped quote char, + # then the quote is ending. We should push the current token. + _push_curr_token() + end + else + # If we are inside a quote and we see a non-quote character, + # we should append that character to the current token. + _push_curr_char() + end + else + if isspace(mp_line[curr_pos]) && !isspace(mp_line[curr_pos + 1]) + # If we are not inside a quote and we see a transition from + # space to non-space character, then the current token is done. + _push_curr_token() + elseif isquote(mp_line[curr_pos]) + # If we are not inside a quote and we see a quote character, + # then a new quote is starting. We should append the quote + # character to the current token and switch to quote mode. + curr_token = strip(curr_token * mp_line[curr_pos]) + is_curr_token_quote = true + curr_pos += 1 + else + # If we are not inside a quote and we see a regular character, + # we should append that character to the current token. + _push_curr_char() + end + end + end + _push_curr_token() + return tokens +end + +"" +function _add_line_delimiter(mp_line::AbstractString, start_char, end_char) + if strip(mp_line) == string(start_char) + return mp_line + end + + if !occursin(";", mp_line) && !occursin(string(end_char), mp_line) + mp_line = "$(mp_line);" + end + + if occursin(string(end_char), mp_line) + prefix = strip(split(mp_line, end_char)[1]) + if length(prefix) > 0 && !occursin(";", prefix) + mp_line = replace(mp_line, end_char => ";$(end_char)") + end + end + + return mp_line +end + +"Checks if the given value is of a given type, if not tries to make it that type" +function check_type(typ, value) + if isa(value, typ) + return value + elseif isa(value, String) || isa(value, SubString) + try + value = parse(typ, value) + return value + catch e + @error "parsing error, the matlab string \"$(value)\" can not be parsed to $(typ) data" + rethrow(e) + end + else + try + value = typ(value) + return value + catch e + @error "parsing error, the matlab value $(value) of type $(typeof(value)) can not be parsed to $(typ) data" + rethrow(e) + end + end +end diff --git a/src/pm_io.jl b/src/pm_io.jl new file mode 100644 index 0000000..e7c217f --- /dev/null +++ b/src/pm_io.jl @@ -0,0 +1,5 @@ +include("pm_io/matpower.jl") +include("pm_io/common.jl") +include("pm_io/pti.jl") +include("pm_io/psse.jl") +include("pm_io/data.jl") diff --git a/src/pm_io/LICENSE.md b/src/pm_io/LICENSE.md new file mode 100644 index 0000000..a371758 --- /dev/null +++ b/src/pm_io/LICENSE.md @@ -0,0 +1,9 @@ +Copyright (c) 2016, Los Alamos National Security, LLC +All rights reserved. +Copyright 2016. Los Alamos National Security, LLC. This software was produced under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National Laboratory (LANL), which is operated by Los Alamos National Security, LLC for the U.S. Department of Energy. The U.S. Government has rights to use, reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is modified to produce derivative works, such modified software should be clearly marked, so as not to confuse it with the version available from LANL. + +Additionally, redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of Los Alamos National Security, LLC, Los Alamos National Laboratory, LANL, the U.S. Government, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/pm_io/common.jl b/src/pm_io/common.jl new file mode 100644 index 0000000..6b57c0d --- /dev/null +++ b/src/pm_io/common.jl @@ -0,0 +1,129 @@ +""" + parse_file( + file; + import_all = false, + validate = true, + correct_branch_rating = true, + ) + +Parses a Matpower .m `file` or PTI (PSS(R)E-v33) .raw `file` into a +PowerModels data structure. All fields from PTI files will be imported if +`import_all` is true (Default: false). +""" +function parse_file( + file::String; + import_all = false, + validate = true, + correct_branch_rating = true, +) + pm_data = open(file) do io + pm_data = parse_file( + io; + import_all = import_all, + validate = validate, + correct_branch_rating = correct_branch_rating, + filetype = split(lowercase(file), '.')[end], + ) + end + return pm_data +end + +"Parses the iostream from a file" +function parse_file( + io::IO; + import_all = false, + validate = true, + correct_branch_rating = true, + filetype = "json", +) + if filetype == "m" + pm_data = parse_matpower(io; validate = validate) + elseif filetype == "raw" + pm_data = parse_psse( + io; + import_all = import_all, + validate = validate, + correct_branch_rating = correct_branch_rating, + ) + elseif filetype == "json" + pm_data = parse_json(io; validate = validate) + else + @info("Unrecognized filetype") + end + + # TODO: not sure if this relevant for all three file types, or only .m, JJS 3/7/19 + move_genfuel_and_gentype!(pm_data) + + return pm_data +end + +""" +Runs various data quality checks on a PowerModels data dictionary. +Applies modifications in some cases. Reports modified component ids. +""" +function correct_network_data!(data::Dict{String, <:Any}; correct_branch_rating = true) + mod_gen = Dict{Symbol, Set{Int}}() + mod_branch = Dict{Symbol, Set{Int}}() + mod_dcline = Dict{Symbol, Set{Int}}() + + check_conductors(data) + check_connectivity(data) + check_status(data) + check_reference_bus(data) + + make_per_unit!(data) + + mod_branch[:xfer_fix] = correct_transformer_parameters!(data) + mod_branch[:vad_bounds] = correct_voltage_angle_differences!(data) + mod_branch[:mva_zero] = if (correct_branch_rating) + correct_thermal_limits!(data) + else + # Set rate_a as 0.0 for those branch dict entries witn no "rate_a" key + branches = [branch for branch in values(data["branch"])] + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + for branch in branches + if !haskey(branch, "rate_a") + if haskey(data, "conductors") + error("Multiconductor Not Supported in PowerSystems") + else + branch["rate_a"] = 0.0 + end + end + end + + Set{Int}() + end + + #mod_branch[:ma_zero] = correct_current_limits!(data) + mod_branch[:orientation] = correct_branch_directions!(data) + check_branch_loops(data) + + mod_dcline[:losses] = correct_dcline_limits!(data) + + check_voltage_setpoints(data) + + check_storage_parameters(data) + check_switch_parameters(data) + + gen, dcline = correct_cost_functions!(data) + mod_gen[:cost_pwl] = gen + mod_dcline[:cost_pwl] = dcline + + simplify_cost_terms!(data) + + return Dict( + "gen" => mod_gen, + "branch" => mod_branch, + "dcline" => mod_dcline, + ) +end + +UNIT_SYSTEM_MAPPING = Dict( + "SYSTEM_BASE" => IS.UnitSystem.SYSTEM_BASE, + "DEVICE_BASE" => IS.UnitSystem.DEVICE_BASE, + "NATURAL_UNITS" => IS.UnitSystem.NATURAL_UNITS, + "NA" => nothing, +) diff --git a/src/pm_io/data.jl b/src/pm_io/data.jl new file mode 100644 index 0000000..9335fc7 --- /dev/null +++ b/src/pm_io/data.jl @@ -0,0 +1,2889 @@ +# tools for working with a PowerModels data dict structure + +"" +function calc_branch_t(branch::Dict{String, <:Any}) + tap_ratio = branch["tap"] + angle_shift = branch["shift"] + + tr = tap_ratio .* cos.(angle_shift) + ti = tap_ratio .* sin.(angle_shift) + + return tr, ti +end + +"" +function calc_branch_y(branch::Dict{String, <:Any}) + y = LinearAlgebra.pinv(branch["br_r"] + im * branch["br_x"]) + g, b = real(y), imag(y) + return g, b +end + +"" +function calc_theta_delta_bounds(data::Dict{String, <:Any}) + bus_count = length(data["bus"]) + branches = [branch for branch in values(data["branch"])] + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + angle_min = Real[] + angle_max = Real[] + + conductors = 1 + if haskey(data, "conductors") + conductors = data["conductors"] + end + conductor_ids = 1:conductors + + for c in conductor_ids + angle_mins = [branch["angmin"][c] for branch in branches] + angle_maxs = [branch["angmax"][c] for branch in branches] + + sort!(angle_mins) + sort!(angle_maxs; rev = true) + + if length(angle_mins) > 1 + # note that, this can occur when dclines are present + angle_count = min(bus_count - 1, length(branches)) + + angle_min_val = sum(angle_mins[1:angle_count]) + angle_max_val = sum(angle_maxs[1:angle_count]) + else + angle_min_val = angle_mins[1] + angle_max_val = angle_maxs[1] + end + + push!(angle_min, angle_min_val) + push!(angle_max, angle_max_val) + end + + if haskey(data, "conductors") + error("Multiconductor Not Supported in PowerSystems") + #amin = MultiConductorVector(angle_min) + #amax = MultiConductorVector(angle_max) + #return amin, amax + else + return angle_min[1], angle_max[1] + end +end + +"" +function calc_max_cost_index(data::Dict{String, Any}) + if ismultinetwork(data) # ismultinetwork is in im_io/data.jl + max_index = 0 + for (i, nw_data) in data["nw"] + nw_max_index = _calc_max_cost_index(nw_data) + max_index = max(max_index, nw_max_index) + end + return max_index + else + return _calc_max_cost_index(data) + end +end + +"" +function _calc_max_cost_index(data::Dict{String, <:Any}) + max_index = 0 + + for (i, gen) in data["gen"] + if haskey(gen, "model") + if gen["model"] == 2 + if haskey(gen, "cost") + max_index = max(max_index, length(gen["cost"])) + end + else + @info( + "skipping cost generator $(i) cost model in calc_cost_order, only model 2 is supported." + ) + end + end + end + + for (i, dcline) in data["dcline"] + if haskey(dcline, "model") + if dcline["model"] == 2 + if haskey(dcline, "cost") + max_index = max(max_index, length(dcline["cost"])) + end + else + @info( + "skipping cost dcline $(i) cost model in calc_cost_order, only model 2 is supported." + ) + end + end + end + + return max_index +end + +"maps component types to status parameters" +const pm_component_status = Dict( + "bus" => "bus_type", + "load" => "status", + "shunt" => "status", + "gen" => "gen_status", + "storage" => "status", + "switch" => "status", + "branch" => "br_status", + "dcline" => "br_status", +) + +"maps component types to inactive status values" +const pm_component_status_inactive = Dict( + "bus" => 4, + "load" => 0, + "shunt" => 0, + "gen" => 0, + "storage" => 0, + "switch" => 0, + "branch" => 0, + "dcline" => 0, +) + +const _pm_component_types_order = Dict( + "bus" => 1.0, + "load" => 2.0, + "shunt" => 3.0, + "gen" => 4.0, + "storage" => 5.0, + "switch" => 6.0, + "branch" => 7.0, + "dcline" => 8.0, +) + +const _pm_component_parameter_order = Dict( + "bus_i" => 1.0, + "load_bus" => 2.0, + "shunt_bus" => 3.0, + "gen_bus" => 4.0, + "storage_bus" => 5.0, + "f_bus" => 6.0, + "t_bus" => 7.0, + "bus_name" => 9.1, + "base_kv" => 9.2, + "bus_type" => 9.3, + "vm" => 10.0, + "va" => 11.0, + "pd" => 20.0, + "qd" => 21.0, + "gs" => 30.0, + "bs" => 31.0, + "ps" => 35.0, + "qs" => 36.0, + "psw" => 37.0, + "qsw" => 38.0, + "pg" => 40.0, + "qg" => 41.0, + "vg" => 42.0, + "mbase" => 43.0, + "energy" => 44.0, + "br_r" => 50.0, + "br_x" => 51.0, + "g_fr" => 52.0, + "b_fr" => 53.0, + "g_to" => 54.0, + "b_to" => 55.0, + "tap" => 56.0, + "shift" => 57.0, + "vf" => 58.1, + "pf" => 58.2, + "qf" => 58.3, + "vt" => 58.4, + "pt" => 58.5, + "qt" => 58.6, + "loss0" => 58.7, + "loss1" => 59.8, + "vmin" => 60.0, + "vmax" => 61.0, + "pmin" => 62.0, + "pmax" => 63.0, + "qmin" => 64.0, + "qmax" => 65.0, + "rate_a" => 66.0, + "rate_b" => 67.0, + "rate_c" => 68.0, + "pminf" => 69.0, + "pmaxf" => 70.0, + "qminf" => 71.0, + "qmaxf" => 72.0, + "pmint" => 73.0, + "pmaxt" => 74.0, + "qmint" => 75.0, + "qmaxt" => 76.0, + "energy_rating" => 77.01, + "charge_rating" => 77.02, + "discharge_rating" => 77.03, + "charge_efficiency" => 77.04, + "discharge_efficiency" => 77.05, + "thermal_rating" => 77.06, + "qmin" => 77.07, + "qmax" => 77.08, + "qmin" => 77.09, + "qmax" => 77.10, + "r" => 77.11, + "x" => 77.12, + "p_loss" => 77.13, + "q_loss" => 77.14, + "status" => 80.0, + "gen_status" => 81.0, + "br_status" => 82.0, + "model" => 90.0, + "ncost" => 91.0, + "cost" => 92.0, + "startup" => 93.0, + "shutdown" => 94.0, +) + +const _pm_component_status_parameters = Set(["status", "gen_status", "br_status"]) + +#component_table(data::Dict{String,Any}, component::String, args...) = component_table(data, component, args...) + +#= +"recursively applies new_data to data, overwriting information" +function update_data!(data::Dict{String,<:Any}, new_data::Dict{String,<:Any}) + if haskey(data, "conductors") && haskey(new_data, "conductors") + if data["conductors"] != new_data["conductors"] + error("update_data requires datasets with the same number of conductors") + end + else + if (haskey(data, "conductors") && !haskey(new_data, "conductors")) || (!haskey(data, "conductors") && haskey(new_data, "conductors")) + @info("running update_data with missing onductors fields, conductors may be incorrect") + end + end + update_data!(data, new_data) +end +=# +""" +Turns in given single network data in multinetwork data with a `count` +replicate of the given network. Note that this function performs a deepcopy +of the network data. Significant multinetwork space savings can often be +achieved by building application specific methods of building multinetwork +with minimal data replication. +""" +function replicate( + sn_data::Dict{String, <:Any}, + count::Int; + global_keys::Set{String} = Set{String}(), +) + pm_global_keys = Set(["baseMVA", "per_unit"]) + return im_replicate(sn_data, count, union(global_keys, pm_global_keys)) +end + +"" +function _apply_func!(data::Dict{String, <:Any}, key::String, func) + if haskey(data, key) + data[key] = func(data[key]) # multiconductor not supported in PowerSystems + end +end + +"Transforms network data into per-unit" +function make_per_unit!(data::Dict{String, <:Any}) + if !haskey(data, "per_unit") || data["per_unit"] == false + data["per_unit"] = true + mva_base = data["baseMVA"] + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _make_per_unit!(nw_data, mva_base) + end + else + _make_per_unit!(data, mva_base) + end + end +end + +"" +function _make_per_unit!(data::Dict{String, <:Any}, mva_base::Real) + # to be consistent with matpower's opf.flow_lim= 'I' with current magnitude + # limit defined in MVA at 1 p.u. voltage + ka_base = mva_base + + rescale = x -> x / mva_base + rescale_dual = x -> x * mva_base + rescale_ampere = x -> x / ka_base + + if haskey(data, "bus") + for (i, bus) in data["bus"] + _apply_func!(bus, "va", deg2rad) + + _apply_func!(bus, "lam_kcl_r", rescale_dual) + _apply_func!(bus, "lam_kcl_i", rescale_dual) + end + end + + if haskey(data, "load") + for (i, load) in data["load"] + _apply_func!(load, "pd", rescale) + _apply_func!(load, "qd", rescale) + _apply_func!(load, "pi", rescale) + _apply_func!(load, "qi", rescale) + _apply_func!(load, "py", rescale) + _apply_func!(load, "qy", rescale) + end + end + + if haskey(data, "shunt") + for (i, shunt) in data["shunt"] + _apply_func!(shunt, "gs", rescale) + _apply_func!(shunt, "bs", rescale) + end + end + + if haskey(data, "switched_shunt") + for (i, sw_shunt) in data["switched_shunt"] + _apply_func!(sw_shunt, "gs", rescale) + _apply_func!(sw_shunt, "bs", rescale) + _apply_func!(sw_shunt, "y_increment", rescale) + end + end + + if haskey(data, "gen") + for (i, gen) in data["gen"] + _apply_func!(gen, "pg", rescale) + _apply_func!(gen, "qg", rescale) + + _apply_func!(gen, "pmax", rescale) + _apply_func!(gen, "pmin", rescale) + + _apply_func!(gen, "qmax", rescale) + _apply_func!(gen, "qmin", rescale) + + _apply_func!(gen, "ramp_agc", rescale) + _apply_func!(gen, "ramp_10", rescale) + _apply_func!(gen, "ramp_30", rescale) + _apply_func!(gen, "ramp_q", rescale) + + _rescale_cost_model!(gen, mva_base) + end + end + + if haskey(data, "storage") + for (i, strg) in data["storage"] + _apply_func!(strg, "energy", rescale) + _apply_func!(strg, "energy_rating", rescale) + _apply_func!(strg, "charge_rating", rescale) + _apply_func!(strg, "discharge_rating", rescale) + _apply_func!(strg, "thermal_rating", rescale) + _apply_func!(strg, "current_rating", rescale) + _apply_func!(strg, "qmin", rescale) + _apply_func!(strg, "qmax", rescale) + _apply_func!(strg, "p_loss", rescale) + _apply_func!(strg, "q_loss", rescale) + end + end + + if haskey(data, "switch") + for (i, switch) in data["switch"] + _apply_func!(switch, "psw", rescale) + _apply_func!(switch, "qsw", rescale) + _apply_func!(switch, "thermal_rating", rescale) + _apply_func!(switch, "current_rating", rescale) + end + end + + branches = [] + if haskey(data, "branch") + append!(branches, values(data["branch"])) + end + + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + for branch in branches + _apply_func!(branch, "rate_a", rescale) + _apply_func!(branch, "rate_b", rescale) + _apply_func!(branch, "rate_c", rescale) + + _apply_func!(branch, "c_rating_a", rescale_ampere) + _apply_func!(branch, "c_rating_b", rescale_ampere) + _apply_func!(branch, "c_rating_c", rescale_ampere) + + _apply_func!(branch, "shift", deg2rad) + _apply_func!(branch, "angmax", deg2rad) + _apply_func!(branch, "angmin", deg2rad) + + _apply_func!(branch, "pf", rescale) + _apply_func!(branch, "pt", rescale) + _apply_func!(branch, "qf", rescale) + _apply_func!(branch, "qt", rescale) + + _apply_func!(branch, "mu_sm_fr", rescale_dual) + _apply_func!(branch, "mu_sm_to", rescale_dual) + + _apply_func!(branch, "ta_max", deg2rad) + _apply_func!(branch, "ta_min", deg2rad) + end + + if haskey(data, "dcline") + for (i, dcline) in data["dcline"] + _apply_func!(dcline, "loss0", rescale) + _apply_func!(dcline, "pf", rescale) + _apply_func!(dcline, "pt", rescale) + _apply_func!(dcline, "qf", rescale) + _apply_func!(dcline, "qt", rescale) + _apply_func!(dcline, "pmaxt", rescale) + _apply_func!(dcline, "pmint", rescale) + _apply_func!(dcline, "pmaxf", rescale) + _apply_func!(dcline, "pminf", rescale) + _apply_func!(dcline, "qmaxt", rescale) + _apply_func!(dcline, "qmint", rescale) + _apply_func!(dcline, "qmaxf", rescale) + _apply_func!(dcline, "qminf", rescale) + + _rescale_cost_model!(dcline, mva_base) + end + end +end + +"Transforms network data into mixed-units (inverse of per-unit)" +function make_mixed_units!(data::Dict{String, <:Any}) + if haskey(data, "per_unit") && data["per_unit"] == true + data["per_unit"] = false + mva_base = data["baseMVA"] + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _make_mixed_units!(nw_data, mva_base) + end + else + _make_mixed_units!(data, mva_base) + end + end +end + +"" +function _make_mixed_units!(data::Dict{String, <:Any}) + mva_base = data["baseMVA"] + + # to be consistent with matpower's opf.flow_lim= 'I' with current magnitude + # limit defined in MVA at 1 p.u. voltage + ka_base = mva_base + + rescale = x -> x * mva_base + rescale_dual = x -> x / mva_base + rescale_ampere = x -> x * ka_base + + if haskey(data, "bus") + for (i, bus) in data["bus"] + _apply_func!(bus, "va", rad2deg) + + _apply_func!(bus, "lam_kcl_r", rescale_dual) + _apply_func!(bus, "lam_kcl_i", rescale_dual) + end + end + + if haskey(data, "load") + for (i, load) in data["load"] + _apply_func!(load, "pd", rescale) + _apply_func!(load, "qd", rescale) + end + end + + if haskey(data, "shunt") + for (i, shunt) in data["shunt"] + _apply_func!(shunt, "gs", rescale) + _apply_func!(shunt, "bs", rescale) + end + end + + if haskey(data, "gen") + for (i, gen) in data["gen"] + _apply_func!(gen, "pg", rescale) + _apply_func!(gen, "qg", rescale) + + _apply_func!(gen, "pmax", rescale) + _apply_func!(gen, "pmin", rescale) + + _apply_func!(gen, "qmax", rescale) + _apply_func!(gen, "qmin", rescale) + + _apply_func!(gen, "ramp_agc", rescale) + _apply_func!(gen, "ramp_10", rescale) + _apply_func!(gen, "ramp_30", rescale) + _apply_func!(gen, "ramp_q", rescale) + + _rescale_cost_model!(gen, 1.0 / mva_base) + end + end + + if haskey(data, "storage") + for (i, strg) in data["storage"] + _apply_func!(strg, "energy", rescale) + _apply_func!(strg, "energy_rating", rescale) + _apply_func!(strg, "charge_rating", rescale) + _apply_func!(strg, "discharge_rating", rescale) + _apply_func!(strg, "thermal_rating", rescale) + _apply_func!(strg, "current_rating", rescale) + _apply_func!(strg, "qmin", rescale) + _apply_func!(strg, "qmax", rescale) + _apply_func!(strg, "p_loss", rescale) + _apply_func!(strg, "q_loss", rescale) + end + end + + if haskey(data, "switch") + for (i, switch) in data["switch"] + _apply_func!(switch, "psw", rescale) + _apply_func!(switch, "qsw", rescale) + _apply_func!(switch, "thermal_rating", rescale) + _apply_func!(switch, "current_rating", rescale) + end + end + + branches = [] + if haskey(data, "branch") + append!(branches, values(data["branch"])) + end + + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + for branch in branches + _apply_func!(branch, "rate_a", rescale) + _apply_func!(branch, "rate_b", rescale) + _apply_func!(branch, "rate_c", rescale) + + _apply_func!(branch, "c_rating_a", rescale_ampere) + _apply_func!(branch, "c_rating_b", rescale_ampere) + _apply_func!(branch, "c_rating_c", rescale_ampere) + + _apply_func!(branch, "shift", rad2deg) + _apply_func!(branch, "angmax", rad2deg) + _apply_func!(branch, "angmin", rad2deg) + + _apply_func!(branch, "pf", rescale) + _apply_func!(branch, "pt", rescale) + _apply_func!(branch, "qf", rescale) + _apply_func!(branch, "qt", rescale) + + _apply_func!(branch, "mu_sm_fr", rescale_dual) + _apply_func!(branch, "mu_sm_to", rescale_dual) + + _apply_func!(branch, "ta", rad2deg) + end + + if haskey(data, "dcline") + for (i, dcline) in data["dcline"] + _apply_func!(dcline, "loss0", rescale) + _apply_func!(dcline, "pf", rescale) + _apply_func!(dcline, "pt", rescale) + _apply_func!(dcline, "qf", rescale) + _apply_func!(dcline, "qt", rescale) + _apply_func!(dcline, "pmaxt", rescale) + _apply_func!(dcline, "pmint", rescale) + _apply_func!(dcline, "pmaxf", rescale) + _apply_func!(dcline, "pminf", rescale) + _apply_func!(dcline, "qmaxt", rescale) + _apply_func!(dcline, "qmint", rescale) + _apply_func!(dcline, "qmaxf", rescale) + _apply_func!(dcline, "qminf", rescale) + + _rescale_cost_model!(dcline, 1.0 / mva_base) + end + end +end + +"" +function _rescale_cost_model!(comp::Dict{String, <:Any}, scale::Real) + if "model" in keys(comp) && "cost" in keys(comp) + if comp["model"] == 1 + for i in 1:2:length(comp["cost"]) + comp["cost"][i] = comp["cost"][i] / scale + end + elseif comp["model"] == 2 + degree = length(comp["cost"]) + for (i, item) in enumerate(comp["cost"]) + comp["cost"][i] = item * (scale^(degree - i)) + end + else + @info("Skipping cost model of type $(comp["model"]) in per unit transformation") + end + end +end + +"computes the generator cost from given network data" +function calc_gen_cost(data::Dict{String, <:Any}) + @assert("per_unit" in keys(data) && data["per_unit"]) + @assert(!haskey(data, "conductors")) + + if ismultinetwork(data) + nw_costs = Dict{String, Any}() + for (i, nw_data) in data["nw"] + nw_costs[i] = _calc_gen_cost(nw_data) + end + return sum(nw_cost for (i, nw_cost) in nw_costs) + else + return _calc_gen_cost(data) + end +end + +function _calc_gen_cost(data::Dict{String, <:Any}) + cost = 0.0 + for (i, gen) in data["gen"] + if gen["gen_status"] == 1 + if haskey(gen, "model") + if gen["model"] == 1 + cost += _calc_cost_pwl(gen, "pg") + elseif gen["model"] == 2 + cost += _calc_cost_polynomial(gen, "pg") + else + @info "generator $(i) has an unknown cost model $(gen["model"])" maxlog = + PS_MAX_LOG + end + else + @info "generator $(i) does not have a cost model" maxlog = PS_MAX_LOG + end + end + end + return cost +end + +"computes the dcline cost from given network data" +function calc_dcline_cost(data::Dict{String, <:Any}) + @assert("per_unit" in keys(data) && data["per_unit"]) + @assert(!haskey(data, "conductors")) + + if ismultinetwork(data) + nw_costs = Dict{String, Any}() + for (i, nw_data) in data["nw"] + nw_costs[i] = _calc_dcline_cost(nw_data) + end + return sum(nw_cost for (i, nw_cost) in nw_costs) + else + return _calc_dcline_cost(data) + end +end + +function _calc_dcline_cost(data::Dict{String, <:Any}) + cost = 0.0 + for (i, dcline) in data["dcline"] + if dcline["br_status"] == 1 + if haskey(dcline, "model") + if dcline["model"] == 1 + cost += _calc_cost_pwl(dcline, "pf") + elseif dcline["model"] == 2 + cost += _calc_cost_polynomial(dcline, "pf") + else + @info "dcline $(i) has an unknown cost model $(dcline["model"])" maxlog = + PS_MAX_LOG + end + else + @info "dcline $(i) does not have a cost model" maxlog = PS_MAX_LOG + end + end + end + return cost +end + +""" +compute lines in m and b from from pwl cost models data is a list of components. + +Can be run on data or ref data structures +""" +function calc_cost_pwl_lines(comp_dict::Dict) + lines = Dict() + for (i, comp) in comp_dict + lines[i] = _calc_comp_lines(comp) + end + return lines +end + +""" +compute lines in m and b from from pwl cost models +""" +function _calc_comp_lines(component::Dict{String, <:Any}) + @assert component["model"] == 1 + points = component["cost"] + + line_data = [] + for i in 3:2:length(points) + x1 = points[i - 2] + y1 = points[i - 1] + x2 = points[i - 0] + y2 = points[i + 1] + + m = (y2 - y1) / (x2 - x1) + b = y1 - m * x1 + + push!(line_data, (slope = m, intercept = b)) + end + + for i in 2:length(line_data) + if line_data[i - 1].slope > line_data[i].slope + @info "non-convex pwl function found in points $(component["cost"])\nlines: $(line_data)" maxlog = + PS_MAX_LOG + end + end + + return line_data +end + +function _calc_cost_pwl(component::Dict{String, <:Any}, setpoint_id) + comp_lines = _calc_comp_lines(component) + + setpoint = component[setpoint_id] + cost = -Inf + for line in comp_lines + cost = max(cost, line.slope * setpoint + line.intercept) + end + + return cost +end + +function _calc_cost_polynomial(component::Dict{String, <:Any}, setpoint_id) + cost_terms_rev = reverse(component["cost"]) + + setpoint = component[setpoint_id] + + if length(cost_terms_rev) == 0 + cost = 0.0 + elseif length(cost_terms_rev) == 1 + cost = cost_terms_rev[1] + elseif length(cost_terms_rev) == 2 + cost = cost_terms_rev[1] + cost_terms_rev[2] * setpoint + else + cost_terms_rev_high = cost_terms_rev[3:end] + cost = + cost_terms_rev[1] + + cost_terms_rev[2] * setpoint + + sum(v * setpoint^(d + 1) for (d, v) in enumerate(cost_terms_rev_high)) + end + + return cost +end + +"assumes a vaild ac solution is included in the data and computes the branch flow values" +function calc_branch_flow_ac(data::Dict{String, <:Any}) + @assert("per_unit" in keys(data) && data["per_unit"]) + @assert(!haskey(data, "conductors")) + + if ismultinetwork(data) + nws = Dict{String, Any}() + for (i, nw_data) in data["nw"] + nws[i] = _calc_branch_flow_ac(nw_data) + end + return Dict{String, Any}( + "nw" => nws, + "per_unit" => data["per_unit"], + "baseMVA" => data["baseMVA"], + ) + else + flows = _calc_branch_flow_ac(data) + flows["per_unit"] = data["per_unit"] + flows["baseMVA"] = data["baseMVA"] + return flows + end +end + +"helper function for calc_branch_flow_ac" +function _calc_branch_flow_ac(data::Dict{String, <:Any}) + vm = Dict(bus["index"] => bus["vm"] for (i, bus) in data["bus"]) + va = Dict(bus["index"] => bus["va"] for (i, bus) in data["bus"]) + + flows = Dict{String, Any}() + for (i, branch) in data["branch"] + if branch["br_status"] != 0 + f_bus = branch["f_bus"] + t_bus = branch["t_bus"] + + g, b = calc_branch_y(branch) + tr, ti = calc_branch_t(branch) + g_fr = branch["g_fr"] + b_fr = branch["b_fr"] + g_to = branch["g_to"] + b_to = branch["b_to"] + + tm = branch["tap"] + + vm_fr = vm[f_bus] + vm_to = vm[t_bus] + va_fr = va[f_bus] + va_to = va[t_bus] + + p_fr = + (g + g_fr) / tm^2 * vm_fr^2 + + (-g * tr + b * ti) / tm^2 * (vm_fr * vm_to * cos(va_fr - va_to)) + + (-b * tr - g * ti) / tm^2 * (vm_fr * vm_to * sin(va_fr - va_to)) + q_fr = + -(b + b_fr) / tm^2 * vm_fr^2 - + (-b * tr - g * ti) / tm^2 * (vm_fr * vm_to * cos(va_fr - va_to)) + + (-g * tr + b * ti) / tm^2 * (vm_fr * vm_to * sin(va_fr - va_to)) + + p_to = + (g + g_to) * vm_to^2 + + (-g * tr - b * ti) / tm^2 * (vm_to * vm_fr * cos(va_to - va_fr)) + + (-b * tr + g * ti) / tm^2 * (vm_to * vm_fr * sin(va_to - va_fr)) + q_to = + -(b + b_to) * vm_to^2 - + (-b * tr + g * ti) / tm^2 * (vm_to * vm_fr * cos(va_to - va_fr)) + + (-g * tr - b * ti) / tm^2 * (vm_to * vm_fr * sin(va_to - va_fr)) + else + p_fr = NaN + q_fr = NaN + + p_to = NaN + q_to = NaN + end + + flows[i] = Dict("pf" => p_fr, "qf" => q_fr, "pt" => p_to, "qt" => q_to) + end + + return Dict{String, Any}("branch" => flows) +end + +"assumes a vaild dc solution is included in the data and computes the branch flow values" +function calc_branch_flow_dc(data::Dict{String, <:Any}) + @assert("per_unit" in keys(data) && data["per_unit"]) + @assert(!haskey(data, "conductors")) + + if ismultinetwork(data) + nws = Dict{String, Any}() + for (i, nw_data) in data["nw"] + nws[i] = _calc_branch_flow_dc(nw_data) + end + return Dict{String, Any}( + "nw" => nws, + "per_unit" => data["per_unit"], + "baseMVA" => data["baseMVA"], + ) + else + flows = _calc_branch_flow_dc(data) + flows["per_unit"] = data["per_unit"] + flows["baseMVA"] = data["baseMVA"] + return flows + end +end + +"helper function for calc_branch_flow_dc" +function _calc_branch_flow_dc(data::Dict{String, <:Any}) + vm = Dict(bus["index"] => bus["vm"] for (i, bus) in data["bus"]) + va = Dict(bus["index"] => bus["va"] for (i, bus) in data["bus"]) + + flows = Dict{String, Any}() + for (i, branch) in data["branch"] + if branch["br_status"] != 0 + f_bus = branch["f_bus"] + t_bus = branch["t_bus"] + + g, b = calc_branch_y(branch) + + p_fr = -b * (va[f_bus] - va[t_bus]) + else + p_fr = NaN + end + + flows[i] = Dict("pf" => p_fr, "qf" => NaN, "pt" => -p_fr, "qt" => NaN) + end + + return Dict{String, Any}("branch" => flows) +end + +"assumes a vaild solution is included in the data and computes the power balance at each bus" +function calc_power_balance(data::Dict{String, <:Any}) + @assert("per_unit" in keys(data) && data["per_unit"]) # may not be strictly required + @assert(!haskey(data, "conductors")) + + if ismultinetwork(data) + nws = Dict{String, Any}() + for (i, nw_data) in data["nw"] + nws[i] = _calc_power_balance(nw_data) + end + return Dict{String, Any}( + "nw" => nws, + "per_unit" => data["per_unit"], + "baseMVA" => data["baseMVA"], + ) + else + flows = _calc_power_balance(data) + flows["per_unit"] = data["per_unit"] + flows["baseMVA"] = data["baseMVA"] + return flows + end +end + +"helper function for calc_power_balance" +function _calc_power_balance(data::Dict{String, <:Any}) + bus_values = Dict(bus["index"] => Dict{String, Float64}() for (i, bus) in data["bus"]) + for (i, bus) in data["bus"] + bvals = bus_values[bus["index"]] + bvals["vm"] = bus["vm"] + + bvals["pd"] = 0.0 + bvals["qd"] = 0.0 + + bvals["gs"] = 0.0 + bvals["bs"] = 0.0 + + bvals["ps"] = 0.0 + bvals["qs"] = 0.0 + + bvals["pg"] = 0.0 + bvals["qg"] = 0.0 + + bvals["p"] = 0.0 + bvals["q"] = 0.0 + + bvals["psw"] = 0.0 + bvals["qsw"] = 0.0 + + bvals["p_dc"] = 0.0 + bvals["q_dc"] = 0.0 + end + + for (i, load) in data["load"] + if load["status"] != 0 + bvals = bus_values[load["load_bus"]] + bvals["pd"] += load["pd"] + bvals["qd"] += load["qd"] + end + end + + for (i, shunt) in data["shunt"] + if shunt["status"] != 0 + bvals = bus_values[shunt["shunt_bus"]] + bvals["gs"] += shunt["gs"] + bvals["bs"] += shunt["bs"] + end + end + + for (i, storage) in data["storage"] + if storage["status"] != 0 + bvals = bus_values[storage["storage_bus"]] + bvals["ps"] += storage["ps"] + bvals["qs"] += storage["qs"] + end + end + + for (i, gen) in data["gen"] + if gen["gen_status"] != 0 + bvals = bus_values[gen["gen_bus"]] + bvals["pg"] += gen["pg"] + bvals["qg"] += gen["qg"] + end + end + + for (i, switch) in data["switch"] + if switch["status"] != 0 + bus_fr = switch["f_bus"] + bvals_fr = bus_values[bus_fr] + bvals_fr["psw"] += switch["psw"] + bvals_fr["qsw"] += switch["qsw"] + + bus_to = switch["t_bus"] + bvals_to = bus_values[bus_to] + bvals_to["psw"] -= switch["psw"] + bvals_to["qsw"] -= switch["qsw"] + end + end + + for (i, branch) in data["branch"] + if branch["br_status"] != 0 + bus_fr = branch["f_bus"] + bvals_fr = bus_values[bus_fr] + bvals_fr["p"] += branch["pf"] + bvals_fr["q"] += branch["qf"] + + bus_to = branch["t_bus"] + bvals_to = bus_values[bus_to] + bvals_to["p"] += branch["pt"] + bvals_to["q"] += branch["qt"] + end + end + + for (i, dcline) in data["dcline"] + if dcline["br_status"] != 0 + bus_fr = dcline["f_bus"] + bvals_fr = bus_values[bus_fr] + bvals_fr["p_dc"] += dcline["pf"] + bvals_fr["q_dc"] += dcline["qf"] + + bus_to = dcline["t_bus"] + bvals_to = bus_values[bus_to] + bvals_to["p_dc"] += dcline["pt"] + bvals_to["q_dc"] += dcline["qt"] + end + end + + deltas = Dict{String, Any}() + for (i, bus) in data["bus"] + if bus["bus_type"] != 4 + bvals = bus_values[bus["index"]] + p_delta = + bvals["p"] + bvals["p_dc"] + bvals["psw"] - bvals["pg"] + + bvals["ps"] + + bvals["pd"] + + bvals["gs"] * (bvals["vm"]^2) + q_delta = + bvals["q"] + bvals["q_dc"] + bvals["qsw"] - bvals["qg"] + + bvals["qs"] + + bvals["qd"] - bvals["bs"] * (bvals["vm"]^2) + else + p_delta = NaN + q_delta = NaN + end + + deltas[i] = Dict("p_delta" => p_delta, "q_delta" => q_delta) + end + + return Dict{String, Any}("bus" => deltas) +end + +"" +function check_conductors(data::Dict{String, <:Any}) + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _check_conductors(nw_data) + end + else + _check_conductors(data) + end +end + +"" +function _check_conductors(data::Dict{String, <:Any}) + if haskey(data, "conductors") && data["conductors"] < 1 + error("conductor values must be positive integers, given $(data["conductors"])") + end +end + +"checks that voltage angle differences are within 90 deg., if not tightens" +function correct_voltage_angle_differences!(data::Dict{String, <:Any}, default_pad = 1.0472) + if ismultinetwork(data) + error("check_voltage_angle_differences does not yet support multinetwork data") + end + + @assert("per_unit" in keys(data) && data["per_unit"]) + default_pad_deg = round(rad2deg(default_pad); digits = 2) + + modified = Set{Int}() + + for c in 1:get(data, "conductors", 1) + cnd_str = haskey(data, "conductors") ? ", conductor $(c)" : "" + for (i, branch) in data["branch"] + angmin = branch["angmin"][c] + angmax = branch["angmax"][c] + + if angmin <= -pi / 2 + @info "this code only supports angmin values in -90 deg. to 90 deg., tightening the value on branch $i$(cnd_str) from $(rad2deg(angmin)) to -$(default_pad_deg) deg." maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + branch["angmin"][c] = -default_pad + else + branch["angmin"] = -default_pad + end + push!(modified, branch["index"]) + end + + if angmax >= pi / 2 + @info "this code only supports angmax values in -90 deg. to 90 deg., tightening the value on branch $i$(cnd_str) from $(rad2deg(angmax)) to $(default_pad_deg) deg." maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + branch["angmax"][c] = default_pad + else + branch["angmax"] = default_pad + end + push!(modified, branch["index"]) + end + + if angmin == 0.0 && angmax == 0.0 + @info "angmin and angmax values are 0, widening these values on branch $i$(cnd_str) to +/- $(default_pad_deg) deg." maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + branch["angmin"][c] = -default_pad + branch["angmax"][c] = default_pad + else + branch["angmin"] = -default_pad + branch["angmax"] = default_pad + end + push!(modified, branch["index"]) + end + end + end + + return modified +end + +"checks that each branch has a reasonable thermal rating-a, if not computes one" +function correct_thermal_limits!(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("correct_thermal_limits! does not yet support multinetwork data") + end + + @assert("per_unit" in keys(data) && data["per_unit"]) + mva_base = data["baseMVA"] + + modified = Set{Int}() + + branches = [branch for branch in values(data["branch"])] + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + for branch in branches + if !haskey(branch, "rate_a") + if haskey(data, "conductors") + error("Multiconductor Not Supported in PowerSystems") + else + branch["rate_a"] = 0.0 + end + end + + for c in 1:get(data, "conductors", 1) + cnd_str = haskey(data, "conductors") ? ", conductor $(c)" : "" + if branch["rate_a"][c] <= 0.0 + theta_max = max(abs(branch["angmin"][c]), abs(branch["angmax"][c])) + + r = branch["br_r"] + x = branch["br_x"] + z = r + im * x + y = LinearAlgebra.pinv(z) + y_mag = abs.(y[c, c]) + + fr_vmax = data["bus"][branch["f_bus"]]["vmax"][c] + to_vmax = data["bus"][branch["t_bus"]]["vmax"][c] + m_vmax = max(fr_vmax, to_vmax) + + c_max = sqrt(fr_vmax^2 + to_vmax^2 - 2 * fr_vmax * to_vmax * cos(theta_max)) + + new_rate = y_mag * m_vmax * c_max + + if haskey(branch, "c_rating_a") && branch["c_rating_a"][c] > 0.0 + new_rate = min(new_rate, branch["c_rating_a"][c] * m_vmax) + end + + @info "this code only supports positive rate_a values, changing the value on branch $(branch["index"])$(cnd_str) to $(round(mva_base*new_rate, digits=4))" maxlog = + PS_MAX_LOG + + if haskey(data, "conductors") + branch["rate_a"][c] = new_rate + else + branch["rate_a"] = new_rate + end + + push!(modified, branch["index"]) + end + end + end + + return modified +end + +"checks that each branch has a reasonable current rating-a, if not computes one" +function correct_current_limits!(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("correct_current_limits! does not yet support multinetwork data") + end + + @assert("per_unit" in keys(data) && data["per_unit"]) + mva_base = data["baseMVA"] + + modified = Set{Int}() + + branches = [branch for branch in values(data["branch"])] + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + for branch in branches + if !haskey(branch, "c_rating_a") + if haskey(data, "conductors") + error("Multiconductor Not Supported in PowerSystems") + else + branch["c_rating_a"] = 0.0 + end + end + + for c in 1:get(data, "conductors", 1) + cnd_str = haskey(data, "conductors") ? ", conductor $(c)" : "" + if branch["c_rating_a"][c] <= 0.0 + theta_max = max(abs(branch["angmin"][c]), abs(branch["angmax"][c])) + + r = branch["br_r"] + x = branch["br_x"] + z = r + im * x + y = LinearAlgebra.pinv(z) + y_mag = abs.(y[c, c]) + + fr_vmax = data["bus"][string(branch["f_bus"])]["vmax"][c] + to_vmax = data["bus"][string(branch["t_bus"])]["vmax"][c] + m_vmax = max(fr_vmax, to_vmax) + + new_c_rating = + y_mag * + sqrt(fr_vmax^2 + to_vmax^2 - 2 * fr_vmax * to_vmax * cos(theta_max)) + + if haskey(branch, "rate_a") && branch["rate_a"][c] > 0.0 + fr_vmin = data["bus"][string(branch["f_bus"])]["vmin"][c] + to_vmin = data["bus"][string(branch["t_bus"])]["vmin"][c] + vm_min = min(fr_vmin, to_vmin) + + new_c_rating = min(new_c_rating, branch["rate_a"] / vm_min) + end + + @info( + "this code only supports positive c_rating_a values, changing the value on branch $(branch["index"])$(cnd_str) to $(mva_base*new_c_rating)" + ) + if haskey(data, "conductors") + branch["c_rating_a"][c] = new_c_rating + else + branch["c_rating_a"] = new_c_rating + end + + push!(modified, branch["index"]) + end + end + end + + return modified +end + +"checks that all parallel branches have the same orientation" +function correct_branch_directions!(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("correct_branch_directions! does not yet support multinetwork data") + end + + modified = Set{Int}() + + orientations = Set() + for (i, branch) in data["branch"] + orientation = (branch["f_bus"], branch["t_bus"]) + orientation_rev = (branch["t_bus"], branch["f_bus"]) + + if in(orientation_rev, orientations) + @info( + "reversing the orientation of branch $(i) $(orientation) to be consistent with other parallel branches" + ) + branch_orginal = copy(branch) + branch["f_bus"] = branch_orginal["t_bus"] + branch["t_bus"] = branch_orginal["f_bus"] + branch["g_to"] = branch_orginal["g_fr"] .* branch_orginal["tap"]' .^ 2 + branch["b_to"] = branch_orginal["b_fr"] .* branch_orginal["tap"]' .^ 2 + branch["g_fr"] = branch_orginal["g_to"] ./ branch_orginal["tap"]' .^ 2 + branch["b_fr"] = branch_orginal["b_to"] ./ branch_orginal["tap"]' .^ 2 + branch["tap"] = 1 ./ branch_orginal["tap"] + branch["br_r"] = branch_orginal["br_r"] .* branch_orginal["tap"]' .^ 2 + branch["br_x"] = branch_orginal["br_x"] .* branch_orginal["tap"]' .^ 2 + branch["shift"] = -branch_orginal["shift"] + branch["angmin"] = -branch_orginal["angmax"] + branch["angmax"] = -branch_orginal["angmin"] + + push!(modified, branch["index"]) + else + push!(orientations, orientation) + end + end + + return modified +end + +"checks that all branches connect two distinct buses" +function check_branch_loops(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_branch_loops does not yet support multinetwork data") + end + + for (i, branch) in data["branch"] + if branch["f_bus"] == branch["t_bus"] + throw( + DataFormatError( + "both sides of branch $(i) connect to bus $(branch["f_bus"])", + ), + ) + end + end +end + +"checks that all buses are unique and other components link to valid buses" +function check_connectivity(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_connectivity does not yet support multinetwork data") + end + + bus_ids = Set(bus["index"] for (i, bus) in data["bus"]) + @assert(length(bus_ids) == length(data["bus"])) # if this is not true something very bad is going on + + for (i, load) in data["load"] + if !(load["load_bus"] in bus_ids) + throw(DataFormatError("bus $(load["load_bus"]) in load $(i) is not defined")) + end + end + + for (i, shunt) in data["shunt"] + if !(shunt["shunt_bus"] in bus_ids) + throw(DataFormatError("bus $(shunt["shunt_bus"]) in shunt $(i) is not defined")) + end + end + + for (i, gen) in data["gen"] + if !(gen["gen_bus"] in bus_ids) + throw(DataFormatError("bus $(gen["gen_bus"]) in generator $(i) is not defined")) + end + end + + for (i, strg) in data["storage"] + if !(strg["storage_bus"] in bus_ids) + throw( + DataFormatError( + "bus $(strg["storage_bus"]) in storage unit $(i) is not defined", + ), + ) + end + end + + if haskey(data, "switch") + for (i, switch) in data["switch"] + if !(switch["f_bus"] in bus_ids) + throw( + DataFormatError( + "from bus $(branch["f_bus"]) in switch $(i) is not defined", + ), + ) + end + + if !(switch["t_bus"] in bus_ids) + throw( + DataFormatError( + "to bus $(branch["t_bus"]) in switch $(i) is not defined", + ), + ) + end + end + end + + for (i, branch) in data["branch"] + if !(branch["f_bus"] in bus_ids) + throw( + DataFormatError( + "from bus $(branch["f_bus"]) in branch $(i) is not defined", + ), + ) + end + + if !(branch["t_bus"] in bus_ids) + throw( + DataFormatError("to bus $(branch["t_bus"]) in branch $(i) is not defined"), + ) + end + end + + for (i, dcline) in data["dcline"] + if !(dcline["f_bus"] in bus_ids) + throw( + DataFormatError( + "from bus $(dcline["f_bus"]) in dcline $(i) is not defined", + ), + ) + end + + if !(dcline["t_bus"] in bus_ids) + throw( + DataFormatError("to bus $(dcline["t_bus"]) in dcline $(i) is not defined"), + ) + end + end +end + +"checks that active components are not connected to inactive buses, otherwise prints warnings" +function check_status(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_status does not yet support multinetwork data") + end + + active_bus_ids = Set(bus["index"] for (i, bus) in data["bus"] if bus["bus_type"] != 4) + + for (i, load) in data["load"] + if load["status"] != 0 && !(load["load_bus"] in active_bus_ids) + @warn("active load $(i) is connected to inactive bus $(load["load_bus"])") + end + end + + for (i, shunt) in data["shunt"] + if shunt["status"] != 0 && !(shunt["shunt_bus"] in active_bus_ids) + @warn("active shunt $(i) is connected to inactive bus $(shunt["shunt_bus"])") + end + end + + for (i, gen) in data["gen"] + if gen["gen_status"] != 0 && !(gen["gen_bus"] in active_bus_ids) + @warn("active generator $(i) is connected to inactive bus $(gen["gen_bus"])") + end + end + + for (i, strg) in data["storage"] + if strg["status"] != 0 && !(strg["storage_bus"] in active_bus_ids) + @warn( + "active storage unit $(i) is connected to inactive bus $(strg["storage_bus"])" + ) + end + end + + for (i, branch) in data["branch"] + if branch["br_status"] != 0 && !(branch["f_bus"] in active_bus_ids) + @warn("active branch $(i) is connected to inactive bus $(branch["f_bus"])") + end + + if branch["br_status"] != 0 && !(branch["t_bus"] in active_bus_ids) + @warn("active branch $(i) is connected to inactive bus $(branch["t_bus"])") + end + end + + for (i, dcline) in data["dcline"] + if dcline["br_status"] != 0 && !(dcline["f_bus"] in active_bus_ids) + @warn("active dcline $(i) is connected to inactive bus $(dcline["f_bus"])") + end + + if dcline["br_status"] != 0 && !(dcline["t_bus"] in active_bus_ids) + @warn("active dcline $(i) is connected to inactive bus $(dcline["t_bus"])") + end + end +end + +"checks that contains at least one refrence bus" +function check_reference_bus(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_reference_bus does not yet support multinetwork data") + end + + ref_buses = Dict{Int, Any}() + for (k, v) in data["bus"] + if v["bus_type"] == 3 + ref_buses[k] = v + end + end + + if length(ref_buses) == 0 + if length(data["gen"]) > 0 + big_gen = _biggest_generator(data["gen"]) + gen_bus = big_gen["gen_bus"] + ref_bus = data["bus"][gen_bus] + ref_bus["bus_type"] = 3 + @warn( + "no reference bus found, setting bus $(gen_bus) as reference based on generator $(big_gen["index"])" + ) + else + (bus_item, state) = Base.iterate(values(data["bus"])) + bus_item["bus_type"] = 3 + @warn( + "no reference bus found, setting bus $(bus_item["index"]) as reference" + ) + end + end + return +end + +"find the largest active generator in the network" +function _biggest_generator(gens) + biggest_gen = nothing + biggest_value = -Inf + for (k, gen) in gens + pmax = maximum(gen["pmax"]) + if pmax > biggest_value + biggest_gen = gen + biggest_value = pmax + end + end + @assert(biggest_gen !== nothing) + return biggest_gen +end + +""" +checks that each branch has a reasonable transformer parameters + +this is important because setting tap == 0.0 leads to NaN computations, which are hard to debug +""" +function correct_transformer_parameters!(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_transformer_parameters does not yet support multinetwork data") + end + + @assert("per_unit" in keys(data) && data["per_unit"]) + + modified = Set{Int}() + + for (i, branch) in data["branch"] + if !haskey(branch, "tap") + @info "branch found without tap value, setting a tap to 1.0" maxlog = PS_MAX_LOG + if haskey(data, "conductors") + error("Multiconductor Not Supported in PowerSystems") + else + branch["tap"] = 1.0 + end + push!(modified, branch["index"]) + else + for c in 1:get(data, "conductors", 1) + cnd_str = haskey(data, "conductors") ? " on conductor $(c)" : "" + if branch["tap"][c] <= 0.0 + @info( + "branch found with non-positive tap value of $(branch["tap"][c]), setting a tap to 1.0$(cnd_str)" + ) + if haskey(data, "conductors") + branch["tap"][c] = 1.0 + else + branch["tap"] = 1.0 + end + push!(modified, branch["index"]) + end + end + end + if !haskey(branch, "shift") + @info("branch found without shift value, setting a shift to 0.0") + if haskey(data, "conductors") + error("Multiconductor Not Supported in PowerSystems") + else + branch["shift"] = 0.0 + end + push!(modified, branch["index"]) + end + end + + return modified +end + +""" +checks that each storage unit has a reasonable parameters +""" +function check_storage_parameters(data::Dict{String, Any}) + if ismultinetwork(data) + error("check_storage_parameters does not yet support multinetwork data") + end + + for (i, strg) in data["storage"] + if strg["energy"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive energy level $(strg["energy"])", + ), + ) + end + if strg["energy_rating"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive energy rating $(strg["energy_rating"])", + ), + ) + end + if strg["charge_rating"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive charge rating $(strg["energy_rating"])", + ), + ) + end + if strg["discharge_rating"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive discharge rating $(strg["energy_rating"])", + ), + ) + end + + if strg["r"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive resistance $(strg["r"])", + ), + ) + end + if strg["x"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive reactance $(strg["x"])", + ), + ) + end + if haskey(strg, "thermal_rating") && strg["thermal_rating"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive thermal rating $(strg["thermal_rating"])", + ), + ) + end + if haskey(strg, "current_rating") && strg["current_rating"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive current rating $(strg["thermal_rating"])", + ), + ) + end + if !isapprox(strg["x"], 0.0; atol = 1e-6, rtol = 1e-6) + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-zero reactance $(strg["x"]), which is currently ignored", + ), + ) + end + + if strg["charge_efficiency"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive charge efficiency of $(strg["charge_efficiency"])", + ), + ) + end + if strg["charge_efficiency"] <= 0.0 || strg["charge_efficiency"] > 1.0 + @info "storage unit $(strg["index"]) charge efficiency of $(strg["charge_efficiency"]) is out of the valid range (0.0. 1.0]" maxlog = + PS_MAX_LOG + end + if strg["discharge_efficiency"] < 0.0 + throw( + DataFormatError( + "storage unit $(strg["index"]) has a non-positive discharge efficiency of $(strg["discharge_efficiency"])", + ), + ) + end + if strg["discharge_efficiency"] <= 0.0 || strg["discharge_efficiency"] > 1.0 + @info "storage unit $(strg["index"]) discharge efficiency of $(strg["discharge_efficiency"]) is out of the valid range (0.0. 1.0]" maxlog = + PS_MAX_LOG + end + + if strg["p_loss"] > 0.0 && strg["energy"] <= 0.0 + @info "storage unit $(strg["index"]) has positive active power losses but zero initial energy. This can lead to model infeasiblity." maxlog = + PS_MAX_LOG + end + if strg["q_loss"] > 0.0 && strg["energy"] <= 0.0 + @info "storage unit $(strg["index"]) has positive reactive power losses but zero initial energy. This can lead to model infeasiblity." maxlog = + PS_MAX_LOG + end + end +end + +""" +checks that each switch has a reasonable parameters +""" +function check_switch_parameters(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_switch_parameters does not yet support multinetwork data") + end + + for (i, switch) in data["switch"] + if switch["state"] <= 0.0 && + (!isapprox(switch["psw"], 0.0) || !isapprox(switch["qsw"], 0.0)) + @info "switch $(switch["index"]) is open with non-zero power values $(switch["psw"]), $(switch["qsw"])" maxlog = + PS_MAX_LOG + end + if haskey(switch, "thermal_rating") && switch["thermal_rating"] < 0.0 + throw( + DataFormatError( + "switch $(switch["index"]) has a non-positive thermal_rating $(switch["thermal_rating"])", + ), + ) + end + if haskey(switch, "current_rating") && switch["current_rating"] < 0.0 + throw( + DataFormatError( + "switch $(switch["index"]) has a non-positive current_rating $(switch["current_rating"])", + ), + ) + end + end +end + +"checks that parameters for dc lines are reasonable" +function correct_dcline_limits!(data::Dict{String, Any}) + if ismultinetwork(data) + error("check_dcline_limits does not yet support multinetwork data") + end + + @assert("per_unit" in keys(data) && data["per_unit"]) + mva_base = data["baseMVA"] + + modified = Set{Int}() + + for c in 1:get(data, "conductors", 1) + cnd_str = haskey(data, "conductors") ? ", conductor $(c)" : "" + for (i, dcline) in data["dcline"] + if dcline["loss0"][c] < 0.0 + new_rate = 0.0 + @info "this code only supports positive loss0 values, changing the value on dcline $(dcline["index"])$(cnd_str) from $(mva_base*dcline["loss0"][c]) to $(mva_base*new_rate)" maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + dcline["loss0"][c] = new_rate + else + dcline["loss0"] = new_rate + end + push!(modified, dcline["index"]) + end + + if dcline["loss0"][c] >= + dcline["pmaxf"][c] * (1 - dcline["loss1"][c]) + dcline["pmaxt"][c] + new_rate = 0.0 + @info "this code only supports loss0 values which are consistent with the line flow bounds, changing the value on dcline $(dcline["index"])$(cnd_str) from $(mva_base*dcline["loss0"][c]) to $(mva_base*new_rate)" maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + dcline["loss0"][c] = new_rate + else + dcline["loss0"] = new_rate + end + push!(modified, dcline["index"]) + end + + if dcline["loss1"][c] < 0.0 + new_rate = 0.0 + @info "this code only supports positive loss1 values, changing the value on dcline $(dcline["index"])$(cnd_str) from $(dcline["loss1"][c]) to $(new_rate)" maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + dcline["loss1"][c] = new_rate + else + dcline["loss1"] = new_rate + end + push!(modified, dcline["index"]) + end + + if dcline["loss1"][c] >= 1.0 + new_rate = 0.0 + @info "this code only supports loss1 values < 1, changing the value on dcline $(dcline["index"])$(cnd_str) from $(dcline["loss1"][c]) to $(new_rate)" maxlog = + PS_MAX_LOG + if haskey(data, "conductors") + dcline["loss1"][c] = new_rate + else + dcline["loss1"] = new_rate + end + push!(modified, dcline["index"]) + end + + if dcline["pmint"][c] < 0.0 && dcline["loss1"][c] > 0.0 + #new_rate = 0.0 + @info "the dc line model is not meant to be used bi-directionally when loss1 > 0, be careful interpreting the results as the dc line losses can now be negative. change loss1 to 0 to avoid this warning" maxlog = + PS_MAX_LOG + #dcline["loss0"] = new_rate + end + end + end + + return modified +end + +"throws warnings if generator and dc line voltage setpoints are not consistent with the bus voltage setpoint" +function check_voltage_setpoints(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_voltage_setpoints does not yet support multinetwork data") + end + + for c in 1:get(data, "conductors", 1) + cnd_str = haskey(data, "conductors") ? "conductor $(c) " : "" + for (i, gen) in data["gen"] + bus_id = gen["gen_bus"] + bus = data["bus"][bus_id] + if gen["vg"][c] != bus["vm"][c] + @info "the $(cnd_str)voltage setpoint on generator $(i) does not match the value at bus $(bus_id)" maxlog = + PS_MAX_LOG + end + end + + for (i, dcline) in data["dcline"] + bus_fr_id = dcline["f_bus"] + bus_to_id = dcline["t_bus"] + + bus_fr = data["bus"][bus_fr_id] + bus_to = data["bus"][bus_to_id] + + if dcline["vf"][c] != bus_fr["vm"][c] + @info( + "the $(cnd_str)from bus voltage setpoint on dc line $(i) does not match the value at bus $(bus_fr_id)" + ) + end + + if dcline["vt"][c] != bus_to["vm"][c] + @info( + "the $(cnd_str)to bus voltage setpoint on dc line $(i) does not match the value at bus $(bus_to_id)" + ) + end + end + end +end + +"throws warnings if cost functions are malformed" +function correct_cost_functions!(data::Dict{String, <:Any}) + if ismultinetwork(data) + error("check_cost_functions does not yet support multinetwork data") + end + + modified_gen = Set{Int}() + for (i, gen) in data["gen"] + if _correct_cost_function!(i, gen, "generator") + push!(modified_gen, gen["index"]) + end + end + + modified_dcline = Set{Int}() + for (i, dcline) in data["dcline"] + if _correct_cost_function!(i, dcline, "dcline") + push!(modified_dcline, dcline["index"]) + end + end + + return (modified_gen, modified_dcline) +end + +"" +function _correct_cost_function!(id, comp, type_name) + modified = false + + if "model" in keys(comp) && "cost" in keys(comp) + if comp["model"] == 1 + if length(comp["cost"]) != 2 * comp["ncost"] + error( + "ncost of $(comp["ncost"]) not consistent with $(length(comp["cost"])) cost values on $(type_name) $(id)", + ) + end + if length(comp["cost"]) < 4 + error( + "cost includes $(comp["ncost"]) points, but at least two points are required on $(type_name) $(id)", + ) + end + + modified = _remove_pwl_cost_duplicates!(id, comp, type_name) + + for i in 3:2:length(comp["cost"]) + if comp["cost"][i - 2] >= comp["cost"][i] + error("non-increasing x values in pwl cost model on $(type_name) $(id)") + end + end + if "pmin" in keys(comp) && "pmax" in keys(comp) + pmin = sum(comp["pmin"]) # sum supports multi-conductor case + pmax = sum(comp["pmax"]) + for i in 3:2:length(comp["cost"]) + if comp["cost"][i] < pmin || comp["cost"][i] > pmax + @info( + "pwl x value $(comp["cost"][i]) is outside the bounds $(pmin)-$(pmax) on $(type_name) $(id)" + ) + end + end + end + modified |= _simplify_pwl_cost!(id, comp, type_name) + elseif comp["model"] == 2 + if length(comp["cost"]) != comp["ncost"] + error( + "ncost of $(comp["ncost"]) not consistent with $(length(comp["cost"])) cost values on $(type_name) $(id)", + ) + end + else + @info "Unknown cost model of type $(comp["model"]) on $(type_name) $(id)" maxlog = + PS_MAX_LOG + end + end + + return modified +end + +"checks that each point in the a pwl function is unique, simplifies the function if duplicates appear" +function _remove_pwl_cost_duplicates!(id, comp, type_name, tolerance = 1e-2) + @assert comp["model"] == 1 + + unique_costs = Float64[comp["cost"][1], comp["cost"][2]] + for i in 3:2:length(comp["cost"]) + x1 = unique_costs[end - 1] + y1 = unique_costs[end] + x2 = comp["cost"][i + 0] + y2 = comp["cost"][i + 1] + if !(isapprox(x1, x2) && isapprox(y1, y2)) + push!(unique_costs, x2) + push!(unique_costs, y2) + end + end + + # in the event that all of the given points are the same + # this code ensures that at least two of the points remain + if length(unique_costs) <= 2 + push!(unique_costs, comp["cost"][end - 1]) + push!(unique_costs, comp["cost"][end]) + end + + if length(unique_costs) < length(comp["cost"]) + @info "removing duplicate points from pwl cost on $(type_name) $(id), $(comp["cost"]) -> $(unique_costs)" maxlog = + PS_MAX_LOG + comp["cost"] = unique_costs + comp["ncost"] = length(unique_costs) / 2 + return true + end + + return false +end + +"checks the slope of each segment in a pwl function, simplifies the function if the slope changes is below a tolerance" +function _simplify_pwl_cost!(id, comp, type_name, tolerance = 1e-2) + @assert comp["model"] == 1 + + slopes = Float64[] + smpl_cost = Float64[] + prev_slope = nothing + + x2, y2 = 0.0, 0.0 + + for i in 3:2:length(comp["cost"]) + x1 = comp["cost"][i - 2] + y1 = comp["cost"][i - 1] + x2 = comp["cost"][i - 0] + y2 = comp["cost"][i + 1] + + m = (y2 - y1) / (x2 - x1) + + if prev_slope === nothing || (abs(prev_slope - m) > tolerance) + push!(smpl_cost, x1) + push!(smpl_cost, y1) + prev_slope = m + end + + push!(slopes, m) + end + + push!(smpl_cost, x2) + push!(smpl_cost, y2) + + if length(smpl_cost) < length(comp["cost"]) + @info "simplifying pwl cost on $(type_name) $(id), $(comp["cost"]) -> $(smpl_cost)" maxlog = + PS_MAX_LOG + comp["cost"] = smpl_cost + comp["ncost"] = length(smpl_cost) / 2 + return true + end + return false +end + +"trims zeros from higher order cost terms" +function simplify_cost_terms!(data::Dict{String, <:Any}) + if ismultinetwork(data) + networks = data["nw"] + else + networks = [("0", data)] + end + + modified_gen = Set{Int}() + modified_dcline = Set{Int}() + + for (i, network) in networks + if haskey(network, "gen") + for (i, gen) in network["gen"] + if haskey(gen, "model") && gen["model"] == 2 + ncost = length(gen["cost"]) + for j in 1:ncost + if gen["cost"][1] == 0.0 + gen["cost"] = gen["cost"][2:end] + else + break + end + end + if length(gen["cost"]) != ncost + gen["ncost"] = length(gen["cost"]) + @info "removing $(ncost - gen["ncost"]) cost terms from generator $(i): $(gen["cost"])" maxlog = + PS_MAX_LOG + push!(modified_gen, gen["index"]) + end + end + end + end + + if haskey(network, "dcline") + for (i, dcline) in network["dcline"] + if haskey(dcline, "model") && dcline["model"] == 2 + ncost = length(dcline["cost"]) + for j in 1:ncost + if dcline["cost"][1] == 0.0 + dcline["cost"] = dcline["cost"][2:end] + else + break + end + end + if length(dcline["cost"]) != ncost + dcline["ncost"] = length(dcline["cost"]) + @info "removing $(ncost - dcline["ncost"]) cost terms from dcline $(i): $(dcline["cost"])" maxlog = + PS_MAX_LOG + push!(modified_dcline, dcline["index"]) + end + end + end + end + end + + return (modified_gen, modified_dcline) +end + +"ensures all polynomial costs functions have the same number of terms" +function standardize_cost_terms!(data::Dict{String, <:Any}; order = -1) + comp_max_order = 1 + + if ismultinetwork(data) + networks = data["nw"] + else + networks = [("0", data)] + end + + for (i, network) in networks + if haskey(network, "gen") + for (i, gen) in network["gen"] + if haskey(gen, "model") && gen["model"] == 2 + max_nonzero_index = 1 + for i in 1:length(gen["cost"]) + max_nonzero_index = i + if gen["cost"][i] != 0.0 + break + end + end + + max_oder = length(gen["cost"]) - max_nonzero_index + 1 + + comp_max_order = max(comp_max_order, max_oder) + end + end + end + + if haskey(network, "dcline") + for (i, dcline) in network["dcline"] + if haskey(dcline, "model") && dcline["model"] == 2 + max_nonzero_index = 1 + for i in 1:length(dcline["cost"]) + max_nonzero_index = i + if dcline["cost"][i] != 0.0 + break + end + end + + max_oder = length(dcline["cost"]) - max_nonzero_index + 1 + + comp_max_order = max(comp_max_order, max_oder) + end + end + end + end + + if comp_max_order <= order + 1 + comp_max_order = order + 1 + else + if order != -1 # if not the default + @info( + "a standard cost order of $(order) was requested but the given data requires an order of at least $(comp_max_order-1)" + ) + end + end + + for (i, network) in networks + if haskey(network, "gen") + _standardize_cost_terms!(network["gen"], comp_max_order, "generator") + end + if haskey(network, "dcline") + _standardize_cost_terms!(network["dcline"], comp_max_order, "dcline") + end + end +end + +"ensures all polynomial costs functions have at exactly comp_order terms" +function _standardize_cost_terms!( + components::Dict{String, <:Any}, + comp_order::Int, + cost_comp_name::String, +) + modified = Set{Int}() + for (i, comp) in components + if haskey(comp, "model") && comp["model"] == 2 && length(comp["cost"]) != comp_order + std_cost = [0.0 for i in 1:comp_order] + current_cost = reverse(comp["cost"]) + #println("gen cost: $(comp["cost"])") + for i in 1:min(comp_order, length(current_cost)) + std_cost[i] = current_cost[i] + end + comp["cost"] = reverse(std_cost) + comp["ncost"] = comp_order + #println("std gen cost: $(comp["cost"])") + + @info "Updated $(cost_comp_name) $(comp["index"]) cost function with order $(length(current_cost)) to a function of order $(comp_order): $(comp["cost"])" maxlog = + PS_MAX_LOG + push!(modified, comp["index"]) + end + end + return modified +end + +""" +finds active network buses and branches that are not necessary for the +computation and sets their status to off. + +Works on a PowerModels data dict, so that a it can be used without a GenericPowerModel object + +Warning: this implementation has quadratic complexity, in the worst case +""" +function propagate_topology_status!(data::Dict{String, <:Any}) + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _propagate_topology_status!(nw_data) + end + else + _propagate_topology_status!(data) + end +end + +"" +function _propagate_topology_status!(data::Dict{String, <:Any}) + buses = Dict(bus["bus_i"] => bus for (i, bus) in data["bus"]) + + for (i, load) in data["load"] + if load["status"] != 0 && all(load["pd"] .== 0.0) && all(load["qd"] .== 0.0) + @info("deactivating load $(load["index"]) due to zero pd and qd") + load["status"] = 0 + end + end + + for (i, shunt) in data["shunt"] + if shunt["status"] != 0 && all(shunt["gs"] .== 0.0) && all(shunt["bs"] .== 0.0) + @info("deactivating shunt $(shunt["index"]) due to zero gs and bs") + shunt["status"] = 0 + end + end + + # compute what active components are incident to each bus + incident_load = bus_load_lookup(data["load"], data["bus"]) + incident_active_load = Dict() + for (i, load_list) in incident_load + incident_active_load[i] = [load for load in load_list if load["status"] != 0] + end + + incident_shunt = bus_shunt_lookup(data["shunt"], data["bus"]) + incident_active_shunt = Dict() + for (i, shunt_list) in incident_shunt + incident_active_shunt[i] = [shunt for shunt in shunt_list if shunt["status"] != 0] + end + + incident_gen = bus_gen_lookup(data["gen"], data["bus"]) + incident_active_gen = Dict() + for (i, gen_list) in incident_gen + incident_active_gen[i] = [gen for gen in gen_list if gen["gen_status"] != 0] + end + + incident_strg = bus_storage_lookup(data["storage"], data["bus"]) + incident_active_strg = Dict() + for (i, strg_list) in incident_strg + incident_active_strg[i] = [strg for strg in strg_list if strg["status"] != 0] + end + + incident_branch = Dict(bus["bus_i"] => [] for (i, bus) in data["bus"]) + for (i, branch) in data["branch"] + push!(incident_branch[branch["f_bus"]], branch) + push!(incident_branch[branch["t_bus"]], branch) + end + + incident_dcline = Dict(bus["bus_i"] => [] for (i, bus) in data["bus"]) + for (i, dcline) in data["dcline"] + push!(incident_dcline[dcline["f_bus"]], dcline) + push!(incident_dcline[dcline["t_bus"]], dcline) + end + + incident_switch = Dict(bus["bus_i"] => [] for (i, bus) in data["bus"]) + for (i, switch) in data["switch"] + push!(incident_switch[switch["f_bus"]], switch) + push!(incident_switch[switch["t_bus"]], switch) + end + + revised = false + + for (i, branch) in data["branch"] + if branch["br_status"] != 0 + f_bus = buses[branch["f_bus"]] + t_bus = buses[branch["t_bus"]] + + if f_bus["bus_type"] == 4 || t_bus["bus_type"] == 4 + @info "deactivating branch $(i):($(branch["f_bus"]),$(branch["t_bus"])) due to connecting bus status" maxlog = + PS_MAX_LOG + branch["br_status"] = 0 + revised = true + end + end + end + + for (i, dcline) in data["dcline"] + if dcline["br_status"] != 0 + f_bus = buses[dcline["f_bus"]] + t_bus = buses[dcline["t_bus"]] + + if f_bus["bus_type"] == 4 || t_bus["bus_type"] == 4 + @info "deactivating dcline $(i):($(dcline["f_bus"]),$(dcline["t_bus"])) due to connecting bus status" maxlog = + PS_MAX_LOG + dcline["br_status"] = 0 + revised = true + end + end + end + + for (i, switch) in data["switch"] + if switch["status"] != 0 + f_bus = buses[switch["f_bus"]] + t_bus = buses[switch["t_bus"]] + + if f_bus["bus_type"] == 4 || t_bus["bus_type"] == 4 + @info "deactivating switch $(i):($(switch["f_bus"]),$(switch["t_bus"])) due to connecting bus status" maxlog = + PS_MAX_LOG + switch["status"] = 0 + revised = true + end + end + end + + for (i, bus) in buses + if bus["bus_type"] == 4 + for load in incident_active_load[i] + if load["status"] != 0 + @info "deactivating load $(load["index"]) due to inactive bus $(i)" maxlog = + PS_MAX_LOG + load["status"] = 0 + revised = true + end + end + + for shunt in incident_active_shunt[i] + if shunt["status"] != 0 + @info "deactivating shunt $(shunt["index"]) due to inactive bus $(i)" maxlog = + PS_MAX_LOG + shunt["status"] = 0 + revised = true + end + end + + for gen in incident_active_gen[i] + if gen["gen_status"] != 0 + @info "deactivating generator $(gen["index"]) due to inactive bus $(i)" maxlog = + PS_MAX_LOG + gen["gen_status"] = 0 + revised = true + end + end + + for strg in incident_active_strg[i] + if strg["status"] != 0 + @info "deactivating storage $(strg["index"]) due to inactive bus $(i)" maxlog = + PS_MAX_LOG + strg["status"] = 0 + revised = true + end + end + end + end + + return revised +end + +""" +removes buses with single branch connections and without any other attached +components. Also removes connected components without suffuceint generation +or loads. + +also deactivates 0 valued loads and shunts. +""" +function deactivate_isolated_components!(data::Dict{String, <:Any}) + revised = false + pm_data = get_pm_data(data) + + if _IM.ismultinetwork(pm_data) + for (i, pm_nw_data) in pm_data["nw"] + revised |= _deactivate_isolated_components!(pm_nw_data) + end + else + revised = _deactivate_isolated_components!(pm_data) + end + + return revised +end + +"" +function _deactivate_isolated_components!(data::Dict{String, <:Any}) + buses = Dict(bus["bus_i"] => bus for (i, bus) in data["bus"]) + + revised = false + + for (i, load) in data["load"] + if load["status"] != 0 && all(load["pd"] .== 0.0) && all(load["qd"] .== 0.0) + @info "deactivating load $(load["index"]) due to zero pd and qd" maxlog = + PS_MAX_LOG + load["status"] = 0 + revised = true + end + end + + for (i, shunt) in data["shunt"] + if shunt["status"] != 0 && all(shunt["gs"] .== 0.0) && all(shunt["bs"] .== 0.0) + @info "deactivating shunt $(shunt["index"]) due to zero gs and bs" maxlog = + PS_MAX_LOG + shunt["status"] = 0 + revised = true + end + end + + # compute what active components are incident to each bus + incident_load = bus_load_lookup(data["load"], data["bus"]) + incident_active_load = Dict() + for (i, load_list) in incident_load + incident_active_load[i] = [load for load in load_list if load["status"] != 0] + end + + incident_shunt = bus_shunt_lookup(data["shunt"], data["bus"]) + incident_active_shunt = Dict() + for (i, shunt_list) in incident_shunt + incident_active_shunt[i] = [shunt for shunt in shunt_list if shunt["status"] != 0] + end + + incident_gen = bus_gen_lookup(data["gen"], data["bus"]) + incident_active_gen = Dict() + for (i, gen_list) in incident_gen + incident_active_gen[i] = [gen for gen in gen_list if gen["gen_status"] != 0] + end + + incident_strg = bus_storage_lookup(data["storage"], data["bus"]) + incident_active_strg = Dict() + for (i, strg_list) in incident_strg + incident_active_strg[i] = [strg for strg in strg_list if strg["status"] != 0] + end + + incident_branch = Dict(bus["bus_i"] => [] for (i, bus) in data["bus"]) + for (i, branch) in data["branch"] + push!(incident_branch[branch["f_bus"]], branch) + push!(incident_branch[branch["t_bus"]], branch) + end + + incident_dcline = Dict(bus["bus_i"] => [] for (i, bus) in data["bus"]) + for (i, dcline) in data["dcline"] + push!(incident_dcline[dcline["f_bus"]], dcline) + push!(incident_dcline[dcline["t_bus"]], dcline) + end + + incident_switch = Dict(bus["bus_i"] => [] for (i, bus) in data["bus"]) + for (i, switch) in data["switch"] + push!(incident_switch[switch["f_bus"]], switch) + push!(incident_switch[switch["t_bus"]], switch) + end + + changed = true + while changed + changed = false + + for (i, bus) in buses + if bus["bus_type"] != 4 + incident_active_edge = 0 + if length(incident_branch[i]) + + length(incident_dcline[i]) + + length(incident_switch[i]) > 0 + incident_branch_count = + sum([0; [branch["br_status"] for branch in incident_branch[i]]]) + incident_dcline_count = + sum([0; [dcline["br_status"] for dcline in incident_dcline[i]]]) + incident_switch_count = + sum([0; [switch["status"] for switch in incident_switch[i]]]) + incident_active_edge = + incident_branch_count + + incident_dcline_count + + incident_switch_count + end + + if incident_active_edge == 1 && + length(incident_active_gen[i]) == 0 && + length(incident_active_load[i]) == 0 && + length(incident_active_shunt[i]) == 0 && + length(incident_active_strg[i]) == 0 + @info "deactivating bus $(i) due to dangling bus without generation, load, or storage" maxlog = + PS_MAX_LOG + bus["bus_type"] = 4 + revised = true + changed = true + end + end + end + + if changed + for (i, branch) in data["branch"] + if branch["br_status"] != 0 + f_bus = buses[branch["f_bus"]] + t_bus = buses[branch["t_bus"]] + + if f_bus["bus_type"] == 4 || t_bus["bus_type"] == 4 + @info "deactivating branch $(i):($(branch["f_bus"]),$(branch["t_bus"])) due to connecting bus status" maxlog = + PS_MAX_LOG + branch["br_status"] = 0 + end + end + end + + for (i, dcline) in data["dcline"] + if dcline["br_status"] != 0 + f_bus = buses[dcline["f_bus"]] + t_bus = buses[dcline["t_bus"]] + + if f_bus["bus_type"] == 4 || t_bus["bus_type"] == 4 + @info "deactivating dcline $(i):($(dcline["f_bus"]),$(dcline["t_bus"])) due to connecting bus status" maxlog = + PS_MAX_LOG + dcline["br_status"] = 0 + end + end + end + + for (i, switch) in data["switch"] + if switch["status"] != 0 + f_bus = buses[switch["f_bus"]] + t_bus = buses[switch["t_bus"]] + + if f_bus["bus_type"] == 4 || t_bus["bus_type"] == 4 + @info "deactivating switch $(i):($(switch["f_bus"]),$(switch["t_bus"])) due to connecting bus status" maxlog = + PS_MAX_LOG + switch["status"] = 0 + end + end + end + end + end + + ccs = calc_connected_components(data) + + for cc in ccs + cc_active_loads = [0] + cc_active_shunts = [0] + cc_active_gens = [0] + cc_active_strg = [0] + + for i in cc + cc_active_loads = push!(cc_active_loads, length(incident_active_load[i])) + cc_active_shunts = push!(cc_active_shunts, length(incident_active_shunt[i])) + cc_active_gens = push!(cc_active_gens, length(incident_active_gen[i])) + end + + active_load_count = sum(cc_active_loads) + active_shunt_count = sum(cc_active_shunts) + active_gen_count = sum(cc_active_gens) + + if (active_load_count == 0 && active_shunt_count == 0 && active_strg_count == 0) || + active_gen_count == 0 + @info "deactivating connected component $(cc) due to isolation without generation and load" maxlog = + PS_MAX_LOG + for i in cc + buses[i]["bus_type"] = 4 + end + revised = true + end + end + + return revised +end + +""" +attempts to deactive components that are not needed in the network by repeated +calls to `propagate_topology_status!` and `deactivate_isolated_components!` + +warning: this implementation has quadratic complexity, in the worst case +""" +function simplify_network!(data::Dict{String, <:Any}) + revised = true + iteration = 0 + + while revised + iteration += 1 + revised = false + revised |= propagate_topology_status!(data) + revised |= deactivate_isolated_components!(data) + end + + @info "network simplification fixpoint reached in $(iteration) rounds" maxlog = + PS_MAX_LOG + return revised +end + +""" +determines the largest connected component of the network and turns everything else off +""" +function select_largest_component(data::Dict{String, Any}) + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _select_largest_component(nw_data) + end + else + _select_largest_component(data) + end +end + +"" +function _select_largest_component!(data::Dict{String, <:Any}) + ccs = calc_connected_components(data) + @info "found $(length(ccs)) components" maxlog = PS_MAX_LOG + + if length(ccs) > 1 + ccs_order = sort(collect(ccs); by = length) + largest_cc = ccs_order[end] + + @info "largest component has $(length(largest_cc)) buses" maxlog = PS_MAX_LOG + + for (i, bus) in data["bus"] + if bus["bus_type"] != 4 && !(bus["index"] in largest_cc) + bus["bus_type"] = 4 + @info "deactivating bus $(i) due to small connected component" maxlog = + PS_MAX_LOG + end + end + + correct_reference_buses!(data) + end +end + +""" +checks that each connected components has a reference bus, if not, adds one +""" +function check_reference_buses(data::Dict{String, Any}) + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _correct_reference_buses!(nw_data) + end + else + _correct_reference_buses!(data) + end +end + +"" +function _correct_reference_buses!(data::Dict{String, <:Any}) + bus_lookup = Dict(bus["bus_i"] => bus for (i, bus) in data["bus"]) + bus_gen = bus_gen_lookup(data["gen"], data["bus"]) + + ccs = calc_connected_components(data) + ccs_order = sort(collect(ccs); by = length) + + bus_to_cc = Dict() + for (i, cc) in enumerate(ccs_order) + for bus_i in cc + bus_to_cc[bus_i] = i + end + end + + cc_gens = Dict(i => Dict() for (i, cc) in enumerate(ccs_order)) + for (i, gen) in data["gen"] + bus_id = gen["gen_bus"] + if haskey(bus_to_cc, bus_id) + cc_id = bus_to_cc[bus_id] + cc_gens[cc_id][i] = gen + end + end + + for (i, cc) in enumerate(ccs_order) + correct_component_refrence_bus!(cc, bus_lookup, cc_gens[i]) + end +end + +""" +checks that a connected component has a reference bus, if not, tries to add one +""" +function correct_component_refrence_bus!(component_bus_ids, bus_lookup, component_gens) + refrence_buses = Set() + for bus_id in component_bus_ids + bus = bus_lookup[bus_id] + if bus["bus_type"] == 3 + push!(refrence_buses, bus_id) + end + end + + if length(refrence_buses) == 0 + @info("no reference bus found in connected component $(component_bus_ids)") + component_gens_active = + Dict(k => v for (k, v) in component_gens if v["gen_status"] != 0) + + if length(component_gens_active) > 0 + big_gen = _biggest_generator(component_gens_active) + gen_bus = bus_lookup[big_gen["gen_bus"]] + gen_bus["bus_type"] = 3 + @info( + "setting bus $(gen_bus["index"]) as reference bus in connected component $(component_bus_ids), based on generator $(big_gen["index"])" + ) + else + @info( + "no generators found in connected component $(component_bus_ids), try running propagate_topology_status" + ) + end + end +end + +"builds a lookup list of what generators are connected to a given bus" +function bus_gen_lookup(gen_data::Dict{String, <:Any}, bus_data::Dict{String, <:Any}) + bus_gen = Dict(bus["bus_i"] => [] for (i, bus) in bus_data) + for (i, gen) in gen_data + push!(bus_gen[gen["gen_bus"]], gen) + end + return bus_gen +end + +"builds a lookup list of what loads are connected to a given bus" +function bus_load_lookup(load_data::Dict{String, <:Any}, bus_data::Dict{String, <:Any}) + bus_load = Dict(bus["bus_i"] => [] for (i, bus) in bus_data) + for (i, load) in load_data + push!(bus_load[load["load_bus"]], load) + end + return bus_load +end + +"builds a lookup list of what shunts are connected to a given bus" +function bus_shunt_lookup(shunt_data::Dict{String, <:Any}, bus_data::Dict{String, <:Any}) + bus_shunt = Dict(bus["bus_i"] => [] for (i, bus) in bus_data) + for (i, shunt) in shunt_data + push!(bus_shunt[shunt["shunt_bus"]], shunt) + end + return bus_shunt +end + +"builds a lookup list of what storage is connected to a given bus" +function bus_storage_lookup( + storage_data::Dict{String, <:Any}, + bus_data::Dict{String, <:Any}, +) + bus_storage = Dict(bus["bus_i"] => [] for (i, bus) in bus_data) + for (i, storage) in storage_data + push!(bus_storage[storage["storage_bus"]], storage) + end + return bus_storage +end + +""" +computes the connected components of the network graph +returns a set of sets of bus ids, each set is a connected component +""" +function calc_connected_components( + pm_data::Dict{String, <:Any}; + edges = ["branch", "dcline", "switch"], +) + if ismultinetwork(pm_data) + error("connected_components does not yet support multinetwork data") + end + + active_bus = Dict(x for x in pm_data["bus"] if x.second["bus_type"] != 4) + active_bus_ids = Set{Int64}([bus["bus_i"] for (i, bus) in active_bus]) + + neighbors = Dict(i => Int[] for i in active_bus_ids) + for comp_type in edges + status_key = get(pm_component_status, comp_type, "status") + status_inactive = get(pm_component_status_inactive, comp_type, 0) + for edge in values(get(pm_data, comp_type, Dict())) + if get(edge, status_key, 1) != status_inactive && + edge["f_bus"] in active_bus_ids && + edge["t_bus"] in active_bus_ids + push!(neighbors[edge["f_bus"]], edge["t_bus"]) + push!(neighbors[edge["t_bus"]], edge["f_bus"]) + end + end + end + + component_lookup = Dict(i => Set{Int}([i]) for i in active_bus_ids) + touched = Set{Int64}() + + for i in active_bus_ids + if !(i in touched) + _cc_dfs(i, neighbors, component_lookup, touched) + end + end + + ccs = (Set(values(component_lookup))) + + return ccs +end + +""" +DFS on a graph +""" +function _cc_dfs(i, neighbors, component_lookup, touched) + push!(touched, i) + for j in neighbors[i] + if !(j in touched) + for k in component_lookup[j] + push!(component_lookup[i], k) + end + for k in component_lookup[j] + component_lookup[k] = component_lookup[i] + end + _cc_dfs(j, neighbors, component_lookup, touched) + end + end +end + +""" +given a network data dict and a mapping of current-bus-ids to new-bus-ids +modifies the data dict to reflect the proposed new bus ids. +""" +function update_bus_ids!( + data::Dict{String, <:Any}, + bus_id_map::Dict{Int, Int}; + injective = true, +) + data_it = ismultiinfrastructure(data) ? data["it"][pm_it_name] : data + + if _IM.ismultinetwork(data_it) && apply_to_subnetworks + for (nw, nw_data) in data_it["nw"] + _update_bus_ids!(nw_data, bus_id_map; injective = injective) + end + else + _update_bus_ids!(data_it, bus_id_map; injective = injective) + end +end + +function _update_bus_ids!( + data::Dict{String, <:Any}, + bus_id_map::Dict{Int, Int}; + injective = true, +) + # verify bus id map is injective + if injective + new_bus_ids = Set{Int}() + for (i, bus) in data["bus"] + new_id = get(bus_id_map, bus["index"], bus["index"]) + if !(new_id in new_bus_ids) + push!(new_bus_ids, new_id) + else + throw( + error( + "bus id mapping given to update_bus_ids has an id clash on new bus id $(new_id)", + ), + ) + end + end + end + + # start renumbering process + renumbered_bus_dict = Dict{String, Any}() + + for (i, bus) in data["bus"] + new_id = get(bus_id_map, bus["index"], bus["index"]) + bus["index"] = new_id + bus["bus_i"] = new_id + renumbered_bus_dict["$new_id"] = bus + end + + data["bus"] = renumbered_bus_dict + + # update bus numbering in dependent components + for (i, load) in data["load"] + load["load_bus"] = get(bus_id_map, load["load_bus"], load["load_bus"]) + end + + for (i, shunt) in data["shunt"] + shunt["shunt_bus"] = get(bus_id_map, shunt["shunt_bus"], shunt["shunt_bus"]) + end + + for (i, gen) in data["gen"] + gen["gen_bus"] = get(bus_id_map, gen["gen_bus"], gen["gen_bus"]) + end + + for (i, strg) in data["storage"] + strg["storage_bus"] = get(bus_id_map, strg["storage_bus"], strg["storage_bus"]) + end + + for (i, switch) in data["switch"] + switch["f_bus"] = get(bus_id_map, switch["f_bus"], switch["f_bus"]) + switch["t_bus"] = get(bus_id_map, switch["t_bus"], switch["t_bus"]) + end + + branches = [] + if haskey(data, "branch") + append!(branches, values(data["branch"])) + end + + if haskey(data, "ne_branch") + append!(branches, values(data["ne_branch"])) + end + + for branch in branches + branch["f_bus"] = get(bus_id_map, branch["f_bus"], branch["f_bus"]) + branch["t_bus"] = get(bus_id_map, branch["t_bus"], branch["t_bus"]) + end + + for (i, dcline) in data["dcline"] + dcline["f_bus"] = get(bus_id_map, dcline["f_bus"], dcline["f_bus"]) + dcline["t_bus"] = get(bus_id_map, dcline["t_bus"], dcline["t_bus"]) + end +end + +""" +given a network data dict merges buses that are connected by closed switches +converting the dataset into a pure bus-branch model. +""" +function resolve_swithces!(data::Dict{String, <:Any}) + if ismultinetwork(data) + for (i, nw_data) in data["nw"] + _resolve_swithces!(nw_data, mva_base) + end + else + _resolve_swithces!(data, mva_base) + end +end + +"" +function _resolve_swithces!(data::Dict{String, <:Any}) + if length(data["switch"]) <= 0 + return + end + + bus_sets = Dict{Int, Set{Int}}() + + switch_status_key = pm_component_status["switch"] + switch_status_value = pm_component_status_inactive["switch"] + + for (i, switch) in data["switch"] + if switch[switch_status_key] != switch_status_value && switch["state"] == 1 + if !haskey(bus_sets, switch["f_bus"]) + bus_sets[switch["f_bus"]] = Set{Int}([switch["f_bus"]]) + end + if !haskey(bus_sets, switch["t_bus"]) + bus_sets[switch["t_bus"]] = Set{Int}([switch["t_bus"]]) + end + + merged_set = + Set{Int}([bus_sets[switch["f_bus"]]..., bus_sets[switch["t_bus"]]...]) + bus_sets[switch["f_bus"]] = merged_set + bus_sets[switch["t_bus"]] = merged_set + end + end + + bus_id_map = Dict{Int, Int}() + for bus_set in Set(values(bus_sets)) + bus_min = minimum(bus_set) + @info "merged buses $(join(bus_set, ",")) in to bus $(bus_min) based on switch status" maxlog = + PS_MAX_LOG + for i in bus_set + if i != bus_min + bus_id_map[i] = bus_min + end + end + end + + update_bus_ids!(data, bus_id_map; injective = false) + + for (i, branch) in data["branch"] + if branch["f_bus"] == branch["t_bus"] + @warn "switch removal resulted in both sides of branch $(i) connect to bus $(branch["f_bus"]), deactivating branch" + branch[pm_component_status["branch"]] = pm_component_status_inactive["branch"] + end + end + + for (i, dcline) in data["dcline"] + if dcline["f_bus"] == dcline["t_bus"] + @warn "switch removal resulted in both sides of dcline $(i) connect to bus $(branch["f_bus"]), deactivating dcline" + branch[pm_component_status["dcline"]] = pm_component_status_inactive["dcline"] + end + end + + @info "removed $(length(data["switch"])) switch components" + data["switch"] = Dict{String, Any}() + return +end + +""" +Move gentype and genfuel fields to be subfields of gen +""" +function move_genfuel_and_gentype!(data::Dict{String, Any}) # added by PSY + ngen = length(data["gen"]) + + toplevkeys = ("genfuel", "gentype") + sublevkeys = ("fuel", "type") + for i in range(1; stop = length(toplevkeys)) + if haskey(data, toplevkeys[i]) + # check that lengths of category and generators match + if length(data[toplevkeys[i]]) != ngen + str = toplevkeys[i] + throw( + DataFormatError( + "length of $str does not equal the number of generators", + ), + ) + end + for (key, val) in data[toplevkeys[i]] + data["gen"][key][sublevkeys[i]] = val["col_1"] + end + delete!(data, toplevkeys[i]) + end + end +end diff --git a/src/pm_io/matpower.jl b/src/pm_io/matpower.jl new file mode 100644 index 0000000..7c7958e --- /dev/null +++ b/src/pm_io/matpower.jl @@ -0,0 +1,825 @@ +######################################################################### +# # +# This file provides functions for interfacing with Matpower data files # +# # +######################################################################### + +const MP_FIX_VOLTAGE_BUSES = [2, 3] + +"Parses the matpwer data from either a filename or an IO object" +function parse_matpower(io::IO; validate = true)::Dict + mp_data = _parse_matpower_string(read(io, String)) + pm_data = _matpower_to_powermodels!(mp_data) + if validate + correct_network_data!(pm_data) + end + return pm_data +end + +function parse_matpower(file::String; kwargs...)::Dict + mp_data = open(file) do io + parse_matpower(io; kwargs...) + end + return mp_data +end + +### Data and functions specific to Matpower format ### + +const _mp_data_names = [ + "mpc.version", + "mpc.baseMVA", + "mpc.bus", + "mpc.gen", + "mpc.branch", + "mpc.dcline", + "mpc.gencost", + "mpc.dclinecost", + "mpc.bus_name", + "mpc.storage", + "mpc.switch", +] + +const _mp_bus_columns = [ + ("bus_i", Int), + ("bus_type", Int), + ("pd", Float64), + ("qd", Float64), + ("gs", Float64), + ("bs", Float64), + ("area", Int), + ("vm", Float64), + ("va", Float64), + ("base_kv", Float64), + ("zone", Int), + ("vmax", Float64), + ("vmin", Float64), + ("lam_p", Float64), + ("lam_q", Float64), + ("mu_vmax", Float64), + ("mu_vmin", Float64), +] + +const _mp_bus_name_columns = [("name", Union{String, SubString{String}})] + +const _mp_gen_columns = [ + ("gen_bus", Int), + ("pg", Float64), + ("qg", Float64), + ("qmax", Float64), + ("qmin", Float64), + ("vg", Float64), + ("mbase", Float64), + ("gen_status", Int), + ("pmax", Float64), + ("pmin", Float64), + ("pc1", Float64), + ("pc2", Float64), + ("qc1min", Float64), + ("qc1max", Float64), + ("qc2min", Float64), + ("qc2max", Float64), + ("ramp_agc", Float64), + ("ramp_10", Float64), + ("ramp_30", Float64), + ("ramp_q", Float64), + ("apf", Float64), + ("mu_pmax", Float64), + ("mu_pmin", Float64), + ("mu_qmax", Float64), + ("mu_qmin", Float64), +] + +const _mp_branch_columns = [ + ("f_bus", Int), + ("t_bus", Int), + ("br_r", Float64), + ("br_x", Float64), + ("br_b", Float64), + ("rate_a", Float64), + ("rate_b", Float64), + ("rate_c", Float64), + ("tap", Float64), + ("shift", Float64), + ("br_status", Int), + ("angmin", Float64), + ("angmax", Float64), + ("pf", Float64), + ("qf", Float64), + ("pt", Float64), + ("qt", Float64), + ("mu_sf", Float64), + ("mu_st", Float64), + ("mu_angmin", Float64), + ("mu_angmax", Float64), +] + +const _mp_dcline_columns = [ + ("f_bus", Int), + ("t_bus", Int), + ("br_status", Int), + ("pf", Float64), + ("pt", Float64), + ("qf", Float64), + ("qt", Float64), + ("vf", Float64), + ("vt", Float64), + ("pmin", Float64), + ("pmax", Float64), + ("qminf", Float64), + ("qmaxf", Float64), + ("qmint", Float64), + ("qmaxt", Float64), + ("loss0", Float64), + ("loss1", Float64), + ("mu_pmin", Float64), + ("mu_pmax", Float64), + ("mu_qminf", Float64), + ("mu_qmaxf", Float64), + ("mu_qmint", Float64), + ("mu_qmaxt", Float64), +] + +const _mp_storage_columns = [ + ("storage_bus", Int), + ("ps", Float64), + ("qs", Float64), + ("energy", Float64), + ("energy_rating", Float64), + ("charge_rating", Float64), + ("discharge_rating", Float64), + ("charge_efficiency", Float64), + ("discharge_efficiency", Float64), + ("thermal_rating", Float64), + ("qmin", Float64), + ("qmax", Float64), + ("r", Float64), + ("x", Float64), + ("p_loss", Float64), + ("q_loss", Float64), + ("status", Int), +] + +const _mp_switch_columns = [ + ("f_bus", Int), + ("t_bus", Int), + ("psw", Float64), + ("qsw", Float64), + ("state", Int), + ("thermal_rating", Float64), + ("status", Int), +] + +"" +function _parse_matpower_string(data_string::String) + matlab_data, func_name, colnames = parse_matlab_string(data_string; extended = true) + + case = Dict{String, Any}() + + if func_name !== nothing + case["name"] = func_name + else + @info( + string( + "no case name found in matpower file. The file seems to be missing \"function mpc = ...\"", + ) + ) + case["name"] = "no_name_found" + end + + case["source_type"] = "matpower" + if haskey(matlab_data, "mpc.version") + case["source_version"] = matlab_data["mpc.version"] + else + @info( + string( + "no case version found in matpower file. The file seems to be missing \"mpc.version = ...\"", + ) + ) + case["source_version"] = "0.0.0+" + end + + if haskey(matlab_data, "mpc.baseMVA") + case["baseMVA"] = Float64(matlab_data["mpc.baseMVA"]) + else + @info( + string( + "no baseMVA found in matpower file. The file seems to be missing \"mpc.baseMVA = ...\"", + ) + ) + case["baseMVA"] = 1.0 + end + + if haskey(matlab_data, "mpc.bus") + buses = [] + pv_bus_lookup = Dict{Int, Any}() + for bus_row in matlab_data["mpc.bus"] + bus_data = row_to_typed_dict(bus_row, _mp_bus_columns) + bus_data["index"] = check_type(Int, bus_row[1]) + bus_data["source_id"] = ["bus", bus_data["index"]] + push!(buses, bus_data) + if bus_data["bus_type"] ∈ MP_FIX_VOLTAGE_BUSES + pv_bus_lookup[bus_data["index"]] = bus_data + end + end + case["bus"] = buses + else + error( + string( + "no bus table found in matpower file. The file seems to be missing \"mpc.bus = [...];\"", + ), + ) + end + + if haskey(matlab_data, "mpc.gen") + gens = [] + corrected_pv_bus_vm = Dict{Int, Float64}() + for (i, gen_row) in enumerate(matlab_data["mpc.gen"]) + gen_data = row_to_typed_dict(gen_row, _mp_gen_columns) + bus_data = get(pv_bus_lookup, gen_data["gen_bus"], nothing) + if bus_data !== nothing + if bus_data["bus_type"] ∈ MP_FIX_VOLTAGE_BUSES && + bus_data["vm"] != gen_data["vg"] + @info "Correcting vm in bus $(gen_data["gen_bus"]) to $(gen_data["vg"]) to match generator set-point" + if gen_data["gen_bus"] ∈ keys(corrected_pv_bus_vm) + if corrected_pv_bus_vm[gen_data["gen_bus"]] != gen_data["vg"] + @error( + "Generator voltage set-points for bus $(gen_data["gen_bus"]) are inconsistent. This can lead to unexpected results" + ) + end + else + bus_data["vm"] = gen_data["vg"] + corrected_pv_bus_vm[gen_data["gen_bus"]] = gen_data["vg"] + end + end + end + gen_data["index"] = i + gen_data["source_id"] = ["gen", i] + push!(gens, gen_data) + end + case["gen"] = gens + else + error( + string( + "no gen table found in matpower file. The file seems to be missing \"mpc.gen = [...];\"", + ), + ) + end + + if haskey(matlab_data, "mpc.branch") + branches = [] + for (i, branch_row) in enumerate(matlab_data["mpc.branch"]) + branch_data = row_to_typed_dict(branch_row, _mp_branch_columns) + branch_data["index"] = i + branch_data["source_id"] = ["branch", i] + push!(branches, branch_data) + end + case["branch"] = branches + else + error( + string( + "no branch table found in matpower file. The file seems to be missing \"mpc.branch = [...];\"", + ), + ) + end + + if haskey(matlab_data, "mpc.dcline") + dclines = [] + for (i, dcline_row) in enumerate(matlab_data["mpc.dcline"]) + dcline_data = row_to_typed_dict(dcline_row, _mp_dcline_columns) + dcline_data["index"] = i + dcline_data["source_id"] = ["dcline", i] + push!(dclines, dcline_data) + end + case["dcline"] = dclines + end + + if haskey(matlab_data, "mpc.storage") + storage = [] + for (i, storage_row) in enumerate(matlab_data["mpc.storage"]) + storage_data = row_to_typed_dict(storage_row, _mp_storage_columns) + storage_data["index"] = i + storage_data["source_id"] = ["storage", i] + push!(storage, storage_data) + end + case["storage"] = storage + end + + if haskey(matlab_data, "mpc.switch") + switch = [] + for (i, switch_row) in enumerate(matlab_data["mpc.switch"]) + switch_data = row_to_typed_dict(switch_row, _mp_switch_columns) + switch_data["index"] = i + switch_data["source_id"] = ["switch", i] + push!(switch, switch_data) + end + case["switch"] = switch + end + + if haskey(matlab_data, "mpc.bus_name") + bus_names = [] + for (i, bus_name_row) in enumerate(matlab_data["mpc.bus_name"]) + bus_name_data = row_to_typed_dict(bus_name_row, _mp_bus_name_columns) + bus_name_data["index"] = i + bus_name_data["source_id"] = ["bus_name", i] + push!(bus_names, bus_name_data) + end + case["bus_name"] = bus_names + + if length(case["bus_name"]) != length(case["bus"]) + error( + "incorrect Matpower file, the number of bus names ($(length(case["bus_name"]))) is inconsistent with the number of buses ($(length(case["bus"]))).\n", + ) + end + end + + if haskey(matlab_data, "mpc.gencost") + gencost = [] + for (i, gencost_row) in enumerate(matlab_data["mpc.gencost"]) + gencost_data = _mp_cost_data(gencost_row) + gencost_data["index"] = i + gencost_data["source_id"] = ["gencost", i] + push!(gencost, gencost_data) + end + case["gencost"] = gencost + + if length(case["gencost"]) != length(case["gen"]) && + length(case["gencost"]) != 2 * length(case["gen"]) + error( + "incorrect Matpower file, the number of generator cost functions ($(length(case["gencost"]))) is inconsistent with the number of generators ($(length(case["gen"]))).\n", + ) + end + end + + if haskey(matlab_data, "mpc.dclinecost") + dclinecosts = [] + for (i, dclinecost_row) in enumerate(matlab_data["mpc.dclinecost"]) + dclinecost_data = _mp_cost_data(dclinecost_row) + dclinecost_data["index"] = i + dclinecost_data["source_id"] = ["dclinecost", i] + push!(dclinecosts, dclinecost_data) + end + case["dclinecost"] = dclinecosts + + if length(case["dclinecost"]) != length(case["dcline"]) + error( + "incorrect Matpower file, the number of dcline cost functions ($(length(case["dclinecost"]))) is inconsistent with the number of dclines ($(length(case["dcline"]))).\n", + ) + end + end + + for k in keys(matlab_data) + if !in(k, _mp_data_names) && startswith(k, "mpc.") + case_name = k[5:length(k)] + value = matlab_data[k] + if isa(value, Array) + column_names = [] + if haskey(colnames, k) + column_names = colnames[k] + end + tbl = [] + for (i, row) in enumerate(matlab_data[k]) + row_data = row_to_dict(row, column_names) + row_data["index"] = i + row_data["source_id"] = [case_name, i] + push!(tbl, row_data) + end + case[case_name] = tbl + @info( + "extending matpower format with data: $(case_name) $(length(tbl))x$(length(tbl[1])-1)" + ) + else + case[case_name] = value + @info("extending matpower format with constant data: $(case_name)") + end + end + end + + return case +end + +"" +function _mp_cost_data(cost_row) + ncost = check_type(Int, cost_row[4]) + model = check_type(Int, cost_row[1]) + if model == 1 + nr_parameters = ncost * 2 + elseif model == 2 + nr_parameters = ncost + end + + cost_data = Dict( + "model" => model, + "startup" => check_type(Float64, cost_row[2]), + "shutdown" => check_type(Float64, cost_row[3]), + "ncost" => ncost, + "cost" => [check_type(Float64, x) for x in cost_row[5:(5 + nr_parameters - 1)]], + ) + + #= + # skip this literal interpretation, as its hard to invert + cost_values = [check_type(Float64, x) for x in cost_row[5:length(cost_row)]] + if cost_data["model"] == 1: + if length(cost_values)%2 != 0 + error("incorrect matpower file, odd number of pwl cost function values") + end + for i in 0:(length(cost_values)/2-1) + p_idx = 1+2*i + f_idx = 2+2*i + cost_data["p_$(i)"] = cost_values[p_idx] + cost_data["f_$(i)"] = cost_values[f_idx] + end + else: + for (i,v) in enumerate(cost_values) + cost_data["c_$(length(cost_values)+1-i)"] = v + end + =# + return cost_data +end + +### Data and functions specific to PowerModels format ### + +""" +Converts a Matpower dict into a PowerModels dict +""" +function _matpower_to_powermodels!(mp_data::Dict{String, <:Any}) + pm_data = mp_data + + # required default values + if !haskey(pm_data, "dcline") + pm_data["dcline"] = [] + end + if !haskey(pm_data, "gencost") + pm_data["gencost"] = [] + end + if !haskey(pm_data, "dclinecost") + pm_data["dclinecost"] = [] + end + if !haskey(pm_data, "storage") + pm_data["storage"] = [] + end + if !haskey(pm_data, "switch") + pm_data["switch"] = [] + end + + # Add conformity key to bus data if not present + missing_conformity_loads = [ + bus for bus in pm_data["bus"] + if (bus["pd"] != 0.0 || bus["qd"] != 0.0) && !haskey(bus, "conformity") + ] + for bus in missing_conformity_loads + bus["conformity"] = 1 + end + missing_conformity_count = length(missing_conformity_loads) + if missing_conformity_count > 0 + @info "No conformity field found for $missing_conformity_count load(s). Setting to default value of 1 (Conforming Load)." + end + + # translate component models + _mp2pm_branch!(pm_data) + _mp2pm_dcline!(pm_data) + + # translate cost models + _add_dcline_costs!(pm_data) + + # merge data tables + _merge_bus_name_data!(pm_data) + _merge_cost_data!(pm_data) + _merge_generic_data!(pm_data) + + # split loads and shunts from buses + _split_loads_shunts!(pm_data) + + # use once available + arrays_to_dicts!(pm_data) + + base_voltages = Dict{Int64, Float64}( + bus_ind => bus_data["base_kv"] for (bus_ind, bus_data) in pm_data["bus"] + ) + for transf in values(pm_data["branch"]) + if transf["transformer"] == true && !haskey(transf, "base_voltage_from") + transf["base_voltage_from"] = base_voltages[transf["f_bus"]] + transf["base_voltage_to"] = base_voltages[transf["t_bus"]] + end + end + + for optional in ["dcline", "load", "shunt", "storage", "switch"] + if length(pm_data[optional]) == 0 + pm_data[optional] = Dict{String, Any}() + end + end + + return pm_data +end + +""" + _split_loads_shunts!(data) + +Seperates Loads and Shunts in `data` under separate "load" and "shunt" keys in the +PowerModels data format. Includes references to originating bus via "load_bus" +and "shunt_bus" keys, respectively. +""" +function _split_loads_shunts!(data::Dict{String, Any}) + data["load"] = [] + data["shunt"] = [] + + load_num = 1 + shunt_num = 1 + for (i, bus) in enumerate(data["bus"]) + if bus["pd"] != 0.0 || bus["qd"] != 0.0 + append!( + data["load"], + [ + Dict{String, Any}( + "pd" => bus["pd"], + "qd" => bus["qd"], + "load_bus" => bus["bus_i"], + "status" => convert(Int8, bus["bus_type"] != 4), + "conformity" => get(bus, "conformity", 1), + "index" => load_num, + "source_id" => ["bus", bus["bus_i"]], + ), + ], + ) + load_num += 1 + end + + if bus["gs"] != 0.0 || bus["bs"] != 0.0 + append!( + data["shunt"], + [ + Dict{String, Any}( + "gs" => bus["gs"], + "bs" => bus["bs"], + "shunt_bus" => bus["bus_i"], + "status" => convert(Int8, bus["bus_type"] != 4), + "index" => shunt_num, + "source_id" => ["bus", bus["bus_i"]], + ), + ], + ) + shunt_num += 1 + end + + for key in ["pd", "qd", "gs", "bs"] + delete!(bus, key) + end + end +end + +"sets all branch transformer taps to 1.0, to simplify branch models" +function _mp2pm_branch!(data::Dict{String, Any}) + branches = [branch for branch in data["branch"]] + if haskey(data, "ne_branch") + append!(branches, data["ne_branch"]) + end + for branch in branches + if branch["tap"] == 0.0 + branch["transformer"] = false + branch["tap"] = 1.0 + # Evenly split the susceptance between the `from` and `to` ends + branch["b_fr"] = branch["br_b"] / 2.0 + branch["b_to"] = branch["br_b"] / 2.0 + else + branch["transformer"] = true + if branch["br_b"] != 0.0 + @warn "Reflecting transformer shunts to primary; the ybus matrix will differ from matpower" maxlog = + 5 + branch["b_fr"] = (branch["br_b"] / branch["tap"]^2) + else + branch["b_fr"] = 0.0 + end + branch["b_to"] = 0.0 + end + branch["g_fr"] = 0.0 + branch["g_to"] = 0.0 + + branch["base_power"] = data["baseMVA"] + + delete!(branch, "br_b") + + if branch["rate_a"] == 0.0 + delete!(branch, "rate_a") + end + if branch["rate_b"] == 0.0 + delete!(branch, "rate_b") + end + if branch["rate_c"] == 0.0 + delete!(branch, "rate_c") + end + end +end + +"adds pmin and pmax values at to and from buses" +function _mp2pm_dcline!(data::Dict{String, Any}) + for dcline in data["dcline"] + pmin = dcline["pmin"] + pmax = dcline["pmax"] + loss0 = dcline["loss0"] + loss1 = dcline["loss1"] + + delete!(dcline, "pmin") + delete!(dcline, "pmax") + + if pmin >= 0 && pmax >= 0 + pminf = pmin + pmaxf = pmax + pmint = loss0 - pmaxf * (1 - loss1) + pmaxt = loss0 - pminf * (1 - loss1) + end + if pmin >= 0 && pmax < 0 + pminf = pmin + pmint = pmax + pmaxf = (-pmint + loss0) / (1 - loss1) + pmaxt = loss0 - pminf * (1 - loss1) + end + if pmin < 0 && pmax >= 0 + pmaxt = -pmin + pmaxf = pmax + pminf = (-pmaxt + loss0) / (1 - loss1) + pmint = loss0 - pmaxf * (1 - loss1) + end + if pmin < 0 && pmax < 0 + pmaxt = -pmin + pmint = pmax + pmaxf = (-pmint + loss0) / (1 - loss1) + pminf = (-pmaxt + loss0) / (1 - loss1) + end + + dcline["pmaxt"] = pmaxt + dcline["pmint"] = pmint + dcline["pmaxf"] = pmaxf + dcline["pminf"] = pminf + + # preserve the old pmin and pmax values + dcline["mp_pmin"] = pmin + dcline["mp_pmax"] = pmax + + dcline["pt"] = -dcline["pt"] # matpower has opposite convention + dcline["qf"] = -dcline["qf"] # matpower has opposite convention + dcline["qt"] = -dcline["qt"] # matpower has opposite convention + end +end + +"adds dcline costs, if gen costs exist" +function _add_dcline_costs!(data::Dict{String, Any}) + if length(data["gencost"]) > 0 && + length(data["dclinecost"]) <= 0 && + length(data["dcline"]) > 0 + @info("added zero cost function data for dclines") + model = data["gencost"][1]["model"] + if model == 1 + for (i, dcline) in enumerate(data["dcline"]) + dclinecost = Dict( + "index" => i, + "model" => 1, + "startup" => 0.0, + "shutdown" => 0.0, + "ncost" => 2, + "cost" => [dcline["pminf"], 0.0, dcline["pmaxf"], 0.0], + ) + push!(data["dclinecost"], dclinecost) + end + else + for (i, dcline) in enumerate(data["dcline"]) + dclinecost = Dict( + "index" => i, + "model" => 2, + "startup" => 0.0, + "shutdown" => 0.0, + "ncost" => 3, + "cost" => [0.0, 0.0, 0.0], + ) + push!(data["dclinecost"], dclinecost) + end + end + end +end + +"merges generator cost functions into generator data, if costs exist" +function _merge_cost_data!(data::Dict{String, Any}) + if haskey(data, "gencost") + gen = data["gen"] + gencost = data["gencost"] + + if length(gen) != length(gencost) + if length(gencost) > length(gen) + @warn( + "The last $(length(gencost) - length(gen)) generator cost records will be ignored due to too few generator records.", + ) + gencost = gencost[1:length(gen)] + else + @warn( + "The number of generators ($(length(gen))) does not match the number of generator cost records ($(length(gencost))).", + ) + end + end + + for (i, gc) in enumerate(gencost) + g = gen[i] + @assert(g["index"] == gc["index"]) + delete!(gc, "index") + delete!(gc, "source_id") + + _check_keys(g, keys(gc)) + merge!(g, gc) + end + + delete!(data, "gencost") + end + + if haskey(data, "dclinecost") + dcline = data["dcline"] + dclinecost = data["dclinecost"] + + if length(dcline) != length(dclinecost) + @warn( + "The number of dclines ($(length(dcline))) does not match the number of dcline cost records ($(length(dclinecost))).", + ) + end + + for (i, dclc) in enumerate(dclinecost) + dcl = dcline[i] + @assert(dcl["index"] == dclc["index"]) + delete!(dclc, "index") + delete!(dclc, "source_id") + + _check_keys(dcl, keys(dclc)) + merge!(dcl, dclc) + end + delete!(data, "dclinecost") + end +end + +"merges bus name data into buses, if names exist" +function _merge_bus_name_data!(data::Dict{String, Any}) + if haskey(data, "bus_name") + # can assume same length is same as bus + # this is validated during matpower parsing + for (i, bus_name) in enumerate(data["bus_name"]) + bus = data["bus"][i] + delete!(bus_name, "index") + delete!(bus_name, "source_id") + + _check_keys(bus, keys(bus_name)) + merge!(bus, bus_name) + end + delete!(data, "bus_name") + end +end + +"merges Matpower tables based on the table extension syntax" +function _merge_generic_data!(data::Dict{String, Any}) + mp_matrix_names = [name[5:length(name)] for name in _mp_data_names] + + key_to_delete = [] + for (k, v) in data + if isa(v, Array) + for mp_name in mp_matrix_names + if startswith(k, "$(mp_name)_") + mp_matrix = data[mp_name] + push!(key_to_delete, k) + + if length(mp_matrix) != length(v) + error( + "failed to extend the matpower matrix \"$(mp_name)\" with the matrix \"$(k)\" because they do not have the same number of rows, $(length(mp_matrix)) and $(length(v)) respectively.", + ) + end + + @info( + "extending matpower format by appending matrix \"$(k)\" in to \"$(mp_name)\"" + ) + + for (i, row) in enumerate(mp_matrix) + merge_row = v[i] + #@assert(row["index"] == merge_row["index"]) # note this does not hold for the bus table + delete!(merge_row, "index") + delete!(merge_row, "source_id") + for key in keys(merge_row) + if haskey(row, key) + error( + "failed to extend the matpower matrix \"$(mp_name)\" with the matrix \"$(k)\" because they both share \"$(key)\" as a column name.", + ) + end + row[key] = merge_row[key] + end + end + + break # out of mp_matrix_names loop + end + end + end + end + + for key in key_to_delete + delete!(data, key) + end +end + +"" +function _check_keys(data, keys) + for key in keys + if haskey(data, key) + error("attempting to overwrite value of $(key) in PowerModels data,\n$(data)") + end + end +end diff --git a/src/pm_io/psse.jl b/src/pm_io/psse.jl new file mode 100644 index 0000000..8377273 --- /dev/null +++ b/src/pm_io/psse.jl @@ -0,0 +1,2347 @@ +# Parse PSS(R)E data from PTI file into PowerModels data format +""" + _init_bus!(bus, id) + +Initializes a `bus` of id `id` with default values given in the PSS(R)E +specification. +""" +function _init_bus!(bus::Dict{String, Any}, id::Int) + bus["bus_i"] = id + bus["bus_type"] = 1 + bus["area"] = 1 + bus["vm"] = 1.0 + bus["va"] = 0.0 + bus["base_kv"] = 1.0 + bus["zone"] = 1 + bus["name"] = " " + bus["vmax"] = 1.1 + bus["vmin"] = 0.9 + bus["index"] = id + return +end + +function _find_bus_value(bus_i::Int, field::String, pm_bus_data::Array) + for bus in pm_bus_data + if bus["index"] == bus_i + return bus[field] + end + end + @info("Could not find bus $bus_i, returning 0 for field $field") + return 0 +end + +function _find_bus_value(bus_i::Int, field::String, pm_bus_data::Dict) + if !haskey(pm_bus_data, bus_i) + @info("Could not find bus $bus_i, returning 0 for field $field") + return 0 + else + return pm_bus_data[bus_i][field] + end +end + +""" + _get_bus_value(bus_i, field, pm_data) + +Returns the value of `field` of `bus_i` from the PowerModels data. Requires +"bus" Dict to already be populated. +""" +function _get_bus_value(bus_i::Int, field::String, pm_data::Dict{String, Any}) + return _find_bus_value(bus_i, field, pm_data["bus"]) +end + +""" + _find_max_bus_id(pm_data) + +Returns the maximum bus id in `pm_data` +""" +function _find_max_bus_id(pm_data::Dict)::Int + max_id = 0 + for bus in values(pm_data["bus"]) + if bus["index"] > max_id && !endswith(bus["name"], "starbus") + max_id = bus["index"] + end + end + + return max_id +end + +""" + create_starbus(pm_data, transformer) + +Creates a starbus from a given three-winding `transformer`. "source_id" is given +by `["bus_i", "name", "I", "J", "K", "CKT"]` where "bus_i" and "name" are the +modified names for the starbus, and "I", "J", "K" and "CKT" come from the +originating transformer, in the PSS(R)E transformer specification. +""" +function _create_starbus_from_transformer( + pm_data::Dict, + transformer::Dict, + starbus_id::Int, +)::Dict + starbus = Dict{String, Any}() + + _init_bus!(starbus, starbus_id) + + starbus["name"] = "starbus_$(transformer["I"])_$(transformer["J"])_$(transformer["K"])_$(strip(transformer["CKT"]))" + + bus_type = 1 + starbus["vm"] = transformer["VMSTAR"] + starbus["va"] = transformer["ANSTAR"] + starbus["bus_type"] = bus_type + if transformer["STAT"] != 0 + starbus["bus_status"] = true + else + starbus["bus_status"] = false + end + starbus["area"] = _get_bus_value(transformer["I"], "area", pm_data) + starbus["zone"] = _get_bus_value(transformer["I"], "zone", pm_data) + starbus["hidden"] = true + starbus["source_id"] = push!( + ["transformer", starbus["bus_i"], starbus["name"]], + transformer["I"], + transformer["J"], + transformer["K"], + transformer["CKT"], + ) + + return starbus +end + +"Imports remaining top level component lists from `data_in` into `data_out`, excluding keys in `exclude`" +function _import_remaining_comps!(data_out::Dict, data_in::Dict; exclude = []) + for (comp_class, v) in data_in + if !(comp_class in exclude) + comps_out = Dict{String, Any}() + + if isa(v, Array) + for (n, item) in enumerate(v) + if isa(item, Dict) + comp_out = Dict{String, Any}() + _import_remaining_keys!(comp_out, item) + if !("index" in keys(item)) + comp_out["index"] = n + end + comps_out["$(n)"] = comp_out + else + @error("psse data parsing error, please post an issue") + end + end + elseif isa(v, Dict) + comps_out = Dict{String, Any}() + _import_remaining_keys!(comps_out, v) + else + @error("psse data parsing error, please post an issue") + end + + data_out[lowercase(comp_class)] = comps_out + end + end +end + +"Imports remaining keys from a source component into detestation component, excluding keys in `exclude`" +function _import_remaining_keys!(comp_dest::Dict, comp_src::Dict; exclude = []) + for (k, v) in comp_src + if !(k in exclude) + key = lowercase(k) + if !haskey(comp_dest, key) + comp_dest[key] = v + else + if key != "index" + @warn("duplicate key $(key), please post an issue") + end + end + end + end +end + +""" + _psse2pm_branch!(pm_data, pti_data) + +Parses PSS(R)E-style Branch data into a PowerModels-style Dict. "source_id" is +given by `["I", "J", "CKT"]` in PSS(R)E Branch specification. +""" +function _psse2pm_branch!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Branch data into a PowerModels Dict..." + pm_data["branch"] = [] + if haskey(pti_data, "BRANCH") + for branch in pti_data["BRANCH"] + if !haskey(branch, "I") || !haskey(branch, "J") + @error "Bus Data Incomplete for $(branch). Skipping branch creation" + continue + end + if first(branch["CKT"]) != '@' && first(branch["CKT"]) != '*' + sub_data = Dict{String, Any}() + sub_data["f_bus"] = pop!(branch, "I") + sub_data["t_bus"] = pop!(branch, "J") + bus_from = pm_data["bus"][sub_data["f_bus"]] + sub_data["base_voltage_from"] = bus_from["base_kv"] + bus_to = pm_data["bus"][sub_data["t_bus"]] + sub_data["base_voltage_to"] = bus_to["base_kv"] + if pm_data["has_isolated_type_buses"] + if !(bus_from["bus_type"] == 4 || bus_to["bus_type"] == 4) + push!(pm_data["connected_buses"], sub_data["f_bus"]) + push!(pm_data["connected_buses"], sub_data["t_bus"]) + end + end + sub_data["br_r"] = pop!(branch, "R") + sub_data["br_x"] = pop!(branch, "X") + sub_data["g_fr"] = pop!(branch, "GI") + # Evenly split the susceptance between the `from` and `to` ends + sub_data["b_fr"] = (branch["B"] / 2) + pop!(branch, "BI") + sub_data["g_to"] = pop!(branch, "GJ") + sub_data["b_to"] = (branch["B"] / 2) + pop!(branch, "BJ") + + sub_data["ext"] = Dict{String, Any}( + "LEN" => pop!(branch, "LEN"), + ) + + if pm_data["source_version"] ∈ ("32", "33") + sub_data["rate_a"] = pop!(branch, "RATEA") + sub_data["rate_b"] = pop!(branch, "RATEB") + sub_data["rate_c"] = pop!(branch, "RATEC") + elseif pm_data["source_version"] == "35" + sub_data["rate_a"] = pop!(branch, "RATE1") + sub_data["rate_b"] = pop!(branch, "RATE2") + sub_data["rate_c"] = pop!(branch, "RATE3") + + for i in 4:12 + rate_key = "RATE$i" + if haskey(branch, rate_key) + sub_data["ext"][rate_key] = pop!(branch, rate_key) + end + end + else + error( + "Unsupported PSS(R)E source version: $(pm_data["source_version"])", + ) + end + + sub_data["tap"] = 1.0 + sub_data["shift"] = 0.0 + sub_data["br_status"] = pop!(branch, "ST") + sub_data["angmin"] = 0.0 + sub_data["angmax"] = 0.0 + sub_data["transformer"] = false + + sub_data["source_id"] = + ["branch", sub_data["f_bus"], sub_data["t_bus"], pop!(branch, "CKT")] + sub_data["index"] = length(pm_data["branch"]) + 1 + + if import_all + _import_remaining_keys!(sub_data, branch; exclude = ["B", "BI", "BJ"]) + end + + if sub_data["rate_a"] == 0.0 + delete!(sub_data, "rate_a") + end + if sub_data["rate_b"] == 0.0 + delete!(sub_data, "rate_b") + end + if sub_data["rate_c"] == 0.0 + delete!(sub_data, "rate_c") + end + branch_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data["branch"], sub_data) + else + from_bus = branch["I"] + to_bus = branch["J"] + ckt = branch["CKT"] + @info "Branch $from_bus -> $to_bus with CKT=$ckt will be parsed as DiscreteControlledACBranch" + end + end + end + return +end + +function branch_isolated_bus_modifications!(pm_data::Dict, branch_data::Dict) + bus_data = pm_data["bus"] + from_bus_no = branch_data["f_bus"] + to_bus_no = branch_data["t_bus"] + from_bus = bus_data[from_bus_no] + to_bus = bus_data[to_bus_no] + + status_field = haskey(branch_data, "br_status") ? "br_status" : "state" + if (from_bus["bus_type"] == 4 || to_bus["bus_type"] == 4) && + branch_data[status_field] == 1 + @warn "Branch connected between buses $(from_bus_no) -> $(to_bus_no) is connected to an isolated bus. Setting branch status to 0." + branch_data[status_field] = 0 + end + if from_bus["bus_type"] == 4 + push!(pm_data["candidate_isolated_to_pq_buses"], from_bus_no) + end + if to_bus["bus_type"] == 4 + push!(pm_data["candidate_isolated_to_pq_buses"], to_bus_no) + end + return +end + +function transformer3W_isolated_bus_modifications!(pm_data::Dict, branch_data::Dict) + bus_data = pm_data["bus"] + primary_bus_number = branch_data["bus_primary"] + secondary_bus_number = branch_data["bus_secondary"] + tertiary_bus_number = branch_data["bus_tertiary"] + primary_bus = bus_data[primary_bus_number] + secondary_bus = bus_data[secondary_bus_number] + tertiary_bus = bus_data[tertiary_bus_number] + if branch_data["available"] == 1 + if primary_bus["bus_type"] == 4 + branch_data["available_primary"] = 0 + @warn "Three winding transformer primary bus $(primary_bus_number) is isolated. Setting primary winding status to 0." + end + if secondary_bus["bus_type"] == 4 + branch_data["available_secondary"] = 0 + @warn "Three winding transformer secondary bus $(secondary_bus_number) is isolated. Setting secondary winding status to 0." + end + if tertiary_bus["bus_type"] == 4 + branch_data["available_tertiary"] = 0 + @warn "Three winding transformer tertiary bus $(tertiary_bus_number) is isolated. Setting tertiary winding status to 0." + end + if ( + branch_data["available_primary"] == 0 && + branch_data["available_secondary"] == 0 && + branch_data["available_tertiary"] == 0 + ) + branch_data["available"] = 0 + @warn "All three windings are unavailable. Setting overall transformer availability to 0" + end + end + if primary_bus["bus_type"] == 4 + push!(pm_data["candidate_isolated_to_pq_buses"], primary_bus_number) + end + if secondary_bus["bus_type"] == 4 + push!(pm_data["candidate_isolated_to_pq_buses"], secondary_bus_number) + end + if tertiary_bus["bus_type"] == 4 + push!(pm_data["candidate_isolated_to_pq_buses"], tertiary_bus_number) + end + return +end + +""" + _is_synch_condenser(sub_data, pm_data) + +Returns `true` if the generator described by `sub_data` and `pm_data` meets the criteria for a synchronous condenser. +""" +function _is_synch_condenser(sub_data::Dict{String, Any}, pm_data::Dict{String, Any}) + is_zero_pg = sub_data["pg"] == 0.0 + has_q_limits = (sub_data["qmax"] != 0.0 || sub_data["qmin"] != 0.0) + has_zero_p_limits = (sub_data["pmax"] == 0.0 && sub_data["pmin"] == 0.0) + zero_control_mode = sub_data["m_control_mode"] == 0 + is_pv_bus = pm_data["bus"][sub_data["gen_bus"]]["bus_type"] == 2 + + if is_zero_pg && has_q_limits && has_zero_p_limits && zero_control_mode + if !is_pv_bus + @warn "Generator $(sub_data["gen_bus"]) is likely a synchronous condenser but not connected to a PV bus." + end + return true + end + return false +end + +function _determine_injector_status( + sub_data::Dict{String, Any}, + pm_data::Dict{String, Any}, + gen_bus::Int, + status_key::String, + bus_conversion_list::String, +) + # Special case for FACTS: MODE = 0 -> Unavailable, MODE = 1 -> Normal mode, MODE = 2 -> Link bypassed + if status_key == "MODE" + device_status = pop!(sub_data, status_key) != 0 ? true : false + else + device_status = pop!(sub_data, status_key) == 1 ? true : false + end + # If device is off keep it off. + if !device_status + return false + end + # If device is on check the topology and status of the bus it is connected to. + if pm_data["bus"][gen_bus]["bus_type"] == 4 + gen_bus_connected = gen_bus ∈ pm_data["connected_buses"] + if gen_bus_connected && device_status + @warn "Device connected to bus $(gen_bus) is marked as available, but the bus is set isolated and not topologically isolated. Setting device status to 1 and the bus added to candidate for conversion." + push!(pm_data[bus_conversion_list], gen_bus) + pm_data["bus"][gen_bus]["bus_status"] = true + return true + elseif !gen_bus_connected && device_status + @warn "Device connected to bus $(gen_bus) is marked as available, but the bus is set isolated. Setting device status to 0." + pm_data["bus"][gen_bus]["bus_status"] = false + return false + else + error("Unrecognized generator and bus status combination.") + end + else + sub_data["gen_status"] = true + return true + end +end + +""" + _psse2pm_generator!(pm_data, pti_data) + +Parses PSS(R)E-style Generator data in a PowerModels-style Dict. "source_id" is +given by `["I", "ID"]` in PSS(R)E Generator specification. +""" +function _psse2pm_generator!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Generator data into a PowerModels Dict..." + if haskey(pti_data, "GENERATOR") + pm_data["gen"] = Vector{Dict{String, Any}}(undef, length(pti_data["GENERATOR"])) + for (ix, gen) in enumerate(pti_data["GENERATOR"]) + sub_data = Dict{String, Any}() + sub_data["gen_bus"] = pop!(gen, "I") + sub_data["gen_status"] = + _determine_injector_status( + gen, + pm_data, + sub_data["gen_bus"], + "STAT", + "candidate_isolated_to_pv_buses", + ) + sub_data["pg"] = pop!(gen, "PG") + sub_data["qg"] = pop!(gen, "QG") + sub_data["vg"] = pop!(gen, "VS") + sub_data["mbase"] = pop!(gen, "MBASE") + sub_data["pmin"] = pop!(gen, "PB") + sub_data["pmax"] = pop!(gen, "PT") + sub_data["qmin"] = pop!(gen, "QB") + sub_data["qmax"] = pop!(gen, "QT") + sub_data["rt_source"] = pop!(gen, "RT") + sub_data["xt_source"] = pop!(gen, "XT") + sub_data["r_source"] = pop!(gen, "ZR") + sub_data["x_source"] = pop!(gen, "ZX") + sub_data["m_control_mode"] = pop!(gen, "WMOD") + + if _is_synch_condenser(sub_data, pm_data) + sub_data["fuel"] = "SYNC_COND" + sub_data["type"] = "SYNC_COND" + end + + if pm_data["source_version"] == "35" + sub_data["ext"] = Dict{String, Any}( + "NREG" => pop!(gen, "NREG"), + "BASLOD" => pop!(gen, "BASLOD"), + ) + elseif pm_data["source_version"] ∈ ("32", "33") + sub_data["ext"] = Dict{String, Any}( + "IREG" => pop!(gen, "IREG"), + "WPF" => pop!(gen, "WPF"), + "WMOD" => sub_data["m_control_mode"], + "GTAP" => pop!(gen, "GTAP"), + "RMPCT" => pop!(gen, "RMPCT"), + ) + else + error("Unsupported PSS(R)E source version: $(pm_data["source_version"])") + end + + # Default Cost functions + sub_data["model"] = 2 + sub_data["startup"] = 0.0 + sub_data["shutdown"] = 0.0 + sub_data["ncost"] = 2 + sub_data["cost"] = [1.0, 0.0] + + sub_data["source_id"] = + ["generator", string(sub_data["gen_bus"]), pop!(gen, "ID")] + sub_data["index"] = ix + + if import_all + _import_remaining_keys!(sub_data, gen) + end + + pm_data["gen"][ix] = sub_data + end + else + pm_data["gen"] = Vector{Dict{String, Any}}() + end +end + +function _psse2pm_area_interchange!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E AreaInterchange data into a PowerModels Dict..." + pm_data["area_interchange"] = [] + + if haskey(pti_data, "AREA INTERCHANGE") + for area_int in pti_data["AREA INTERCHANGE"] + sub_data = Dict{String, Any}() + sub_data["area_name"] = pop!(area_int, "ARNAME") + sub_data["area_number"] = pop!(area_int, "I") + sub_data["bus_number"] = pop!(area_int, "ISW") + sub_data["net_interchange"] = pop!(area_int, "PDES") + sub_data["tol_interchange"] = pop!(area_int, "PTOL") + sub_data["index"] = length(pm_data["area_interchange"]) + 1 + if import_all + _import_remaining_keys!(sub_data, area_int) + end + + push!(pm_data["area_interchange"], sub_data) + end + end +end + +function _psse2pm_interarea_transfer!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E InterAreaTransfer data into a PowerModels Dict..." + pm_data["interarea_transfer"] = [] + + if haskey(pti_data, "INTER-AREA TRANSFER") + for interarea in pti_data["INTER-AREA TRANSFER"] + sub_data = Dict{String, Any}() + sub_data["area_from"] = pop!(interarea, "ARFROM") + sub_data["area_to"] = pop!(interarea, "ARTO") + sub_data["transfer_id"] = pop!(interarea, "TRID") + sub_data["power_transfer"] = pop!(interarea, "PTRAN") + + sub_data["index"] = length(pm_data["interarea_transfer"]) + 1 + if import_all + _import_remaining_keys!(sub_data, interarea) + end + + push!(pm_data["interarea_transfer"], sub_data) + end + end +end + +function _psse2pm_zone!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Zone data into a PowerModels Dict..." + pm_data["zone"] = [] + + if haskey(pti_data, "ZONE") + for zone in pti_data["ZONE"] + sub_data = Dict{String, Any}() + sub_data["zone_number"] = pop!(zone, "I") + sub_data["zone_name"] = pop!(zone, "ZONAME") + sub_data["index"] = length(pm_data["zone"]) + 1 + if import_all + _import_remaining_keys!(sub_data, zone) + end + + push!(pm_data["zone"], sub_data) + end + end +end + +""" + _psse2pm_bus!(pm_data, pti_data) + +Parses PSS(R)E-style Bus data into a PowerModels-style Dict. "source_id" is given +by ["I", "NAME"] in PSS(R)E Bus specification. +""" +function _psse2pm_bus!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Bus data into a PowerModels Dict..." + pm_data["has_isolated_type_buses"] = false + pm_data["bus"] = Dict{Int, Any}() + if haskey(pti_data, "BUS") + for bus in pti_data["BUS"] + sub_data = Dict{String, Any}() + + sub_data["bus_i"] = bus["I"] + sub_data["bus_type"] = pop!(bus, "IDE") + if sub_data["bus_type"] == 4 + @warn "The PSS(R)E data contains buses designated as isolated. The parser will check if the buses are connected or topologically isolated." + pm_data["has_isolated_type_buses"] = true + sub_data["bus_status"] = false + pm_data["connected_buses"] = Set{Int}() + pm_data["candidate_isolated_to_pq_buses"] = Set{Int}() + pm_data["candidate_isolated_to_pv_buses"] = Set{Int}() + else + sub_data["bus_status"] = true + end + sub_data["area"] = pop!(bus, "AREA") + sub_data["vm"] = pop!(bus, "VM") + sub_data["va"] = pop!(bus, "VA") + sub_data["base_kv"] = pop!(bus, "BASKV") + sub_data["zone"] = pop!(bus, "ZONE") + sub_data["name"] = pop!(bus, "NAME") + sub_data["vmax"] = pop!(bus, "NVHI") + sub_data["vmin"] = pop!(bus, "NVLO") + sub_data["hidden"] = false + + sub_data["source_id"] = ["bus", "$(bus["I"])"] + sub_data["index"] = pop!(bus, "I") + + if import_all + _import_remaining_keys!(sub_data, bus) + end + + if haskey(pm_data["bus"], sub_data["bus_i"]) + error("Repeated $(sub_data["bus_i"])") + end + pm_data["bus"][sub_data["bus_i"]] = sub_data + end + end + return +end + +""" + _psse2pm_load!(pm_data, pti_data) + +Parses PSS(R)E-style Load data into a PowerModels-style Dict. "source_id" is given +by `["I", "ID"]` in the PSS(R)E Load specification. +""" +function _psse2pm_load!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Load data into a PowerModels Dict..." + pm_data["load"] = [] + if haskey(pti_data, "LOAD") + for load in pti_data["LOAD"] + sub_data = Dict{String, Any}() + sub_data["load_bus"] = pop!(load, "I") + sub_data["pd"] = pop!(load, "PL") + sub_data["qd"] = pop!(load, "QL") + sub_data["pi"] = pop!(load, "IP") + sub_data["qi"] = pop!(load, "IQ") + sub_data["py"] = pop!(load, "YP") + # Reactive power component of constant Y load. + # Positive for an inductive load (consumes Q) + # Negative for a capacitive load (injects Q) + sub_data["qy"] = -pop!(load, "YQ") + sub_data["conformity"] = pop!(load, "SCALE") + sub_data["source_id"] = ["load", sub_data["load_bus"], pop!(load, "ID")] + sub_data["interruptible"] = pop!(load, "INTRPT") + sub_data["ext"] = Dict{String, Any}() + + if pm_data["source_version"] ∈ ("32", "33") + sub_data["ext"]["LOADTYPE"] = "" + elseif pm_data["source_version"] == "35" + sub_data["ext"]["LOADTYPE"] = pop!(load, "LOADTYPE", "") + else + error("Unsupported PSS(R)E source version: $(pm_data["source_version"])") + end + + sub_data["status"] = + _determine_injector_status( + load, + pm_data, + sub_data["load_bus"], + "STATUS", + "candidate_isolated_to_pq_buses", + ) + sub_data["index"] = length(pm_data["load"]) + 1 + if import_all + _import_remaining_keys!(sub_data, load) + end + push!(pm_data["load"], sub_data) + end + end +end + +""" + _psse2pm_shunt!(pm_data, pti_data) + +Parses PSS(R)E-style Fixed and Switched Shunt data into a PowerModels-style +Dict. "source_id" is given by `["I", "ID"]` for Fixed Shunts, and `["I", "SWREM"]` +for Switched Shunts, as given by the PSS(R)E Fixed and Switched Shunts +specifications. +""" +function _psse2pm_shunt!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Fixed & Switched Shunt data into a PowerModels Dict..." + + pm_data["shunt"] = [] + if haskey(pti_data, "FIXED SHUNT") + for shunt in pti_data["FIXED SHUNT"] + sub_data = Dict{String, Any}() + + sub_data["shunt_bus"] = pop!(shunt, "I") + sub_data["gs"] = pop!(shunt, "GL") + sub_data["bs"] = pop!(shunt, "BL") + sub_data["status"] = _determine_injector_status( + shunt, + pm_data, + sub_data["shunt_bus"], + "STATUS", + "candidate_isolated_to_pq_buses", + ) + + sub_data["source_id"] = + ["fixed shunt", sub_data["shunt_bus"], pop!(shunt, "ID")] + sub_data["index"] = length(pm_data["shunt"]) + 1 + + if import_all + _import_remaining_keys!(sub_data, shunt) + end + push!(pm_data["shunt"], sub_data) + end + end + + pm_data["switched_shunt"] = [] + if haskey(pti_data, "SWITCHED SHUNT") + for switched_shunt in pti_data["SWITCHED SHUNT"] + sub_data = Dict{String, Any}() + + sub_data["shunt_bus"] = pop!(switched_shunt, "I") + sub_data["gs"] = 0.0 + sub_data["bs"] = pop!(switched_shunt, "BINIT") + sub_data["status"] = _determine_injector_status( + switched_shunt, + pm_data, + sub_data["shunt_bus"], + "STAT", + "candidate_isolated_to_pq_buses", + ) + sub_data["admittance_limits"] = + (pop!(switched_shunt, "VSWLO"), pop!(switched_shunt, "VSWHI")) + + step_numbers = Dict( + k => v for + (k, v) in switched_shunt if startswith(k, "N") && isdigit(last(k)) + ) + step_numbers_sorted = + sort(collect(keys(step_numbers)); by = x -> parse(Int, x[2:end])) + sub_data["step_number"] = [step_numbers[k] for k in step_numbers_sorted] + sub_data["step_number"] = sub_data["step_number"][sub_data["step_number"] .!= 0] + + sub_data["ext"] = Dict{String, Any}( + "MODSW" => switched_shunt["MODSW"], + "ADJM" => switched_shunt["ADJM"], + "RMPCT" => switched_shunt["RMPCT"], + "RMIDNT" => switched_shunt["RMIDNT"], + ) + + y_increment = Dict( + k => v for + (k, v) in switched_shunt if startswith(k, "B") && isdigit(last(k)) + ) + y_increment_sorted = + sort(collect(keys(y_increment)); by = x -> parse(Int, x[2:end])) + sub_data["y_increment"] = [y_increment[k] for k in y_increment_sorted]im + sub_data["y_increment"] = sub_data["y_increment"][sub_data["y_increment"] .!= 0] + + if pm_data["source_version"] == "35" + sub_data["sw_id"] = pop!(switched_shunt, "ID") + + initial_ss_status = Dict( + k => v for + (k, v) in switched_shunt if startswith(k, "S") && isdigit(last(k)) + ) + initial_ss_status_sorted = + sort(collect(keys(initial_ss_status)); by = x -> parse(Int, x[2:end])) + sub_data["initial_status"] = + [initial_ss_status[k] for k in initial_ss_status_sorted] + sub_data["initial_status"] = + sub_data["initial_status"][1:length(sub_data["step_number"])] + + sub_data["ext"]["NREG"] = pop!(switched_shunt, "NREG") + elseif pm_data["source_version"] ∈ ("32", "33") + sub_data["ext"]["SWREM"] = switched_shunt["SWREM"] + sub_data["initial_status"] = ones(Int, length(sub_data["y_increment"])) + else + error("Unsupported PSS(R)E source version: $(pm_data["source_version"])") + end + + sub_data["index"] = length(pm_data["switched_shunt"]) + 1 + sub_data["source_id"] = + ["switched shunt", sub_data["shunt_bus"], sub_data["index"]] + + if import_all + _import_remaining_keys!(sub_data, switched_shunt) + end + push!(pm_data["switched_shunt"], sub_data) + end + end +end + +function apply_tap_correction!( + windv_value::Float64, + transformer::Dict{String, Any}, + cod_key::String, + rmi_key::String, + rma_key::String, + ntp_key::String, + cw_value::Int64, + winding_name::String, +) + if abs(transformer[cod_key]) ∈ [1, 2] && cw_value ∈ [1, 2, 3] + tap_positions = collect( + range( + transformer[rmi_key], + transformer[rma_key]; + length = Int(transformer[ntp_key]), + ), + ) + closest_tap_ix = argmin(abs.(tap_positions .- windv_value)) + if !isapprox( + windv_value, + tap_positions[closest_tap_ix]; + atol = PARSER_TAP_RATIO_CORRECTION_TOL, + ) + @warn "Transformer $winding_name winding tap setting is not on a step; $windv_value set to $(tap_positions[closest_tap_ix])" + return tap_positions[closest_tap_ix] + end + end + return windv_value +end + +# Base Power has a different key in sub_data depending on the number of windings +function _transformer_mag_pu_conversion( + transformer::Dict, + sub_data::Dict, + base_power::Float64, +) + if isapprox(transformer["MAG1"], ZERO_IMPEDANCE_REACTANCE_THRESHOLD) && + isapprox(transformer["MAG2"], ZERO_IMPEDANCE_REACTANCE_THRESHOLD) + @warn "Transformer $(sub_data["f_bus"]) -> $(sub_data["t_bus"]) has zero MAG1 and MAG2 values." + return 0.0, 0.0 + else + G_pu = 1e-6 * transformer["MAG1"] / base_power + mag_diff = transformer["MAG2"]^2 - G_pu^2 + @assert mag_diff >= -ZERO_IMPEDANCE_REACTANCE_THRESHOLD + B_pu = sqrt(max(0.0, mag_diff)) + return G_pu, B_pu + end +end + +""" + _psse2pm_transformer!(pm_data, pti_data) + +Parses PSS(R)E-style Transformer data into a PowerModels-style Dict. "source_id" +is given by `["I", "J", "K", "CKT", "winding"]`, where "winding" is 0 if +transformer is two-winding, and 1, 2, or 3 for three-winding, and the remaining +keys are defined in the PSS(R)E Transformer specification. +""" +function _psse2pm_transformer!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Transformer data into a PowerModels Dict..." + if !haskey(pm_data, "branch") + pm_data["branch"] = [] + end + + if haskey(pti_data, "TRANSFORMER") + starbus_id = 10^ceil(Int, log10(abs(_find_max_bus_id(pm_data)))) + 1 + for transformer in pti_data["TRANSFORMER"] + if !(transformer["CZ"] in [1, 2, 3]) + @warn( + "transformer CZ value outside of valid bounds assuming the default value of 1. Given $(transformer["CZ"]), should be 1, 2 or 3", + ) + transformer["CZ"] = 1 + end + + if !(transformer["CW"] in [1, 2, 3]) + @warn( + "transformer CW value outside of valid bounds assuming the default value of 1. Given $(transformer["CW"]), should be 1, 2 or 3", + ) + transformer["CW"] = 1 + end + + if !(transformer["CM"] in [1, 2]) + @warn( + "transformer CM value outside of valid bounds assuming the default value of 1. Given $(transformer["CM"]), should be 1 or 2", + ) + transformer["CM"] = 1 + end + + if transformer["K"] == 0 # Two-winding Transformers + sub_data = Dict{String, Any}() + + sub_data["f_bus"] = transformer["I"] + sub_data["t_bus"] = transformer["J"] + if pm_data["has_isolated_type_buses"] + bus_from = pm_data["bus"][sub_data["f_bus"]] + bus_to = pm_data["bus"][sub_data["t_bus"]] + if !(bus_from["bus_type"] == 4 || bus_to["bus_type"] == 4) + push!(pm_data["connected_buses"], sub_data["f_bus"]) + push!(pm_data["connected_buses"], sub_data["t_bus"]) + end + end + + # Store base_power + if transformer["SBASE1-2"] < 0.0 + throw( + IS.InvalidValue( + "Transformer $(sub_data["f_bus"]) -> $(sub_data["t_bus"]) has non-positive base power SBASE1-2: $(transformer["SBASE1-2"])", + ), + ) + end + if iszero(transformer["SBASE1-2"]) + sub_data["base_power"] = pm_data["baseMVA"] + else + sub_data["base_power"] = transformer["SBASE1-2"] + end + if iszero(transformer["NOMV1"]) + sub_data["base_voltage_from"] = + _get_bus_value(transformer["I"], "base_kv", pm_data) + else + sub_data["base_voltage_from"] = transformer["NOMV1"] + end + if iszero(transformer["NOMV2"]) + sub_data["base_voltage_to"] = + _get_bus_value(transformer["J"], "base_kv", pm_data) + else + sub_data["base_voltage_to"] = transformer["NOMV2"] + end + + # Unit Transformations + # Data must be stored in the DEVICE_BASE + # Z_base_device = (V_device)^2 / S_device, Z_base_sys = (V_device)^2 / S_sys + # Z_ohms = Z_pu_sys * Z_base_sys, Z_pu_device = Z_ohms / Z_device = Z_pu_sys * S_device / S_sys + mva_ratio = sub_data["base_power"] / pm_data["baseMVA"] + Z_base_device = sub_data["base_voltage_from"]^2 / sub_data["base_power"] + Z_base_sys = sub_data["base_voltage_from"]^2 / pm_data["baseMVA"] + #_get_bus_value(transformer["I"], "base_kv", pm_data)^2 / + #pm_data["baseMVA"] + if transformer["CZ"] == 2 # "for resistance and reactance in pu on system MVA base and winding voltage base" + # Compute br_r and br_x in pu of device base + br_r, br_x = transformer["R1-2"], transformer["X1-2"] + else # NOT "for resistance and reactance in pu on system MVA base and winding voltage base" + if transformer["CZ"] == 3 # "for transformer load loss in watts and impedance magnitude in pu on a specified MVA base and winding voltage base." + br_r = 1e-6 * transformer["R1-2"] / sub_data["base_power"] # device pu + br_x = sqrt(transformer["X1-2"]^2 - br_r^2) # device pu + else # "CZ" = 1 in system base pu + @assert transformer["CZ"] == 1 + br_r, br_x = transformer["R1-2"], transformer["X1-2"] # sys pu + if iszero(Z_base_device) # NOMV1 = 0.0: use the power ratios + br_r = transformer["R1-2"] * mva_ratio + br_x = transformer["X1-2"] * mva_ratio + else # NOMV1 could potentially be different than the bus_voltage, use impedance ratios + br_r = (transformer["R1-2"] * Z_base_sys) / Z_base_device + br_x = (transformer["X1-2"] * Z_base_sys) / Z_base_device + end + end + end + + # Zeq scaling for tap2 (see eq (4.21b) in PROGRAM APPLICATION GUIDE 1 in PSSE installation folder) + # Unit Transformations + if transformer["CW"] == 1 # "for off-nominal turns ratio in pu of winding bus base voltage" + br_r *= transformer["WINDV2"]^2 + br_x *= transformer["WINDV2"]^2 + # NOT "for off-nominal turns ratio in pu of winding bus base voltage" + elseif transformer["CW"] == 2 # "for winding voltage in kV" + br_r *= + ( + transformer["WINDV2"] / + _get_bus_value(transformer["J"], "base_kv", pm_data) + )^2 + br_x *= + ( + transformer["WINDV2"] / + _get_bus_value(transformer["J"], "base_kv", pm_data) + )^2 + elseif transformer["CW"] == 3 # "for off-nominal turns ratio in pu of nominal winding voltage, NOMV1, NOMV2 and NOMV3." + #The nominal (rated) Winding 2 voltage base in kV, or zero to indicate + # that nominal Winding 2 voltage is assumed to be identical to the base + # voltage of bus J. NOMV2 is used in converting tap ratio data between values + # in per unit of nominal Winding 2 voltage and values in per unit of Winding 2 + #bus base voltage when CW is 3. NOMV2 = 0.0 by default. + if iszero(transformer["NOMV2"]) + nominal_voltage_ratio = 1.0 + else + nominal_voltage_ratio = + transformer["NOMV2"] / + _get_bus_value(transformer["J"], "base_kv", pm_data) + end + + br_r *= (transformer["WINDV2"] * (nominal_voltage_ratio))^2 + br_x *= (transformer["WINDV2"] * (nominal_voltage_ratio))^2 + else + error("invalid transformer $(transformer["CW"])") + end + + if transformer["X1-2"] < 0.0 && br_x < 0.0 + @warn "Transformer $(sub_data["f_bus"]) -> $(sub_data["t_bus"]) has negative impedance values X1-2: $(transformer["X1-2"]), br_x: $(br_x)" + end + + sub_data["br_r"] = br_r + sub_data["br_x"] = br_x + + if transformer["CM"] == 1 + # Transform admittance to device per unit + mva_ratio_12 = sub_data["base_power"] / pm_data["baseMVA"] + sub_data["g_fr"] = transformer["MAG1"] / mva_ratio_12 + sub_data["b_fr"] = transformer["MAG2"] / mva_ratio_12 + else # CM=2: MAG1 are no load loss in Watts and MAG2 is the exciting current in pu, in device base. + @assert transformer["CM"] == 2 + G_pu, B_pu = _transformer_mag_pu_conversion( + transformer, + sub_data, + sub_data["base_power"], + ) + sub_data["g_fr"] = G_pu + sub_data["b_fr"] = B_pu + end + sub_data["g_to"] = 0.0 + sub_data["b_to"] = 0.0 + + sub_data["ext"] = Dict{String, Any}( + "psse_name" => transformer["NAME"], + "CW" => transformer["CW"], + "CZ" => transformer["CZ"], + "CM" => transformer["CM"], + "COD1" => transformer["COD1"], + "CONT1" => transformer["CONT1"], + "NOMV1" => transformer["NOMV1"], + "NOMV2" => transformer["NOMV2"], + "WINDV1" => transformer["WINDV1"], + "WINDV2" => transformer["WINDV2"], + "SBASE1-2" => transformer["SBASE1-2"], + "RMI1" => transformer["RMI1"], + "RMA1" => transformer["RMA1"], + "NTP1" => transformer["NTP1"], + "R1-2" => transformer["R1-2"], + "X1-2" => transformer["X1-2"], + "MAG1" => transformer["MAG1"], + "MAG2" => transformer["MAG2"], + ) + + if pm_data["source_version"] ∈ ("32", "33") + sub_data["rate_a"] = pop!(transformer, "RATA1") + sub_data["rate_b"] = pop!(transformer, "RATB1") + sub_data["rate_c"] = pop!(transformer, "RATC1") + elseif pm_data["source_version"] == "35" + sub_data["rate_a"] = pop!(transformer, "RATE11") + sub_data["rate_b"] = pop!(transformer, "RATE12") + sub_data["rate_c"] = pop!(transformer, "RATE13") + + for i in 4:12 + rate_key = "RATE1$i" + if haskey(transformer, rate_key) + sub_data["ext"][rate_key] = pop!(transformer, rate_key) + end + end + else + error( + "Unsupported PSS(R)E source version: $(pm_data["source_version"])", + ) + end + + if sub_data["rate_a"] == 0.0 + delete!(sub_data, "rate_a") + end + if sub_data["rate_b"] == 0.0 + delete!(sub_data, "rate_b") + end + if sub_data["rate_c"] == 0.0 + delete!(sub_data, "rate_c") + end + + if import_all + sub_data["windv1"] = transformer["WINDV1"] + sub_data["windv2"] = transformer["WINDV2"] + sub_data["nomv1"] = transformer["NOMV1"] + sub_data["nomv2"] = transformer["NOMV2"] + end + + windv1 = pop!(transformer, "WINDV1") + windv1 = apply_tap_correction!( + windv1, + transformer, + "COD1", + "RMI1", + "RMA1", + "NTP1", + transformer["CW"], + "primary", + ) + sub_data["tap"] = windv1 / pop!(transformer, "WINDV2") + sub_data["shift"] = pop!(transformer, "ANG1") + + if transformer["CW"] != 1 # NOT "for off-nominal turns ratio in pu of winding bus base voltage" + sub_data["tap"] *= + _get_bus_value(transformer["J"], "base_kv", pm_data) / + _get_bus_value(transformer["I"], "base_kv", pm_data) + if transformer["CW"] == 3 # "for off-nominal turns ratio in pu of nominal winding voltage, NOMV1, NOMV2 and NOMV3." + if iszero(transformer["NOMV1"]) + winding1_nominal_voltage = + _get_bus_value(transformer["I"], "base_kv", pm_data) + else + winding1_nominal_voltage = transformer["NOMV1"] + end + + if iszero(transformer["NOMV2"]) + winding2_nominal_voltage = + _get_bus_value(transformer["J"], "base_kv", pm_data) + else + winding2_nominal_voltage = transformer["NOMV2"] + end + + sub_data["tap"] *= + winding1_nominal_voltage / winding2_nominal_voltage + end + end + + if import_all + sub_data["cw"] = transformer["CW"] + end + + if transformer["STAT"] == 0 || transformer["STAT"] == 2 + sub_data["br_status"] = 0 + else + sub_data["br_status"] = 1 + end + + sub_data["angmin"] = 0.0 + sub_data["angmax"] = 0.0 + + sub_data["source_id"] = [ + "transformer", + pop!(transformer, "I"), + pop!(transformer, "J"), + pop!(transformer, "K"), + pop!(transformer, "CKT"), + 0, + ] + + sub_data["transformer"] = true + sub_data["correction_table"] = transformer["TAB1"] + + sub_data["index"] = length(pm_data["branch"]) + 1 + sub_data["COD1"] = transformer["COD1"] + if import_all + _import_remaining_keys!( + sub_data, + transformer; + exclude = [ + "I", + "J", + "K", + "CZ", + "CW", + "R1-2", + "R2-3", + "R3-1", + "X1-2", + "X2-3", + "X3-1", + "SBASE1-2", + "SBASE2-3", + "SBASE3-1", + "MAG1", + "MAG2", + "STAT", + "NOMV1", + "NOMV2", + ], + ) + end + branch_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data["branch"], sub_data) + else # Three-winding Transformers + # Create 3w-transformer key + if !haskey(pm_data, "3w_transformer") + pm_data["3w_transformer"] = [] + end + + bus_id1, bus_id2, bus_id3 = + transformer["I"], transformer["J"], transformer["K"] + # Creates a starbus (or "dummy" bus) to which each winding of the transformer will connect + starbus = _create_starbus_from_transformer(pm_data, transformer, starbus_id) + pm_data["bus"][starbus_id] = starbus + if pm_data["has_isolated_type_buses"] + bus_primary = pm_data["bus"][bus_id1] + bus_secondary = pm_data["bus"][bus_id2] + bus_tertiary = pm_data["bus"][bus_id3] + push!(pm_data["connected_buses"], starbus_id) # Starbus should never be converted to isolated + # If one bus winding is isolated, the other two buses should still be considered connected: + !(bus_primary["bus_type"] == 4) && + push!(pm_data["connected_buses"], bus_id1) + !(bus_secondary["bus_type"] == 4) && + push!(pm_data["connected_buses"], bus_id2) + !(bus_tertiary["bus_type"] == 4) && + push!(pm_data["connected_buses"], bus_id3) + end + # Add parameters to the 3w-transformer key + sub_data = Dict{String, Any}() + bases = [ + transformer["SBASE1-2"], + transformer["SBASE2-3"], + transformer["SBASE3-1"], + ] + base_names = [ + "base_power_12", + "base_power_23", + "base_power_13", + ] + + for (ix, base) in enumerate(bases) + if base < 0.0 + throw( + IS.InvalidValue( + "Transformer $(transformer[I]) -> $(transformer["J"]) -> $(transformer["K"]) has negative base power $base", + ), + ) + end + if iszero(base) + sub_data[base_names[ix]] = pm_data["baseMVA"] + else + sub_data[base_names[ix]] = base + end + end + + if iszero(transformer["NOMV1"]) + sub_data["base_voltage_primary"] = + _get_bus_value(transformer["I"], "base_kv", pm_data) + else + sub_data["base_voltage_primary"] = transformer["NOMV1"] + end + if iszero(transformer["NOMV2"]) + sub_data["base_voltage_secondary"] = + _get_bus_value(transformer["J"], "base_kv", pm_data) + else + sub_data["base_voltage_secondary"] = transformer["NOMV2"] + end + if iszero(transformer["NOMV3"]) + sub_data["base_voltage_tertiary"] = + _get_bus_value(transformer["K"], "base_kv", pm_data) + else + sub_data["base_voltage_tertiary"] = transformer["NOMV3"] + end + + mva_ratio_12 = sub_data["base_power_12"] / pm_data["baseMVA"] + mva_ratio_23 = sub_data["base_power_23"] / pm_data["baseMVA"] + mva_ratio_31 = sub_data["base_power_13"] / pm_data["baseMVA"] + Z_base_device_1 = transformer["NOMV1"]^2 / sub_data["base_power_12"] + Z_base_device_2 = transformer["NOMV2"]^2 / sub_data["base_power_23"] + Z_base_device_3 = transformer["NOMV3"]^2 / sub_data["base_power_13"] + Z_base_sys_1 = (sub_data["base_voltage_primary"])^2 / pm_data["baseMVA"] + Z_base_sys_2 = + (sub_data["base_voltage_secondary"])^2 / pm_data["baseMVA"] + Z_base_sys_3 = + (sub_data["base_voltage_tertiary"])^2 / pm_data["baseMVA"] + # Create 3 branches from a three winding transformer (one for each winding, which will each connect to the starbus) + br_r12, br_r23, br_r31 = + transformer["R1-2"], transformer["R2-3"], transformer["R3-1"] + br_x12, br_x23, br_x31 = + transformer["X1-2"], transformer["X2-3"], transformer["X3-1"] + + # Unit Transformations + if transformer["CZ"] == 3 # "for transformer load loss in watts and impedance magnitude in pu on a specified MVA base and winding voltage base." + # In device base + br_r12 *= 1e-6 / sub_data["base_power_12"] + br_r23 *= 1e-6 / sub_data["base_power_23"] + br_r31 *= 1e-6 / sub_data["base_power_13"] + + br_x12 = sqrt(br_x12^2 - br_r12^2) + br_x23 = sqrt(br_x23^2 - br_r23^2) + br_x31 = sqrt(br_x31^2 - br_r31^2) + # Unit Transformations + elseif transformer["CZ"] == 1 # "for resistance and reactance in pu on system MVA base (transform to device base)" + if iszero(Z_base_device_1) # NOMV1 = 0.0: use the power ratios + br_r12 *= mva_ratio_12 + br_x12 *= mva_ratio_12 + else # NOMV1 could potentially be different than the bus_voltage, use impedance ratios + br_r12 *= Z_base_sys_1 / Z_base_device_1 + br_x12 *= Z_base_sys_1 / Z_base_device_1 + end + if iszero(Z_base_device_2) # NOMV2 = 0.0: use the power ratios + br_r23 *= mva_ratio_23 + br_x23 *= mva_ratio_23 + else # NOMV2 could potentially be different than the bus_voltage, use impedance ratios + br_r23 *= Z_base_sys_2 / Z_base_device_2 + br_x23 *= Z_base_sys_2 / Z_base_device_2 + end + if iszero(Z_base_device_3) # NOMV3 = 0.0: use the power ratios + br_r31 *= mva_ratio_31 + br_x31 *= mva_ratio_31 + else # NOMV3 could potentially be different than the bus_voltage, use impedance ratios + br_r31 *= Z_base_sys_3 / Z_base_device_3 + br_x31 *= Z_base_sys_3 / Z_base_device_3 + end + end + + # Compute primary,secondary, tertiary impedances in system base, then convert to base power of appropriate winding + if iszero(Z_base_device_1) + br_r12_sysbase = br_r12 / mva_ratio_12 + br_x12_sysbase = br_x12 / mva_ratio_12 + else + br_r12_sysbase = br_r12 * (Z_base_device_1 / Z_base_sys_1) + br_x12_sysbase = br_x12 * (Z_base_device_1 / Z_base_sys_1) + end + if iszero(Z_base_device_2) + br_r23_sysbase = br_r23 / mva_ratio_23 + br_x23_sysbase = br_x23 / mva_ratio_23 + else + br_r23_sysbase = br_r23 * (Z_base_device_2 / Z_base_sys_2) + br_x23_sysbase = br_x23 * (Z_base_device_2 / Z_base_sys_2) + end + if iszero(Z_base_device_3) + br_r31_sysbase = br_r31 / mva_ratio_31 + br_x31_sysbase = br_x31 / mva_ratio_31 + else + br_r31_sysbase = br_r31 * (Z_base_device_3 / Z_base_sys_3) + br_x31_sysbase = br_x31 * (Z_base_device_3 / Z_base_sys_3) + end + # See "Power System Stability and Control", ISBN: 0-07-035958-X, Eq. 6.72 + Zr_p = 1 / 2 * (br_r12_sysbase - br_r23_sysbase + br_r31_sysbase) + Zr_s = 1 / 2 * (br_r23_sysbase - br_r31_sysbase + br_r12_sysbase) + Zr_t = 1 / 2 * (br_r31_sysbase - br_r12_sysbase + br_r23_sysbase) + Zx_p = 1 / 2 * (br_x12_sysbase - br_x23_sysbase + br_x31_sysbase) + Zx_s = 1 / 2 * (br_x23_sysbase - br_x31_sysbase + br_x12_sysbase) + Zx_t = 1 / 2 * (br_x31_sysbase - br_x12_sysbase + br_x23_sysbase) + + # See PSSE Manual (Section 1.15.1 "Three-Winding Transformer Notes" of Data Formats file) + zero_names = [] + if isapprox(Zx_p, 0.0; atol = eps(Float32)) + push!(zero_names, "primary") + Zx_p = ZERO_IMPEDANCE_REACTANCE_THRESHOLD + end + if isapprox(Zx_s, 0.0; atol = eps(Float32)) + push!(zero_names, "secondary") + Zx_s = ZERO_IMPEDANCE_REACTANCE_THRESHOLD + end + if isapprox(Zx_t, 0.0; atol = eps(Float32)) + push!(zero_names, "tertiary") + Zx_t = ZERO_IMPEDANCE_REACTANCE_THRESHOLD + end + if !isempty(zero_names) + @info "Zero impedance reactance detected in 3W Transformer $(transformer["NAME"]) for winding(s): $(join(zero_names, ", ")). Setting to threshold value $(ZERO_IMPEDANCE_REACTANCE_THRESHOLD)." + end + + if iszero(Z_base_device_1) + Zr_p *= mva_ratio_12 + Zx_p *= mva_ratio_12 + else + Zr_p *= Z_base_sys_1 / Z_base_device_1 + Zx_p *= Z_base_sys_1 / Z_base_device_1 + end + if iszero(Z_base_device_2) + Zr_s *= mva_ratio_23 + Zx_s *= mva_ratio_23 + else + Zr_s *= Z_base_sys_2 / Z_base_device_2 + Zx_s *= Z_base_sys_2 / Z_base_device_2 + end + if iszero(Z_base_device_3) + Zr_t *= mva_ratio_31 + Zx_t *= mva_ratio_31 + else + Zr_t *= Z_base_sys_3 / Z_base_device_3 + Zx_t *= Z_base_sys_3 / Z_base_device_3 + end + + sub_data["name"] = transformer["NAME"] + sub_data["bus_primary"] = bus_id1 + sub_data["bus_secondary"] = bus_id2 + sub_data["bus_tertiary"] = bus_id3 + + sub_data["available"] = false + if transformer["STAT"] != 0 + sub_data["available"] = true + end + + sub_data["available_primary"] = true + sub_data["available_secondary"] = true + sub_data["available_tertiary"] = true + + if transformer["STAT"] == 2 + sub_data["available_secondary"] = false + end + + if transformer["STAT"] == 3 + sub_data["available_tertiary"] = false + end + + if transformer["STAT"] == 4 + sub_data["available_primary"] = false + end + + if transformer["STAT"] == 0 + sub_data["available_primary"] = false + sub_data["available_secondary"] = false + sub_data["available_tertiary"] = false + end + + sub_data["star_bus"] = starbus_id + + sub_data["active_power_flow_primary"] = 0.0 + sub_data["reactive_power_flow_primary"] = 0.0 + sub_data["active_power_flow_secondary"] = 0.0 + sub_data["reactive_power_flow_secondary"] = 0.0 + sub_data["active_power_flow_tertiary"] = 0.0 + sub_data["reactive_power_flow_tertiary"] = 0.0 + + sub_data["r_primary"] = Zr_p + sub_data["x_primary"] = Zx_p + sub_data["r_secondary"] = Zr_s + sub_data["x_secondary"] = Zx_s + sub_data["r_tertiary"] = Zr_t + sub_data["x_tertiary"] = Zx_t + + if pm_data["source_version"] ∈ ("32", "33") + sub_data["rating_primary"] = + min( + transformer["RATA1"], + transformer["RATB1"], + transformer["RATC1"], + ) + sub_data["rating_secondary"] = + min( + transformer["RATA2"], + transformer["RATB2"], + transformer["RATC2"], + ) + sub_data["rating_tertiary"] = + min( + transformer["RATA3"], + transformer["RATB3"], + transformer["RATC3"], + ) + sub_data["rating"] = min( + sub_data["rating_primary"], + sub_data["rating_secondary"], + sub_data["rating_tertiary"], + ) + elseif pm_data["source_version"] == "35" + sub_data["rating_primary"] = + min( + transformer["RATE11"], + transformer["RATE12"], + transformer["RATE13"], + ) + sub_data["rating_secondary"] = + min( + transformer["RATE21"], + transformer["RATE22"], + transformer["RATE23"], + ) + sub_data["rating_tertiary"] = + min( + transformer["RATE31"], + transformer["RATE32"], + transformer["RATE33"], + ) + sub_data["rating"] = min( + sub_data["rating_primary"], + sub_data["rating_secondary"], + sub_data["rating_tertiary"], + ) + else + error( + "Unsupported PSS(R)E source version: $(pm_data["source_version"])", + ) + end + + sub_data["r_12"] = br_r12 + sub_data["x_12"] = br_x12 + sub_data["r_23"] = br_r23 + sub_data["x_23"] = br_x23 + sub_data["r_13"] = br_r31 + sub_data["x_13"] = br_x31 + if transformer["CM"] == 1 + # Transform admittance to device per unit + mva_ratio_12 = sub_data["base_power_12"] / pm_data["baseMVA"] + sub_data["g"] = transformer["MAG1"] / mva_ratio_12 + sub_data["b"] = transformer["MAG2"] / mva_ratio_12 + else # CM=2: MAG1 are no load loss in Watts and MAG2 is the exciting current in pu, in device base. + @assert transformer["CM"] == 2 + G_pu, B_pu = _transformer_mag_pu_conversion( + transformer, + sub_data, + sub_data["base_power_12"], + ) + sub_data["g"] = G_pu + sub_data["b"] = B_pu + end + # If CM = 1 & MAG2 != 0 -> MAG2 < 0 + # If CM = 2 & MAG2 != 0 -> MAG2 > 0 + + sub_data["primary_correction_table"] = transformer["TAB1"] + sub_data["secondary_correction_table"] = transformer["TAB2"] + sub_data["tertiary_correction_table"] = transformer["TAB3"] + + sub_data["primary_phase_shift_angle"] = transformer["ANG1"] * pi / 180.0 + sub_data["secondary_phase_shift_angle"] = transformer["ANG2"] * pi / 180.0 + sub_data["tertiary_phase_shift_angle"] = transformer["ANG3"] * pi / 180.0 + + windv1 = transformer["WINDV1"] + windv2 = transformer["WINDV2"] + windv3 = transformer["WINDV3"] + + windv1 = apply_tap_correction!( + windv1, + transformer, + "COD1", + "RMI1", + "RMA1", + "NTP1", + transformer["CW"], + "primary", + ) + windv2 = apply_tap_correction!( + windv2, + transformer, + "COD2", + "RMI2", + "RMA2", + "NTP2", + transformer["CW"], + "secondary", + ) + windv3 = apply_tap_correction!( + windv3, + transformer, + "COD3", + "RMI3", + "RMA3", + "NTP3", + transformer["CW"], + "tertiary", + ) + + if transformer["CW"] == 1 + sub_data["primary_turns_ratio"] = windv1 + sub_data["secondary_turns_ratio"] = windv2 + sub_data["tertiary_turns_ratio"] = windv3 + elseif transformer["CW"] == 2 + sub_data["primary_turns_ratio"] = + windv1 / _get_bus_value(transformer["I"], "base_kv", pm_data) + sub_data["secondary_turns_ratio"] = + windv2 / _get_bus_value(transformer["J"], "base_kv", pm_data) + sub_data["tertiary_turns_ratio"] = + windv3 / _get_bus_value(transformer["K"], "base_kv", pm_data) + else + @assert transformer["CW"] == 3 + sub_data["primary_turns_ratio"] = + windv1 * ( + sub_data["base_voltage_primary"] / + _get_bus_value(transformer["I"], "base_kv", pm_data) + ) + sub_data["secondary_turns_ratio"] = + windv2 * ( + sub_data["base_voltage_secondary"] / + _get_bus_value(transformer["J"], "base_kv", pm_data) + ) + sub_data["tertiary_turns_ratio"] = + windv3 * ( + sub_data["base_voltage_tertiary"] / + _get_bus_value(transformer["K"], "base_kv", pm_data) + ) + end + sub_data["circuit"] = strip(transformer["CKT"]) + sub_data["COD1"] = transformer["COD1"] + sub_data["COD2"] = transformer["COD2"] + sub_data["COD3"] = transformer["COD3"] + + sub_data["ext"] = Dict{String, Any}( + "psse_name" => transformer["NAME"], + "CW" => transformer["CW"], + "CZ" => transformer["CZ"], + "CM" => transformer["CM"], + "MAG1" => transformer["MAG1"], + "MAG2" => transformer["MAG2"], + "VMSTAR" => transformer["VMSTAR"], + "ANSTAR" => transformer["ANSTAR"], + ) + + for prefix in TRANSFORMER3W_PARAMETER_NAMES + for i in 1:length(WINDING_NAMES) + key = "$prefix$i" + if pm_data["source_version"] ∈ ("32", "33") + sub_data["ext"][key] = transformer[key] + else + continue + end + end + end + + for suffix in ["1-2", "2-3", "3-1"] + sub_data["ext"]["R$suffix"] = transformer["R$suffix"] + sub_data["ext"]["X$suffix"] = transformer["X$suffix"] + end + + sub_data["index"] = length(pm_data["3w_transformer"]) + 1 + + if import_all + _import_remaining_keys!( + sub_data, + transformer; + exclude = [ + "NAME", + "STAT", + "MAG1", + "MAG2", + "WINDV1", + "WINDV2", + "WINDV3", + ], + ) + end + transformer3W_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data["3w_transformer"], sub_data) + + starbus_id += 1 # after adding the 1st 3WT, increase the counter + end + end + end + return +end + +""" + _psse2pm_dcline!(pm_data, pti_data) + +Parses PSS(R)E-style Two-Terminal and VSC DC Lines data into a PowerModels +compatible Dict structure by first converting them to a simple DC Line Model. +For Two-Terminal DC lines, "source_id" is given by `["IPR", "IPI", "NAME"]` in the +PSS(R)E Two-Terminal DC specification. For Voltage Source Converters, "source_id" +is given by `["IBUS1", "IBUS2", "NAME"]`, where "IBUS1" is "IBUS" of the first +converter bus, and "IBUS2" is the "IBUS" of the second converter bus, in the +PSS(R)E Voltage Source Converter specification. +""" +function _psse2pm_dcline!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Two-Terminal and VSC DC line data into a PowerModels Dict..." + pm_data["dcline"] = [] + pm_data["vscline"] = [] + baseMVA = pm_data["baseMVA"] + + if haskey(pti_data, "TWO-TERMINAL DC") + for dcline in pti_data["TWO-TERMINAL DC"] + sub_data = Dict{String, Any}() + + # Unit conversions? + power_demand = + if dcline["MDC"] == 1 + abs(dcline["SETVL"]) + elseif dcline["MDC"] == 2 + abs(dcline["SETVL"] * dcline["VSCHD"] / 1000) # Amp * V + else + 0 + end + + sub_data["transfer_setpoint"] = dcline["SETVL"] + + sub_data["name"] = strip(dcline["NAME"], ['"', '\'']) + sub_data["f_bus"] = dcline["IPR"] + sub_data["t_bus"] = dcline["IPI"] + if pm_data["has_isolated_type_buses"] + push!(pm_data["connected_buses"], sub_data["f_bus"]) + push!(pm_data["connected_buses"], sub_data["t_bus"]) + end + + if dcline["MDC"] == 1 + sub_data["power_mode"] = true + else + sub_data["power_mode"] = false + end + sub_data["available"] = dcline["MDC"] == 0 ? false : true + sub_data["br_status"] = sub_data["available"] + + sub_data["scheduled_dc_voltage"] = dcline["VSCHD"] + rectifier_base_voltage = dcline["EBASR"] + if rectifier_base_voltage == 0 + throw( + ArgumentError( + "DC line $(sub_data["name"]): Rectifier base voltage EBASER cannot be 0", + ), + ) + end + ZbaseR = rectifier_base_voltage^2 / baseMVA + sub_data["rectifier_bridges"] = dcline["NBR"] + sub_data["rectifier_rc"] = dcline["RCR"] / ZbaseR + sub_data["rectifier_xc"] = dcline["XCR"] / ZbaseR + sub_data["rectifier_base_voltage"] = rectifier_base_voltage + + inverter_base_voltage = dcline["EBASI"] + if inverter_base_voltage == 0 + throw( + ArgumentError( + "DC line $(sub_data["name"]): Inverter base voltage EBASI cannot be 0", + ), + ) + end + ZbaseI = inverter_base_voltage^2 / baseMVA + sub_data["inverter_bridges"] = dcline["NBI"] + sub_data["inverter_rc"] = dcline["RCI"] / ZbaseI + sub_data["inverter_xc"] = dcline["XCI"] / ZbaseI + sub_data["inverter_base_voltage"] = inverter_base_voltage + + sub_data["switch_mode_voltage"] = dcline["VCMOD"] + sub_data["compounding_resistance"] = dcline["RCOMP"] + sub_data["min_compounding_voltage"] = dcline["DCVMIN"] + + sub_data["rectifier_transformer_ratio"] = dcline["TRR"] + sub_data["rectifier_tap_setting"] = dcline["TAPR"] + sub_data["rectifier_tap_limits"] = (min = dcline["TMNR"], max = dcline["TMXR"]) + sub_data["rectifier_tap_step"] = dcline["STPR"] + + sub_data["inverter_transformer_ratio"] = dcline["TRI"] + sub_data["inverter_tap_setting"] = dcline["TAPI"] + sub_data["inverter_tap_limits"] = (min = dcline["TMNI"], max = dcline["TMXI"]) + sub_data["inverter_tap_step"] = dcline["STPI"] + + sub_data["loss0"] = 0.0 + sub_data["loss1"] = 0.0 + + sub_data["pf"] = power_demand + sub_data["active_power_flow"] = sub_data["pf"] + sub_data["pt"] = power_demand + sub_data["qf"] = 0.0 + sub_data["qt"] = 0.0 + sub_data["vf"] = _get_bus_value(pop!(dcline, "IPR"), "vm", pm_data) + sub_data["vt"] = _get_bus_value(pop!(dcline, "IPI"), "vm", pm_data) + + sub_data["pminf"] = 0.0 + sub_data["pmaxf"] = dcline["SETVL"] > 0 ? power_demand : -power_demand + sub_data["pmint"] = pop!(dcline, "SETVL") > 0 ? -power_demand : power_demand + sub_data["pmaxt"] = 0.0 + + anmn = [] + for key in ["ANMNR", "ANMNI"] + if abs(dcline[key]) <= 90.0 + push!(anmn, dcline[key]) + else + push!(anmn, 0) + @info("$key outside reasonable limits, setting to 0 degress") + end + end + sub_data["rectifier_delay_angle_limits"] = + (min = deg2rad(anmn[1]), max = deg2rad(dcline["ANMXR"])) + sub_data["inverter_extinction_angle_limits"] = + (min = deg2rad(anmn[2]), max = deg2rad(dcline["ANMXI"])) + + sub_data["rectifier_delay_angle"] = deg2rad(anmn[1]) + sub_data["inverter_extinction_angle"] = deg2rad(anmn[2]) + + sub_data["qmaxf"] = 0.0 + sub_data["qmaxt"] = 0.0 + sub_data["qminf"] = + -max(abs(sub_data["pminf"]), abs(sub_data["pmaxf"])) * cosd(anmn[1]) + sub_data["qmint"] = + -max(abs(sub_data["pmint"]), abs(sub_data["pmaxt"])) * cosd(anmn[2]) + + sub_data["active_power_limits_from"] = + (min = sub_data["pminf"], max = sub_data["pmaxf"]) + sub_data["active_power_limits_to"] = + (min = sub_data["pmint"], max = sub_data["pmaxt"]) + sub_data["reactive_power_limits_from"] = + (min = sub_data["qminf"], max = sub_data["qmaxf"]) + sub_data["reactive_power_limits_to"] = + (min = sub_data["qmint"], max = sub_data["qmaxt"]) + + sub_data["rectifier_capacitor_reactance"] = dcline["XCAPR"] / ZbaseR + sub_data["inverter_capacitor_reactance"] = dcline["XCAPI"] / ZbaseI + sub_data["r"] = dcline["RDC"] / ZbaseR + + if pm_data["source_version"] ∈ ("32", "33") + sub_data["ext"] = Dict{String, Any}( + "psse_name" => dcline["NAME"], + ) + elseif pm_data["source_version"] == "35" + sub_data["ext"] = Dict{String, Any}( + "NDR" => dcline["NDR"], + "NDI" => dcline["NDI"], + ) + else + error("Unsupported PSS(R)E source version: $(pm_data["source_version"])") + end + + sub_data["source_id"] = [ + "two-terminal dc", + sub_data["f_bus"], + sub_data["t_bus"], + pop!(dcline, "NAME"), + ] + sub_data["index"] = length(pm_data["dcline"]) + 1 + + if import_all + _import_remaining_keys!(sub_data, dcline) + end + branch_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data["dcline"], sub_data) + end + end + + if haskey(pti_data, "VOLTAGE SOURCE CONVERTER") + for dcline in pti_data["VOLTAGE SOURCE CONVERTER"] + # Converter buses : is the distinction between ac and dc side meaningful? + from_bus, to_bus = dcline["CONVERTER BUSES"] + + # PowerWorld conversion from PTI to matpower seems to create two + # artificial generators from a VSC, but it is not clear to me how + # the value of "pg" is determined and adds shunt to the DC-side bus. + sub_data = Dict{String, Any}() + sub_data["name"] = strip(dcline["NAME"], ['"', '\'']) + + # VSC intended to be one or bi-directional? + sub_data["f_bus"] = from_bus["IBUS"] + sub_data["t_bus"] = to_bus["IBUS"] + if pm_data["has_isolated_type_buses"] + push!(pm_data["connected_buses"], sub_data["f_bus"]) + push!(pm_data["connected_buses"], sub_data["t_bus"]) + end + sub_data["br_status"] = + if dcline["MDC"] == 0 || + from_bus["TYPE"] == 0 || + to_bus["TYPE"] == 0 + 0 + else + 1 + end + sub_data["available"] = sub_data["br_status"] == 0 ? false : true + + sub_data["dc_voltage_control_from"] = from_bus["TYPE"] == 1 ? true : false + sub_data["dc_voltage_control_to"] = to_bus["TYPE"] == 1 ? true : false + sub_data["ac_voltage_control_from"] = from_bus["MODE"] == 1 ? true : false + sub_data["ac_voltage_control_to"] = to_bus["MODE"] == 1 ? true : false + + sub_data["dc_setpoint_from"] = from_bus["DCSET"] + sub_data["dc_setpoint_to"] = to_bus["DCSET"] + sub_data["ac_setpoint_from"] = from_bus["ACSET"] + sub_data["ac_setpoint_to"] = to_bus["ACSET"] + + # ALOSS, MINLOSS in kW, and BLOSS in kW/A. Divide by a 1000 to transform into MW, and divide by baseMVA to normalize to per-unit. + sub_data["converter_loss_from"] = LinearCurve( + from_bus["BLOSS"] / (1000.0 * baseMVA), + (from_bus["ALOSS"] + from_bus["MINLOSS"]) / (1000.0 * baseMVA), + ) + sub_data["converter_loss_to"] = LinearCurve( + to_bus["BLOSS"] / (1000.0 * baseMVA), + (to_bus["ALOSS"] + to_bus["MINLOSS"]) / (1000.0 * baseMVA), + ) + + sub_data["pf"] = 0.0 + sub_data["pt"] = 0.0 + + sub_data["qf"] = 0.0 + sub_data["qt"] = 0.0 + + sub_data["qminf"] = from_bus["MINQ"] / baseMVA + sub_data["qmaxf"] = from_bus["MAXQ"] / baseMVA + sub_data["qmint"] = to_bus["MINQ"] / baseMVA + sub_data["qmaxt"] = to_bus["MAXQ"] / baseMVA + + PTI_INF = 9999.0 + + sub_data["rating_from"] = + from_bus["SMAX"] == 0.0 ? PTI_INF : from_bus["SMAX"] / baseMVA + sub_data["rating_to"] = + to_bus["SMAX"] == 0.0 ? PTI_INF : to_bus["SMAX"] / baseMVA + sub_data["rating"] = min(sub_data["rating_from"], sub_data["rating_to"]) + sub_data["max_dc_current_from"] = + from_bus["IMAX"] == 0.0 ? PTI_INF : from_bus["IMAX"] + sub_data["max_dc_current_to"] = to_bus["IMAX"] == 0.0 ? PTI_INF : to_bus["IMAX"] + sub_data["power_factor_weighting_fraction_from"] = from_bus["PWF"] + sub_data["power_factor_weighting_fraction_to"] = to_bus["PWF"] + qmax_from = max(abs(sub_data["qminf"]), abs(sub_data["qmaxf"])) + qmax_to = max(abs(sub_data["qmint"]), abs(sub_data["qmaxt"])) + sub_data["pmaxf"] = sqrt(sub_data["rating_from"]^2 - qmax_from^2) + sub_data["pmaxt"] = sqrt(sub_data["rating_to"]^2 - qmax_to^2) + sub_data["pminf"] = -sub_data["pmaxf"] + sub_data["pmint"] = -sub_data["pmaxt"] + + if sub_data["dc_voltage_control_from"] && !sub_data["dc_voltage_control_to"] + base_voltage = sub_data["dc_setpoint_from"] + flow_setpoint = sub_data["dc_setpoint_to"] + elseif !sub_data["dc_voltage_control_from"] && sub_data["dc_voltage_control_to"] + base_voltage = sub_data["dc_setpoint_to"] + flow_setpoint = -sub_data["dc_setpoint_from"] + else + error( + "At least one converter in converter $(sub_data["name"]) must set a voltage control.", + ) + end + Zbase = base_voltage^2 / baseMVA + sub_data["r"] = dcline["RDC"] / Zbase + sub_data["pf"] = flow_setpoint / baseMVA + sub_data["if"] = 1000.0 * (flow_setpoint / base_voltage) + + sub_data["ext"] = Dict{String, Any}( + "REMOT_FROM" => from_bus["REMOT"], + "REMOT_TO" => to_bus["REMOT"], + "RMPCT_FROM" => from_bus["RMPCT"], + "RMPCT_TO" => to_bus["RMPCT"], + "ALOSS_FROM" => from_bus["ALOSS"], + "ALOSS_TO" => to_bus["ALOSS"], + "MINLOSS_FROM" => from_bus["MINLOSS"], + "MINLOSS_TO" => to_bus["MINLOSS"], + "TYPE_FROM" => from_bus["TYPE"], + "TYPE_TO" => to_bus["TYPE"], + "MODE_FROM" => from_bus["MODE"], + "MODE_TO" => to_bus["MODE"], + "RDC" => dcline["RDC"], + ) + + sub_data["source_id"] = + ["vsc dc", sub_data["f_bus"], sub_data["t_bus"], dcline["NAME"]] + sub_data["index"] = length(pm_data["vscline"]) + 1 + + if import_all + _import_remaining_keys!(sub_data, dcline) + + for cb in sub_data["converter buses"] + for (k, v) in cb + cb[lowercase(k)] = v + delete!(cb, k) + end + end + end + branch_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data["vscline"], sub_data) + end + end +end + +function _psse2pm_facts!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E FACTs devices data into a PowerModels Dict..." + pm_data["facts"] = [] + + if haskey(pti_data, "FACTS CONTROL DEVICE") + for facts in pti_data["FACTS CONTROL DEVICE"] + @info( + """FACTs are supported via a simplification approach for terminal_bus = 0 (STATCOM operation)""" + ) + sub_data = Dict{String, Any}() + + sub_data["name"] = strip(facts["NAME"], ['"', '\'']) + sub_data["control_mode"] = facts["MODE"] + + sub_data["bus"] = facts["I"] # Sending end bus number + sub_data["tbus"] = facts["J"] # Terminal end bus number + sub_data["available"] = _determine_injector_status( + facts, + pm_data, + sub_data["bus"], + "MODE", + "candidate_isolated_to_pq_buses", + ) + + sub_data["voltage_setpoint"] = facts["VSET"] + sub_data["max_shunt_current"] = facts["SHMX"] + + # % of MVAr required to hold voltage at sending bus + if facts["RMPCT"] < 0 + @warn "% MVAr required must me positive." + end + + sub_data["reactive_power_required"] = facts["RMPCT"] + sub_data["ext"] = Dict{String, Any}() + + if pm_data["source_version"] == "35" + sub_data["ext"]["NREG"] = facts["NREG"] + sub_data["ext"]["MNAME"] = facts["MNAME"] + elseif pm_data["source_version"] ∈ ("32", "33") + sub_data["ext"] = Dict{String, Any}( + "J" => facts["J"], + ) + else + error("Unsupported PSS(R)E source version: $(pm_data["source_version"])") + end + + sub_data["source_id"] = + ["facts", sub_data["bus"], sub_data["name"]] + sub_data["index"] = length(pm_data["facts"]) + 1 + + if import_all + _import_remaining_keys!(sub_data, facts) + end + push!(pm_data["facts"], sub_data) + end + end + return +end + +function _build_switch_breaker_sub_data( + pm_data::Dict, + dict_object::Dict, + device_type::String, + discrete_device_type::Int, + index::Int, +) + sub_data = Dict{String, Any}() + + sub_data["f_bus"] = pop!(dict_object, "I") + sub_data["t_bus"] = pop!(dict_object, "J") + if pm_data["has_isolated_type_buses"] + push!(pm_data["connected_buses"], sub_data["f_bus"]) + push!(pm_data["connected_buses"], sub_data["t_bus"]) + end + + sub_data["x"] = pop!(dict_object, "X") + sub_data["active_power_flow"] = 0.0 + sub_data["reactive_power_flow"] = 0.0 + sub_data["psw"] = sub_data["active_power_flow"] + sub_data["qsw"] = sub_data["reactive_power_flow"] + sub_data["discrete_branch_type"] = discrete_device_type + sub_data["ext"] = Dict{String, Any}() + + if haskey(dict_object, "STAT") + sub_data["state"] = pop!(dict_object, "STAT") + elseif haskey(dict_object, "ST") + sub_data["state"] = pop!(dict_object, "ST") + else + @warn "No STAT or ST field found in the data for switch/breaker. Assuming it is off." + sub_data["state"] = 0.0 + end + + if pm_data["source_version"] == ("35") + sub_data["r"] = 0.0 + sub_data["rating"] = pop!(dict_object, "RATE1") + for i in 2:12 + rate_key = "RATE$i" + if haskey(dict_object, rate_key) + sub_data["ext"][rate_key] = pop!(dict_object, rate_key) + end + end + else + sub_data["r"] = pop!(dict_object, "R") + sub_data["rating"] = pop!(dict_object, "RATEA") + end + + sub_data["source_id"] = + [device_type, sub_data["f_bus"], sub_data["t_bus"], dict_object["CKT"]] + sub_data["index"] = index + + return sub_data +end + +function _psse2pm_switch_breaker!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Switches & Breakers data into a PowerModels Dict..." + pm_data["breaker"] = [] + pm_data["switch"] = [] + mapping = Dict('@' => ("breaker", 1), '*' => ("switch", 0)) + mapping_v35 = Dict(2 => "breaker", 3 => "switch") + + # Always check for legacy entries in PSSe 35 for switches and breakers set as @ or * + if haskey(pti_data, "SWITCHES_AS_BRANCHES") + for branch in pti_data["SWITCHES_AS_BRANCHES"] + branch_init = first(branch["CKT"]) + + # Check if character is in the mapping + if haskey(mapping, branch_init) + branch_type, discrete_branch_type = mapping[branch_init] + + sub_data = _build_switch_breaker_sub_data( + pm_data, + branch, + branch_type, + discrete_branch_type, + length(pm_data[branch_type]) + 1, + ) + + if import_all + _import_remaining_keys!(sub_data, branch) + end + branch_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data[branch_type], sub_data) + end + end + end + + if haskey(pti_data, "SWITCHING DEVICE") + if pm_data["source_version"] == "35" + for switching_device in pti_data["SWITCHING DEVICE"] + device_type = get(mapping_v35, switching_device["STYPE"], "other") + discrete_branch_type = + device_type == "breaker" ? 1 : (device_type == "switch" ? 0 : 2) + + sub_data = _build_switch_breaker_sub_data( + pm_data, + switching_device, + device_type, + discrete_branch_type, + length(pm_data[device_type]) + 1, + ) + + if import_all + _import_remaining_keys!(sub_data, branch) + end + + branch_isolated_bus_modifications!(pm_data, sub_data) + push!(pm_data[device_type], sub_data) + end + else + error("Unsupported PSS(R)E source version: $(pm_data["source_version"])") + end + end + return +end + +function _psse2pm_multisection_line!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Adding PSS(R)E Multi-section Lines data into the branches PowerModels Dict..." + branch_lookup = Dict{Tuple{Int, Int}, Int}() + if haskey(pm_data, "branch") + for branch in pm_data["branch"] + branch_lookup[(branch["f_bus"], branch["t_bus"])] = branch["index"] + end + end + if haskey(pti_data, "MULTI-SECTION LINE") + for multisec_line in pti_data["MULTI-SECTION LINE"] + filter!(x -> x.second != "", multisec_line) + f_bus = multisec_line["I"] + t_bus = multisec_line["J"] + id = filter(isdigit, multisec_line["ID"]) + # Sort by dummy bus index + dummy_buses = sort([ + (k, v) for (k, v) in multisec_line if startswith(k, "DUM") && v != "" + ]) + dummy_bus_numbers = [x[2] for x in dummy_buses] + all_buses = [f_bus; dummy_bus_numbers; t_bus] + for ix in 1:(length(all_buses) - 1) + branch_index = nothing + if haskey(branch_lookup, (all_buses[ix], all_buses[ix + 1])) + branch_index = branch_lookup[(all_buses[ix], all_buses[ix + 1])] + elseif haskey(branch_lookup, (all_buses[ix + 1], all_buses[ix])) + branch_index = branch_lookup[(all_buses[ix + 1], all_buses[ix])] + else + @warn "Branch between buses $(all_buses[ix]) and $(all_buses[ix + 1]) not found in branch data. Skipping segment." + continue + end + # Proceed if a valid branch is found + if branch_index !== nothing + ext = get(pm_data["branch"][branch_index], "ext", Dict{String, Any}()) + ext["from_multisection"] = true + ext["multisection_psse_entry"] = multisec_line + pm_data["branch"][branch_index]["ext"] = ext + end + end + end + end + return +end + +function sort_values_by_key_prefix(imp_correction::Dict{String, <:Any}, prefix::String) + sorted_values = [ + last(pair) for pair in sort( + [ + (parse(Int, k[2:end]), v) for + (k, v) in imp_correction if startswith(k, prefix) + ]; + by = first, + ) + ] + + first_non_zero_index = findfirst(x -> x != 0.0, reverse(sorted_values)) + sorted_values = sorted_values[1:(length(sorted_values) - first_non_zero_index + 1)] + + return sorted_values +end + +function sort_values_by_key_prefix_v35(imp_correction::Dict{String, <:Any}, prefix::String) + # For v35, we need to handle "Re(F1)", "Im(F1)" format + sorted_values = [ + last(pair) for pair in sort( + [ + (parse(Int, match(r"\d+", k).match), v) for + (k, v) in imp_correction if startswith(k, prefix) && contains(k, r"\d+") + ]; + by = first, + ) + ] + + # Remove trailing zeros in IC data that indicate end of entry (0.00000,0.00000,0.00000) + # Acoording to PSSE DataFormat Section 1.18. TIC Tables + # Check if the last entry is all zeros across (T, Re(F), Im(F)) + while !isempty(sorted_values) + last_index = length(sorted_values) + keys_to_check = ["T$last_index", "Re(F$last_index)", "Im(F$last_index)"] + + if all(k -> haskey(imp_correction, k) && imp_correction[k] == 0.0, keys_to_check) + pop!(sorted_values) + else + break + end + end + + return sorted_values +end + +function _psse2pm_impedance_correction!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @info "Parsing PSS(R)E Transformer Impedance Correction Tables data into a PowerModels Dict..." + + pm_data["impedance_correction"] = [] + + if haskey(pti_data, "IMPEDANCE CORRECTION") + for imp_correction in pti_data["IMPEDANCE CORRECTION"] + sub_data = Dict{String, Any}() + + sub_data["table_number"] = imp_correction["I"] + + is_v35 = + any(k -> contains(k, "Re(F") || contains(k, "Im(F"), keys(imp_correction)) + + if is_v35 + sub_data["scaling_factor"] = + sort_values_by_key_prefix_v35(imp_correction, "Re(F") + sub_data["scaling_factor_imag"] = + sort_values_by_key_prefix_v35(imp_correction, "Im(F") + sub_data["tap_or_angle"] = + sort_values_by_key_prefix_v35(imp_correction, "T") + else + sub_data["scaling_factor"] = sort_values_by_key_prefix(imp_correction, "F") + sub_data["tap_or_angle"] = sort_values_by_key_prefix(imp_correction, "T") + end + + sub_data["index"] = length(pm_data["impedance_correction"]) + 1 + + if import_all + _import_remaining_keys!(sub_data, imp_correction) + end + + push!(pm_data["impedance_correction"], sub_data) + end + end + return +end + +function _psse2pm_substation_data!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @warn "Parsing PSS(R)E Substation data into a PowerModels Dict..." + pm_data["substation_data"] = [] + + if haskey(pti_data, "SUBSTATION DATA") + for substation_data in pti_data["SUBSTATION DATA"] + sub_data = Dict{String, Any}() + + sub_data["name"] = substation_data["NAME"] + sub_data["substation_is"] = substation_data["IS"] + + sub_data["latitude"] = substation_data["LATITUDE"] + sub_data["longitude"] = substation_data["LONGITUDE"] + sub_data["nodes"] = substation_data["NODES"] + + if import_all + _import_remaining_keys!(sub_data, substation_data) + end + + sub_data["index"] = length(pm_data["substation_data"]) + 1 + push!(pm_data["substation_data"], sub_data) + end + end +end + +function _psse2pm_storage!(pm_data::Dict, pti_data::Dict, import_all::Bool) + @warn "This PSS(R)E parser currently doesn't support Storage data parsing..." + pm_data["storage"] = [] + return +end + +""" + _pti_to_powermodels!(pti_data) + +Converts PSS(R)E-style data parsed from a PTI raw file, passed by `pti_data` +into a format suitable for use internally in PowerModels. Imports all remaining +data from the PTI file if `import_all` is true (Default: false). +""" +function _pti_to_powermodels!( + pti_data::Dict; + import_all = false, + validate = true, + correct_branch_rating = true, +)::Dict + pm_data = Dict{String, Any}() + + rev = pop!(pti_data["CASE IDENTIFICATION"][1], "REV") + + pm_data["per_unit"] = false + pm_data["source_type"] = "pti" + pm_data["source_version"] = "$rev" + pm_data["baseMVA"] = pop!(pti_data["CASE IDENTIFICATION"][1], "SBASE") + pm_data["name"] = pop!(pti_data["CASE IDENTIFICATION"][1], "NAME") + + if import_all + _import_remaining_keys!(pm_data, pti_data["CASE IDENTIFICATION"][1]) + end + + _psse2pm_interarea_transfer!(pm_data, pti_data, import_all) + _psse2pm_area_interchange!(pm_data, pti_data, import_all) + _psse2pm_zone!(pm_data, pti_data, import_all) + # Order matters here. Buses need to parsed first + _psse2pm_bus!(pm_data, pti_data, import_all) + # Branches need to be parsed after buses to find topologically connected buses + _psse2pm_branch!(pm_data, pti_data, import_all) + _psse2pm_switch_breaker!(pm_data, pti_data, import_all) + _psse2pm_multisection_line!(pm_data, pti_data, import_all) + _psse2pm_transformer!(pm_data, pti_data, import_all) + # Injectors need to be parsed after branches and transformers to find topologically connected buses + _psse2pm_load!(pm_data, pti_data, import_all) + _psse2pm_shunt!(pm_data, pti_data, import_all) + _psse2pm_generator!(pm_data, pti_data, import_all) + _psse2pm_facts!(pm_data, pti_data, import_all) + + _psse2pm_dcline!(pm_data, pti_data, import_all) + _psse2pm_impedance_correction!(pm_data, pti_data, import_all) + _psse2pm_substation_data!(pm_data, pti_data, import_all) + _psse2pm_storage!(pm_data, pti_data, import_all) + + if pm_data["has_isolated_type_buses"] + bus_numbers = Set(v["bus_i"] for (_, v) in pm_data["bus"]) + topologically_isolated_buses = setdiff(bus_numbers, pm_data["connected_buses"]) + convert_to_pq = + setdiff( + pm_data["candidate_isolated_to_pq_buses"], + pm_data["candidate_isolated_to_pv_buses"], + ) + convert_to_pv = pm_data["candidate_isolated_to_pv_buses"] + + for b in setdiff!(convert_to_pq, topologically_isolated_buses) + pm_data["bus"][b]["bus_type"] = 1 + end + for b in setdiff!(convert_to_pv, topologically_isolated_buses) + pm_data["bus"][b]["bus_type"] = 2 + end + + if !isempty(topologically_isolated_buses) + for b in topologically_isolated_buses + if pm_data["bus"][b]["bus_type"] == 4 + continue + else + b_number = pm_data["bus"][b]["bus_i"] + b_type = pm_data["bus"][b]["bus_type"] + if b_type == 3 + error( + "PSEE reference bus $(b_number) that is topologically isolated from the system. Indicates an error in the data.", + ) + end + @error "PSEE data file contains a topologically isolated bus $(b_number) that is disconnected from the system and set to bus_type = $(b_type) instead of 4. Likely indicates an error in the data." + pm_data["bus"][b]["bus_type"] = 4 + pm_data["bus"][b]["bus_status"] = false + end + end + end + end + + if import_all + _import_remaining_comps!( + pm_data, + pti_data; + exclude = [ + "CASE IDENTIFICATION", + "BUS", + "LOAD", + "FIXED SHUNT", + "SWITCHED SHUNT", + "GENERATOR", + "BRANCH", + "TRANSFORMER", + "TWO-TERMINAL DC", + "VOLTAGE SOURCE CONVERTER", + ], + ) + end + + # update lookup structure + for (k, v) in pm_data + if isa(v, Array) + dict = Dict{String, Any}() + for item in v + @assert("index" in keys(item)) + dict[string(item["index"])] = item + end + pm_data[k] = dict + end + end + + if validate + correct_network_data!(pm_data; correct_branch_rating = correct_branch_rating) + end + + return pm_data +end + +""" + parse_psse(filename::String; kwargs...)::Dict + +Parses directly from file +""" +function parse_psse(filename::String; kwargs...)::Dict + pm_data = open(filename) do f + parse_psse(f; kwargs...) + end + + return pm_data +end + +""" + function parse_psse(io::IO; kwargs...)::Dict + +Parses directly from iostream +""" +function parse_psse(io::IO; kwargs...)::Dict + @info( + "The PSS(R)E parser currently supports buses, loads, shunts, generators, branches, switches, breakers, IC tables, transformers, facts, and dc lines", + ) + pti_data = parse_pti(io) + pm = _pti_to_powermodels!(pti_data; kwargs...) + return pm +end diff --git a/src/pm_io/pti.jl b/src/pm_io/pti.jl new file mode 100644 index 0000000..81f5a49 --- /dev/null +++ b/src/pm_io/pti.jl @@ -0,0 +1,2677 @@ +##################################################################### +# # +# This file provides functions for interfacing with pti .raw files # +# # +##################################################################### + +""" +A list of data file sections in the order that they appear in a PTI v33/v35 file +""" +const _pti_sections = [ + "CASE IDENTIFICATION", + "BUS", + "LOAD", + "FIXED SHUNT", + "GENERATOR", + "BRANCH", + "TRANSFORMER", + "AREA INTERCHANGE", + "TWO-TERMINAL DC", + "VOLTAGE SOURCE CONVERTER", + "IMPEDANCE CORRECTION", + "MULTI-TERMINAL DC", + "MULTI-SECTION LINE", + "ZONE", + "INTER-AREA TRANSFER", + "OWNER", + "FACTS CONTROL DEVICE", + "SWITCHED SHUNT", + "GNE DEVICE", + "INDUCTION MACHINE", +] + +const _pti_sections_v35 = vcat( + _pti_sections[1:6], + ["SWITCHING DEVICE"], + _pti_sections[7:end], + "SUBSTATION DATA", +) + +const _transaction_dtypes = [ + ("IC", Int64), + ("SBASE", Float64), + ("REV", Int64), + ("XFRRAT", Float64), + ("NXFRAT", Float64), + ("BASFRQ", Float64), +] + +const _transaction_dtypes_v35 = _transaction_dtypes + +const _system_wide_dtypes_v35 = [ + ("THRSHZ", Float64), + ("PQBRAK", Float64), + ("BLOWUP", Float64), + ("MAXISOLLVLS", Int64), + ("CAMAXREPTSLN", Int64), + ("CHKDUPCNTLBL", Int64), + ("ITMX", Int64), + ("ACCP", Float64), + ("ACCQ", Float64), + ("ACCM", Float64), + ("TOL", Float64), + ("ITMXN", Int64), + ("ACCN", Float64), + ("TOLN", Float64), + ("VCTOLQ", Float64), + ("VCTOLV", Float64), + ("DVLIM", Float64), + ("NDVFCT", Float64), + ("ADJTHR", Float64), + ("ACCTAP", Float64), + ("TAPLIM", Float64), + ("SWVBND", Float64), + ("MXTPSS", Int64), + ("MXSWIM", Int64), + ("ITMXTY", Int64), + ("ACCTY", Float64), + ("TOLTY", Float64), + ("METHOD", String), + ("ACTAPS", Int64), + ("AREAIN", Int64), + ("PHSHFT", Int64), + ("DCTAPS", Int64), + ("SWSHNT", Int64), + ("FLATST", Int64), + ("VARLIM", Int64), + ("NONDIV", Int64), + ("IRATE", Int64), + ("NAME", String), + ("DESC", String), +] + +const _system_wide_data_sections_v35 = Dict{String, Vector{String}}( + "GENERAL" => + ["THRSHZ", "PQBRAK", "BLOWUP", "MAXISOLLVLS", "CAMAXREPTSLN", "CHKDUPCNTLBL"], + "GAUSS" => ["ITMX", "ACCP", "ACCQ", "ACCM", "TOL"], + "NEWTON" => ["ITMXN", "ACCN", "TOLN", "VCTOLQ", "VCTOLV", "DVLIM", "NDVFCT"], + "ADJUST" => ["ADJTHR", "ACCTAP", "TAPLIM", "SWVBND", "MXTPSS", "MXSWIM"], + "TYSL" => ["ITMXTY", "ACCTY", "TOLTY"], + "SOLVER" => [ + "METHOD", + "ACTAPS", + "AREAIN", + "PHSHFT", + "DCTAPS", + "SWSHNT", + "FLATST", + "VARLIM", + "NONDIV", + ], + "RATING" => ["IRATE", "NAME", "DESC"], +) + +const _bus_dtypes = [ + ("I", Int64), + ("NAME", String), + ("BASKV", Float64), + ("IDE", Int64), + ("AREA", Int64), + ("ZONE", Int64), + ("OWNER", Int64), + ("VM", Float64), + ("VA", Float64), + ("NVHI", Float64), + ("NVLO", Float64), + ("EVHI", Float64), + ("EVLO", Float64), +] + +const _bus_dtypes_v35 = _bus_dtypes + +const _load_dtypes = [ + ("I", Int64), + ("ID", String), + ("STATUS", Int64), + ("AREA", Int64), + ("ZONE", Int64), + ("PL", Float64), + ("QL", Float64), + ("IP", Float64), + ("IQ", Float64), + ("YP", Float64), + ("YQ", Float64), + ("OWNER", Int64), + ("SCALE", Int64), + ("INTRPT", Int64), +] + +const _load_dtypes_v35 = vcat( + _load_dtypes[1:14], + [ + ("DGENP", Float64), + ("DGENQ", Float64), + ("DGENM", Float64), + ("LOADTYPE", String), + ], +) + +const _fixed_shunt_dtypes = [ + ("I", Int64), + ("ID", String), + ("STATUS", Int64), + ("GL", Float64), + ("BL", Float64), +] + +const _fixed_shunt_dtypes_v35 = _fixed_shunt_dtypes + +const _generator_dtypes = [ + ("I", Int64), + ("ID", String), + ("PG", Float64), + ("QG", Float64), + ("QT", Float64), + ("QB", Float64), + ("VS", Float64), + ("IREG", Int64), + ("MBASE", Float64), + ("ZR", Float64), + ("ZX", Float64), + ("RT", Float64), + ("XT", Float64), + ("GTAP", Float64), + ("STAT", Int64), + ("RMPCT", Float64), + ("PT", Float64), + ("PB", Float64), + ("O1", Int64), + ("F1", Float64), + ("O2", Int64), + ("F2", Float64), + ("O3", Int64), + ("F3", Float64), + ("O4", Int64), + ("F4", Float64), + ("WMOD", Int64), + ("WPF", Float64), +] + +const _generator_dtypes_v35 = vcat( + _generator_dtypes[1:8], + [("NREG", Int64)], + _generator_dtypes[9:18], + [("BASLOD", Int64)], + _generator_dtypes[19:end], +) + +const _branch_dtypes = [ + ("I", Int64), + ("J", Int64), + ("CKT", String), + ("R", Float64), + ("X", Float64), + ("B", Float64), + ("RATEA", Float64), + ("RATEB", Float64), + ("RATEC", Float64), + ("GI", Float64), + ("BI", Float64), + ("GJ", Float64), + ("BJ", Float64), + ("ST", Int64), + ("MET", Int64), + ("LEN", Float64), + ("O1", Int64), + ("F1", Float64), + ("O2", Int64), + ("F2", Float64), + ("O3", Int64), + ("F3", Float64), + ("O4", Int64), + ("F4", Float64), +] + +const _branch_dtypes_v35 = vcat( + _branch_dtypes[1:6], + [ + ("NAME", String), + ("RATE1", Float64), + ("RATE2", Float64), + ("RATE3", Float64), + ("RATE4", Float64), + ("RATE5", Float64), + ("RATE6", Float64), + ("RATE7", Float64), + ("RATE8", Float64), + ("RATE9", Float64), + ("RATE10", Float64), + ("RATE11", Float64), + ("RATE12", Float64), + ], + _branch_dtypes[10:end], +) + +const _switching_dtypes_v35 = [ + ("I", Int64), + ("J", Int64), + ("CKT", String), + ("X", Float64), + ("RATE1", Float64), + ("RATE2", Float64), + ("RATE3", Float64), + ("RATE4", Float64), + ("RATE5", Float64), + ("RATE6", Float64), + ("RATE7", Float64), + ("RATE8", Float64), + ("RATE9", Float64), + ("RATE10", Float64), + ("RATE11", Float64), + ("RATE12", Float64), + ("STAT", Int64), + ("NSTAT", Int64), + ("MET", Int64), + ("STYPE", Int64), + ("NAME", String), +] + +const _transformer_dtypes = [ + ("I", Int64), + ("J", Int64), + ("K", Int64), + ("CKT", String), + ("CW", Int64), + ("CZ", Int64), + ("CM", Int64), + ("MAG1", Float64), + ("MAG2", Float64), + ("NMETR", Int64), + ("NAME", String), + ("STAT", Int64), + ("O1", Int64), + ("F1", Float64), + ("O2", Int64), + ("F2", Float64), + ("O3", Int64), + ("F3", Float64), + ("O4", Int64), + ("F4", Float64), + ("VECGRP", String), +] + +const _transformer_dtypes_v35 = _transformer_dtypes + +const _transformer_3_1_dtypes = [ + ("R1-2", Float64), + ("X1-2", Float64), + ("SBASE1-2", Float64), + ("R2-3", Float64), + ("X2-3", Float64), + ("SBASE2-3", Float64), + ("R3-1", Float64), + ("X3-1", Float64), + ("SBASE3-1", Float64), + ("VMSTAR", Float64), + ("ANSTAR", Float64), +] + +const _transformer_3_1_dtypes_v35 = _transformer_3_1_dtypes + +const _transformer_3_2_dtypes = [ + ("WINDV1", Float64), + ("NOMV1", Float64), + ("ANG1", Float64), + ("RATA1", Float64), + ("RATB1", Float64), + ("RATC1", Float64), + ("COD1", Int64), + ("CONT1", Int64), + ("RMA1", Float64), + ("RMI1", Float64), + ("VMA1", Float64), + ("VMI1", Float64), + ("NTP1", Float64), + ("TAB1", Int64), + ("CR1", Float64), + ("CX1", Float64), + ("CNXA1", Float64), +] + +const _transformer_3_2_dtypes_v35 = vcat( + _transformer_3_2_dtypes[1:3], + [ + ("RATE11", Float64), + ("RATE12", Float64), + ("RATE13", Float64), + ("RATE14", Float64), + ("RATE15", Float64), + ("RATE16", Float64), + ("RATE17", Float64), + ("RATE18", Float64), + ("RATE19", Float64), + ("RATE110", Float64), + ("RATE111", Float64), + ("RATE112", Float64), + ], + _transformer_3_2_dtypes[7:8], + [("NOD1", Int64)], + _transformer_3_2_dtypes[9:end], +) + +const _transformer_3_3_dtypes = [ + ("WINDV2", Float64), + ("NOMV2", Float64), + ("ANG2", Float64), + ("RATA2", Float64), + ("RATB2", Float64), + ("RATC2", Float64), + ("COD2", Int64), + ("CONT2", Int64), + ("RMA2", Float64), + ("RMI2", Float64), + ("VMA2", Float64), + ("VMI2", Float64), + ("NTP2", Float64), + ("TAB2", Int64), + ("CR2", Float64), + ("CX2", Float64), + ("CNXA2", Float64), +] + +const _transformer_3_3_dtypes_v35 = vcat( + _transformer_3_3_dtypes[1:3], + [ + ("RATE21", Float64), + ("RATE22", Float64), + ("RATE23", Float64), + ("RATE24", Float64), + ("RATE25", Float64), + ("RATE26", Float64), + ("RATE27", Float64), + ("RATE28", Float64), + ("RATE29", Float64), + ("RATE210", Float64), + ("RATE211", Float64), + ("RATE212", Float64), + ], + _transformer_3_3_dtypes[7:8], + [("NOD2", Int64)], + _transformer_3_3_dtypes[9:end], +) + +const _transformer_3_4_dtypes = [ + ("WINDV3", Float64), + ("NOMV3", Float64), + ("ANG3", Float64), + ("RATA3", Float64), + ("RATB3", Float64), + ("RATC3", Float64), + ("COD3", Int64), + ("CONT3", Int64), + ("RMA3", Float64), + ("RMI3", Float64), + ("VMA3", Float64), + ("VMI3", Float64), + ("NTP3", Float64), + ("TAB3", Int64), + ("CR3", Float64), + ("CX3", Float64), + ("CNXA3", Float64), +] + +const _transformer_3_4_dtypes_v35 = vcat( + _transformer_3_4_dtypes[1:3], + [ + ("RATE31", Float64), + ("RATE32", Float64), + ("RATE33", Float64), + ("RATE34", Float64), + ("RATE35", Float64), + ("RATE36", Float64), + ("RATE37", Float64), + ("RATE38", Float64), + ("RATE39", Float64), + ("RATE310", Float64), + ("RATE311", Float64), + ("RATE312", Float64), + ], + _transformer_3_4_dtypes[7:8], + [("NOD3", Int64)], + _transformer_3_4_dtypes[9:end], +) + +const _transformer_2_1_dtypes = + [("R1-2", Float64), ("X1-2", Float64), ("SBASE1-2", Float64)] + +const _transformer_2_1_dtypes_v35 = _transformer_2_1_dtypes + +const _transformer_2_2_dtypes = [ + ("WINDV1", Float64), + ("NOMV1", Float64), + ("ANG1", Float64), + ("RATA1", Float64), + ("RATB1", Float64), + ("RATC1", Float64), + ("COD1", Int64), + ("CONT1", Int64), + ("RMA1", Float64), + ("RMI1", Float64), + ("VMA1", Float64), + ("VMI1", Float64), + ("NTP1", Float64), + ("TAB1", Int64), + ("CR1", Float64), + ("CX1", Float64), + ("CNXA1", Float64), +] + +const _transformer_2_2_dtypes_v35 = vcat( + _transformer_2_2_dtypes[1:3], + [ + ("RATE11", Float64), + ("RATE12", Float64), + ("RATE13", Float64), + ("RATE14", Float64), + ("RATE15", Float64), + ("RATE16", Float64), + ("RATE17", Float64), + ("RATE18", Float64), + ("RATE19", Float64), + ("RATE110", Float64), + ("RATE111", Float64), + ("RATE112", Float64), + ], + _transformer_2_2_dtypes[7:8], + [("NOD1", Int64)], + _transformer_2_2_dtypes[9:end], +) + +const _transformer_2_3_dtypes = [("WINDV2", Float64), ("NOMV2", Float64)] + +const _transformer_2_3_dtypes_v35 = _transformer_2_3_dtypes + +const _area_interchange_dtypes = + [("I", Int64), ("ISW", Int64), ("PDES", Float64), ("PTOL", Float64), ("ARNAME", String)] + +const _area_interchange_dtypes_v35 = _area_interchange_dtypes + +const _two_terminal_line_dtypes = [ + ("NAME", String), + ("MDC", Int64), + ("RDC", Float64), + ("SETVL", Float64), + ("VSCHD", Float64), + ("VCMOD", Float64), + ("RCOMP", Float64), + ("DELTI", Float64), + ("METER", String), + ("DCVMIN", Float64), + ("CCCITMX", Int64), + ("CCCACC", Float64), + ("IPR", Int64), + ("NBR", Int64), + ("ANMXR", Float64), + ("ANMNR", Float64), + ("RCR", Float64), + ("XCR", Float64), + ("EBASR", Float64), + ("TRR", Float64), + ("TAPR", Float64), + ("TMXR", Float64), + ("TMNR", Float64), + ("STPR", Float64), + ("ICR", Int64), + ("IFR", Int64), + ("ITR", Int64), + ("IDR", String), + ("XCAPR", Float64), + ("IPI", Int64), + ("NBI", Int64), + ("ANMXI", Float64), + ("ANMNI", Float64), + ("RCI", Float64), + ("XCI", Float64), + ("EBASI", Float64), + ("TRI", Float64), + ("TAPI", Float64), + ("TMXI", Float64), + ("TMNI", Float64), + ("STPI", Float64), + ("ICI", Int64), + ("IFI", Int64), + ("ITI", Int64), + ("IDI", String), + ("XCAPI", Float64), +] + +const _two_terminal_line_dtypes_v35 = vcat( + _two_terminal_line_dtypes[1:25], + [("NDR", Int64)], + _two_terminal_line_dtypes[26:42], + [("NDI", Int64)], + _two_terminal_line_dtypes[43:end], +) + +const _vsc_line_dtypes = [ + ("NAME", String), + ("MDC", Int64), + ("RDC", Float64), + ("O1", Int64), + ("F1", Float64), + ("O2", Int64), + ("F2", Float64), + ("O3", Int64), + ("F3", Float64), + ("O4", Int64), + ("F4", Float64), +] + +const _vsc_line_dtypes_35 = _vsc_line_dtypes + +const _vsc_subline_dtypes = [ + ("IBUS", Int64), + ("TYPE", Int64), + ("MODE", Int64), + ("DCSET", Float64), + ("ACSET", Float64), + ("ALOSS", Float64), + ("BLOSS", Float64), + ("MINLOSS", Float64), + ("SMAX", Float64), + ("IMAX", Float64), + ("PWF", Float64), + ("MAXQ", Float64), + ("MINQ", Float64), + ("REMOT", Int64), + ("RMPCT", Float64), +] + +const _vsc_subline_dtypes_v35 = _vsc_subline_dtypes + +const _impedance_correction_dtypes = [ + ("I", Int64), + ("T1", Float64), + ("F1", Float64), + ("T2", Float64), + ("F2", Float64), + ("T3", Float64), + ("F3", Float64), + ("T4", Float64), + ("F4", Float64), + ("T5", Float64), + ("F5", Float64), + ("T6", Float64), + ("F6", Float64), + ("T7", Float64), + ("F7", Float64), + ("T8", Float64), + ("F8", Float64), + ("T9", Float64), + ("F9", Float64), + ("T10", Float64), + ("F10", Float64), + ("T11", Float64), + ("F11", Float64), +] + +const _impedance_correction_dtypes_v35 = [ + ("I", Int64), + ("T1", Float64), ("Re(F1)", Float64), ("Im(F1)", Float64), + ("T2", Float64), ("Re(F2)", Float64), ("Im(F2)", Float64), + ("T3", Float64), ("Re(F3)", Float64), ("Im(F3)", Float64), + ("T4", Float64), ("Re(F4)", Float64), ("Im(F4)", Float64), + ("T5", Float64), ("Re(F5)", Float64), ("Im(F5)", Float64), + ("T6", Float64), ("Re(F6)", Float64), ("Im(F6)", Float64), + ("T7", Float64), ("Re(F7)", Float64), ("Im(F7)", Float64), + ("T8", Float64), ("Re(F8)", Float64), ("Im(F8)", Float64), + ("T9", Float64), ("Re(F9)", Float64), ("Im(F9)", Float64), + ("T10", Float64), ("Re(F10)", Float64), ("Im(F10)", Float64), + ("T11", Float64), ("Re(F11)", Float64), ("Im(F11)", Float64), + ("T12", Float64), ("Re(F12)", Float64), ("Im(F12)", Float64), +] + +const _multi_term_main_dtypes = [ + ("NAME", String), + ("NCONV", Int64), + ("NDCBS", Int64), + ("NDCLN", Int64), + ("MDC", Int64), + ("VCONV", Int64), + ("VCMOD", Float64), + ("VCONVN", Float64), +] + +const _multi_term_main_dtypes_v35 = _multi_term_main_dtypes + +const _multi_term_nconv_dtypes = [ + ("IB", Int64), + ("N", Int64), + ("ANGMX", Float64), + ("ANGMN", Float64), + ("RC", Float64), + ("XC", Float64), + ("EBAS", Float64), + ("TR", Float64), + ("TAP", Float64), + ("TPMX", Float64), + ("TPMN", Float64), + ("TSTP", Float64), + ("SETVL", Float64), + ("DCPF", Float64), + ("MARG", Float64), + ("CNVCOD", Int64), +] + +const _multi_term_nconv_dtypes_v35 = _multi_term_nconv_dtypes + +const _multi_term_ndcbs_dtypes = [ + ("IDC", Int64), + ("IB", Int64), + ("AREA", Int64), + ("ZONE", Int64), + ("DCNAME", String), + ("IDC2", Int64), + ("RGRND", Float64), + ("OWNER", Int64), +] + +const _multi_term_ndcbs_dtypes_v35 = _multi_term_ndcbs_dtypes + +const _multi_term_ndcln_dtypes = [ + ("IDC", Int64), + ("JDC", Int64), + ("DCCKT", String), + ("MET", Int64), + ("RDC", Float64), + ("LDC", Float64), +] + +const _multi_term_ndcln_dtypes_v35 = _multi_term_ndcln_dtypes + +const _multi_section_dtypes = [ + ("I", Int64), + ("J", Int64), + ("ID", String), + ("MET", Int64), + ("DUM1", Int64), + ("DUM2", Int64), + ("DUM3", Int64), + ("DUM4", Int64), + ("DUM5", Int64), + ("DUM6", Int64), + ("DUM7", Int64), + ("DUM8", Int64), + ("DUM9", Int64), +] + +const _multi_section_dtypes_v35 = _multi_section_dtypes + +const _zone_dtypes = [("I", Int64), ("ZONAME", String)] + +const _zone_dtypes_v35 = _zone_dtypes + +const _interarea_dtypes = + [("ARFROM", Int64), ("ARTO", Int64), ("TRID", String), ("PTRAN", Float64)] + +const _interarea_dtypes_v35 = _interarea_dtypes + +const _owner_dtypes = [("I", Int64), ("OWNAME", String)] + +const _owner_dtypes_v35 = _owner_dtypes + +const _FACTS_dtypes = [ + ("NAME", String), + ("I", Int64), + ("J", Int64), + ("MODE", Int64), + ("PDES", Float64), + ("QDES", Float64), + ("VSET", Float64), + ("SHMX", Float64), + ("TRMX", Float64), + ("VTMN", Float64), + ("VTMX", Float64), + ("VSMX", Float64), + ("IMX", Float64), + ("LINX", Float64), + ("RMPCT", Float64), + ("OWNER", Int64), + ("SET1", Float64), + ("SET2", Float64), + ("VSREF", Int64), + ("REMOT", Int64), + ("MNAME", String), +] + +const _FACTS_dtypes_v35 = vcat( + _FACTS_dtypes[1:19], + [ + ("FCREG", Int64), + ("NREG", Int64), + ("MNAME", String), + ], +) + +const _switched_shunt_dtypes = [ + ("I", Int64), + ("MODSW", Int64), + ("ADJM", Int64), + ("STAT", Int64), + ("VSWHI", Float64), + ("VSWLO", Float64), + ("SWREM", Int64), + ("RMPCT", Float64), + ("RMIDNT", String), + ("BINIT", Float64), + ("N1", Int64), + ("B1", Float64), + ("N2", Int64), + ("B2", Float64), + ("N3", Int64), + ("B3", Float64), + ("N4", Int64), + ("B4", Float64), + ("N5", Int64), + ("B5", Float64), + ("N6", Int64), + ("B6", Float64), + ("N7", Int64), + ("B7", Float64), + ("N8", Int64), + ("B8", Float64), +] + +const _switched_shunt_dtypes_v35 = vcat( + _switched_shunt_dtypes[1], + [("ID", String)], + _switched_shunt_dtypes[2:7], + [ + ("NREG", Int64), + ("RMPCT", Float64), + ("RMIDNT", String), + ("BINIT", Float64), + ("S1", Int64), ("N1", Int64), ("B1", Float64), + ("S2", Int64), ("N2", Int64), ("B2", Float64), + ("S3", Int64), ("N3", Int64), ("B3", Float64), + ("S4", Int64), ("N4", Int64), ("B4", Float64), + ("S5", Int64), ("N5", Int64), ("B5", Float64), + ("S6", Int64), ("N6", Int64), ("B6", Float64), + ("S7", Int64), ("N7", Int64), ("B7", Float64), + ("S8", Int64), ("N8", Int64), ("B8", Float64), + ], +) + +# TODO: Account for multiple lines in GNE Device entries +const _gne_device_dtypes = [ + ("NAME", String), + ("MODEL", String), + ("NTERM", Int64), + ("BUSi", Int64), + ("NREAL", Int64), + ("NINTG", Int64), + ("NCHAR", Int64), + ("STATUS", Int64), + ("OWNER", Int64), + ("NMETR", Int64), + ("REALi", Float64), + ("INTGi", Int64), + ("CHARi", String), +] + +const _gne_device_dtypes_v35 = _gne_device_dtypes + +const _induction_machine_dtypes = [ + ("I", Int64), + ("ID", String), + ("STAT", Int64), + ("SCODE", Int64), + ("DCODE", Int64), + ("AREA", Int64), + ("ZONE", Int64), + ("OWNER", Int64), + ("TCODE", Int64), + ("BCODE", Int64), + ("MBASE", Float64), + ("RATEKV", Float64), + ("PCODE", Int64), + ("PSET", Float64), + ("H", Float64), + ("A", Float64), + ("B", Float64), + ("D", Float64), + ("E", Float64), + ("RA", Float64), + ("XA", Float64), + ("XM", Float64), + ("R1", Float64), + ("X1", Float64), + ("R2", Float64), + ("X2", Float64), + ("X3", Float64), + ("E1", Float64), + ("SE1", Float64), + ("E2", Float64), + ("SE2", Float64), + ("IA1", Float64), + ("IA2", Float64), + ("XAMULT", Float64), +] + +const _induction_machine_dtypes_v35 = _induction_machine_dtypes + +const _substation_dtypes_v35 = [ + ("IS", Int64), + ("NAME", String), + ("LATITUDE", Float64), + ("LONGITUDE", Float64), + ("SGR", Float64), +] + +""" +lookup array of data types for PTI file sections given by +`field_name`, as enumerated by PSS/E Program Operation Manual. +""" +const _pti_dtypes = Dict{String, Array}( + "BUS" => _bus_dtypes, + "LOAD" => _load_dtypes, + "FIXED SHUNT" => _fixed_shunt_dtypes, + "GENERATOR" => _generator_dtypes, + "BRANCH" => _branch_dtypes, + "TRANSFORMER" => _transformer_dtypes, + "TRANSFORMER TWO-WINDING LINE 1" => _transformer_2_1_dtypes, + "TRANSFORMER TWO-WINDING LINE 2" => _transformer_2_2_dtypes, + "TRANSFORMER TWO-WINDING LINE 3" => _transformer_2_3_dtypes, + "TRANSFORMER THREE-WINDING LINE 1" => _transformer_3_1_dtypes, + "TRANSFORMER THREE-WINDING LINE 2" => _transformer_3_2_dtypes, + "TRANSFORMER THREE-WINDING LINE 3" => _transformer_3_3_dtypes, + "TRANSFORMER THREE-WINDING LINE 4" => _transformer_3_4_dtypes, + "AREA INTERCHANGE" => _area_interchange_dtypes, + "TWO-TERMINAL DC" => _two_terminal_line_dtypes, + "VOLTAGE SOURCE CONVERTER" => _vsc_line_dtypes, + "VOLTAGE SOURCE CONVERTER SUBLINES" => _vsc_subline_dtypes, + "IMPEDANCE CORRECTION" => _impedance_correction_dtypes, + "MULTI-TERMINAL DC" => _multi_term_main_dtypes, + "MULTI-TERMINAL DC NCONV" => _multi_term_nconv_dtypes, + "MULTI-TERMINAL DC NDCBS" => _multi_term_ndcbs_dtypes, + "MULTI-TERMINAL DC NDCLN" => _multi_term_ndcln_dtypes, + "MULTI-SECTION LINE" => _multi_section_dtypes, + "ZONE" => _zone_dtypes, + "INTER-AREA TRANSFER" => _interarea_dtypes, + "OWNER" => _owner_dtypes, + "FACTS CONTROL DEVICE" => _FACTS_dtypes, + "SWITCHED SHUNT" => _switched_shunt_dtypes, + "CASE IDENTIFICATION" => _transaction_dtypes, + "GNE DEVICE" => _gne_device_dtypes, + "INDUCTION MACHINE" => _induction_machine_dtypes, +) + +const _pti_dtypes_v35 = Dict{String, Array}( + "BUS" => _bus_dtypes_v35, + "LOAD" => _load_dtypes_v35, + "FIXED SHUNT" => _fixed_shunt_dtypes_v35, + "GENERATOR" => _generator_dtypes_v35, + "BRANCH" => _branch_dtypes_v35, + "SWITCHING DEVICE" => _switching_dtypes_v35, + "TRANSFORMER" => _transformer_dtypes_v35, + "TRANSFORMER TWO-WINDING LINE 1" => _transformer_2_1_dtypes_v35, + "TRANSFORMER TWO-WINDING LINE 2" => _transformer_2_2_dtypes_v35, + "TRANSFORMER TWO-WINDING LINE 3" => _transformer_2_3_dtypes_v35, + "TRANSFORMER THREE-WINDING LINE 1" => _transformer_3_1_dtypes_v35, + "TRANSFORMER THREE-WINDING LINE 2" => _transformer_3_2_dtypes_v35, + "TRANSFORMER THREE-WINDING LINE 3" => _transformer_3_3_dtypes_v35, + "TRANSFORMER THREE-WINDING LINE 4" => _transformer_3_4_dtypes_v35, + "AREA INTERCHANGE" => _area_interchange_dtypes_v35, + "TWO-TERMINAL DC" => _two_terminal_line_dtypes_v35, + "VOLTAGE SOURCE CONVERTER" => _vsc_line_dtypes_35, + "VOLTAGE SOURCE CONVERTER SUBLINES" => _vsc_subline_dtypes_v35, + "IMPEDANCE CORRECTION" => _impedance_correction_dtypes_v35, + "MULTI-TERMINAL DC" => _multi_term_main_dtypes_v35, + "MULTI-TERMINAL DC NCONV" => _multi_term_nconv_dtypes_v35, + "MULTI-TERMINAL DC NDCBS" => _multi_term_ndcbs_dtypes_v35, + "MULTI-TERMINAL DC NDCLN" => _multi_term_ndcln_dtypes_v35, + "MULTI-SECTION LINE" => _multi_section_dtypes_v35, + "ZONE" => _zone_dtypes_v35, + "INTER-AREA TRANSFER" => _interarea_dtypes_v35, + "OWNER" => _owner_dtypes_v35, + "FACTS CONTROL DEVICE" => _FACTS_dtypes_v35, + "SWITCHED SHUNT" => _switched_shunt_dtypes_v35, + "CASE IDENTIFICATION" => _transaction_dtypes_v35, + "GNE DEVICE" => _gne_device_dtypes_v35, + "INDUCTION MACHINE" => _induction_machine_dtypes_v35, + "SUBSTATION DATA" => _substation_dtypes_v35, +) + +const _default_case_identification = Dict( + "IC" => 0, + "SBASE" => 100.0, + "REV" => 33, + "XFRRAT" => 0, + "NXFRAT" => 0, + "BASFRQ" => 60, +) + +const _default_case_identification_v35 = Dict( + "IC" => 0, + "SBASE" => 100.0, + "REV" => 35, + "XFRRAT" => 0, + "NXFRAT" => 0, + "BASFRQ" => 60, +) + +const _default_bus = Dict( + "BASKV" => 0.0, + "IDE" => 1, + "AREA" => 1, + "ZONE" => 1, + "OWNER" => 1, + "VM" => 1.0, + "VA" => 0.0, + "NVHI" => 1.1, + "NVLO" => 0.9, + "EVHI" => 1.1, + "EVLO" => 0.9, + "NAME" => " ", +) + +const _default_bus_v35 = _default_bus + +const _default_load = Dict( + "ID" => "1", + "STATUS" => 1, + "PL" => 0.0, + "QL" => 0.0, + "IP" => 0.0, + "IQ" => 0.0, + "YP" => 0.0, + "YQ" => 0.0, + "SCALE" => 1, + "INTRPT" => 0, + "AREA" => nothing, + "ZONE" => nothing, + "OWNER" => nothing, +) + +const _default_load_v35 = merge( + _default_load, + Dict( + "DGENP" => 0.0, + "DGENQ" => 0.0, + "DGENM" => 0.0, + "LOADTYPE" => nothing, + ), +) + +const _default_fixed_shunt = Dict("ID" => "1", "STATUS" => 1, "GL" => 0.0, "BL" => 0.0) + +const _default_fixed_shunt_v35 = _default_fixed_shunt + +const _default_generator = Dict( + "ID" => "1", + "PG" => 0.0, + "QG" => 0.0, + "QT" => 9999.0, + "QB" => -9999.0, + "VS" => 1.0, + "IREG" => 0, + "MBASE" => nothing, + "ZR" => 0.0, + "ZX" => 1.0, + "RT" => 0.0, + "XT" => 0.0, + "GTAP" => 1.0, + "STAT" => 1, + "RMPCT" => 100.0, + "PT" => 9999.0, + "PB" => -9999.0, + "O1" => nothing, + "O2" => 0, + "O3" => 0, + "O4" => 0, + "F1" => 1.0, + "F2" => 1.0, + "F3" => 1.0, + "F4" => 1.0, + "WMOD" => 0, + "WPF" => 1.0, +) + +const _default_generator_v35 = merge(_default_generator, Dict( + "NREG" => 0, + "BASLOD" => 0, +)) + +const _default_branch = Dict( + "CKT" => "1", + "B" => 0.0, + "RATEA" => 0.0, + "RATEB" => 0.0, + "RATEC" => 0.0, + "GI" => 0.0, + "BI" => 0.0, + "GJ" => 0.0, + "BJ" => 0.0, + "ST" => 1, + "MET" => 1, + "LEN" => 0.0, + "O1" => nothing, + "O2" => 0, + "O3" => 0, + "O4" => 0, + "F1" => 1.0, + "F2" => 1.0, + "F3" => 1.0, + "F4" => 1.0, +) + +const _default_branch_v35 = merge( + _default_branch, + Dict( + "RATE1" => 0.0, + "RATE2" => 0.0, + "RATE3" => 0.0, + "RATE4" => 0.0, + "RATE5" => 0.0, + "RATE6" => 0.0, + "RATE7" => 0.0, + "RATE8" => 0.0, + "RATE9" => 0.0, + "RATE10" => 0.0, + "RATE11" => 0.0, + "RATE12" => 0.0, + ), +) + +const _default_switching_device_v35 = Dict( + "CKT" => "1", + "RATE1" => 0.0, + "RATE2" => 0.0, + "RATE3" => 0.0, + "RATE4" => 0.0, + "RATE5" => 0.0, + "RATE6" => 0.0, + "RATE7" => 0.0, + "RATE8" => 0.0, + "RATE9" => 0.0, + "RATE10" => 0.0, + "RATE11" => 0.0, + "RATE12" => 0.0, + "STAT" => 1, + "NSTAT" => 1, + "MET" => 0.0, + "STYPE" => 0.0, + "NAME" => "", +) + +const _default_transformer = Dict( + "K" => 0, + "CKT" => "1", + "CW" => 1, + "CZ" => 1, + "CM" => 1, + "MAG1" => 0.0, + "MAG2" => 0.0, + "NMETR" => 2, + "NAME" => " ", + "STAT" => 1, + "O1" => nothing, + "O2" => 0, + "O3" => 0, + "O4" => 0, + "F1" => 1.0, + "F2" => 1.0, + "F3" => 1.0, + "F4" => 1.0, + "VECGRP" => " ", + "R1-2" => 0.0, + "SBASE1-2" => nothing, + "R2-3" => 0.0, + "SBASE2-3" => nothing, + "R3-1" => 0.0, + "SBASE3-1" => nothing, + "VMSTAR" => 1.0, + "ANSTAR" => 0.0, + "WINDV1" => nothing, + "NOMV1" => 0.0, + "ANG1" => 0.0, + "RATA1" => 0.0, + "RATB1" => 0.0, + "RATC1" => 0.0, + "COD1" => 0, + "CONT1" => 0, + "RMA1" => 1.1, + "RMI1" => 0.9, + "VMA1" => 1.1, + "VMI1" => 0.9, + "NTP1" => 33, + "TAB1" => 0, + "CR1" => 0.0, + "CX1" => 0.0, + "CNXA1" => 0.0, + "WINDV2" => nothing, + "NOMV2" => 0.0, + "ANG2" => 0.0, + "RATA2" => 0.0, + "RATB2" => 0.0, + "RATC2" => 0.0, + "COD2" => 0, + "CONT2" => 0, + "RMA2" => 1.1, + "RMI2" => 0.9, + "VMA2" => 1.1, + "VMI2" => 0.9, + "NTP2" => 33, + "TAB2" => 0, + "CR2" => 0.0, + "CX2" => 0.0, + "CNXA2" => 0.0, + "WINDV3" => nothing, + "NOMV3" => 0.0, + "ANG3" => 0.0, + "RATA3" => 0.0, + "RATB3" => 0.0, + "RATC3" => 0.0, + "COD3" => 0, + "CONT3" => 0, + "RMA3" => 1.1, + "RMI3" => 0.9, + "VMA3" => 1.1, + "VMI3" => 0.9, + "NTP3" => 33, + "TAB3" => 0, + "CR3" => 0.0, + "CX3" => 0.0, + "CNXA3" => 0.0, +) + +const _default_transformer_v35 = merge( + _default_transformer, + Dict( + "NOD1" => 0, + "NOD2" => 0, + "NOD3" => 0, + ), +) + +const _default_area_interchange = + Dict("ISW" => 0, "PDES" => 0.0, "PTOL" => 10.0, "ARNAME" => " ") + +const _default_area_interchange_v35 = _default_area_interchange + +const _default_two_terminal_dc = Dict( + "MDC" => 0, + "VCMOD" => 0.0, + "RCOMP" => 0.0, + "DELTI" => 0.0, + "METER" => "I", + "DCVMIN" => 0.0, + "CCCITMX" => 20, + "CCCACC" => 1.0, + "TRR" => 1.0, + "TAPR" => 1.0, + "TMXR" => 1.5, + "TMNR" => 0.51, + "STPR" => 0.00625, + "ICR" => 0, + "IFR" => 0, + "ITR" => 0, + "IDR" => "1", + "XCAPR" => 0.0, + "TRI" => 1.0, + "TAPI" => 1.0, + "TMXI" => 1.5, + "TMNI" => 0.51, + "STPI" => 0.00625, + "ICI" => 0, + "IFI" => 0, + "ITI" => 0, + "IDI" => "1", + "XCAPI" => 0.0, +) + +const _default_two_terminal_dc_v35 = merge(_default_two_terminal_dc, Dict( + "NDR" => 0, + "NDI" => 0, +)) + +const _default_vsc_dc = Dict( + "MDC" => 1, + "O1" => nothing, + "O2" => 0, + "O3" => 0, + "O4" => 0, + "F1" => 1.0, + "F2" => 1.0, + "F3" => 1.0, + "F4" => 1.0, + "CONVERTER BUSES" => Dict( + "MODE" => 1, + "ACSET" => 1.0, + "ALOSS" => 1.0, + "BLOSS" => 0.0, + "MINLOSS" => 0.0, + "SMAX" => 0.0, + "IMAX" => 0.0, + "PWF" => 1.0, + "MAXQ" => 9999.0, + "MINQ" => -9999.0, + "REMOT" => 0, + "RMPCT" => 100.0, + ), +) + +const _default_vsc_dc_v35 = _default_vsc_dc + +const _default_impedance_correction = Dict( + "T1" => 0.0, + "T2" => 0.0, + "T3" => 0.0, + "T4" => 0.0, + "T5" => 0.0, + "T6" => 0.0, + "T7" => 0.0, + "T8" => 0.0, + "T9" => 0.0, + "T10" => 0.0, + "T11" => 0.0, + "F1" => 0.0, + "F2" => 0.0, + "F3" => 0.0, + "F4" => 0.0, + "F5" => 0.0, + "F6" => 0.0, + "F7" => 0.0, + "F8" => 0.0, + "F9" => 0.0, + "F10" => 0.0, + "F11" => 0.0, +) + +const _default_impedance_correction_v35 = Dict( + "T1" => 0.0, "Re(F1)" => 0.0, "Im(F1)" => 0.0, + "T2" => 0.0, "Re(F2)" => 0.0, "Im(F2)" => 0.0, + "T3" => 0.0, "Re(F3)" => 0.0, "Im(F3)" => 0.0, + "T4" => 0.0, "Re(F4)" => 0.0, "Im(F4)" => 0.0, + "T5" => 0.0, "Re(F5)" => 0.0, "Im(F5)" => 0.0, + "T6" => 0.0, "Re(F6)" => 0.0, "Im(F6)" => 0.0, + "T7" => 0.0, "Re(F7)" => 0.0, "Im(F7)" => 0.0, + "T8" => 0.0, "Re(F8)" => 0.0, "Im(F8)" => 0.0, + "T9" => 0.0, "Re(F9)" => 0.0, "Im(F9)" => 0.0, + "T10" => 0.0, "Re(F10)" => 0.0, "Im(F10)" => 0.0, + "T11" => 0.0, "Re(F11)" => 0.0, "Im(F11)" => 0.0, + "T12" => 0.0, "Re(F12)" => 0.0, "Im(F12)" => 0.0, +) + +const _default_multi_term_dc = Dict( + "MDC" => 0, + "VCMOD" => 0.0, + "VCONVN" => 0, + "CONV" => Dict( + "TR" => 1.0, + "TAP" => 1.0, + "TPMX" => 1.5, + "TPMN" => 0.51, + "TSTP" => 0.00625, + "DCPF" => 1, + "MARG" => 0.0, + "CNVCOD" => 1, + ), + "DCBS" => Dict( + "IB" => 0.0, + "AREA" => 1, + "ZONE" => 1, + "DCNAME" => " ", + "IDC2" => 0, + "RGRND" => 0.0, + "OWNER" => 1, + ), + "DCLN" => Dict("DCCKT" => 1, "MET" => 1, "LDC" => 0.0), +) + +const _default_multi_term_dc_v35 = _default_multi_term_dc + +const _default_multi_section = Dict("ID" => "&1", "MET" => 1) + +const _default_multi_section_v35 = _default_multi_section + +const _default_zone = Dict("ZONAME" => " ") + +const _default_zone_v35 = _default_zone + +const _default_interarea = Dict("TRID" => 1, "PTRAN" => 0.0) + +const _default_interarea_v35 = _default_interarea + +const _default_owner = Dict("OWNAME" => " ") + +const _default_owner_v35 = _default_owner + +const _default_facts = Dict( + "J" => 0, + "MODE" => 1, + "PDES" => 0.0, + "QDES" => 0.0, + "VSET" => 1.0, + "SHMX" => 9999.0, + "TRMX" => 9999.0, + "VTMN" => 0.9, + "VTMX" => 1.1, + "VSMX" => 1.0, + "IMX" => 0.0, + "LINX" => 0.05, + "RMPCT" => 100.0, + "OWNER" => 1, + "SET1" => 0.0, + "SET2" => 0.0, + "VSREF" => 0, + "REMOT" => 0, + "MNAME" => "", +) + +const _default_facts_v35 = merge( + Dict(k => v for (k, v) in pairs(_default_facts) if k != "REMOT"), + Dict( + "FCREG" => 0, # Replaces REMOT in v35 + "NREG" => 0, + ), +) + +const _default_switched_shunt = Dict( + "MODSW" => 1, + "ADJM" => 0, + "STAT" => 1, + "VSWHI" => 1.0, + "VSWLO" => 1.0, + "SWREM" => 0, + "RMPCT" => 100.0, + "RMIDNT" => "", + "BINIT" => 0.0, + "S1" => 1, "N1" => 0, "B1" => 0.0, + "S2" => 1, "N2" => 0, "B2" => 0.0, + "S3" => 1, "N3" => 0, "B3" => 0.0, + "S4" => 1, "N4" => 0, "B4" => 0.0, + "S5" => 1, "N5" => 0, "B5" => 0.0, + "S6" => 1, "N6" => 0, "B6" => 0.0, + "S7" => 1, "N7" => 0, "B7" => 0.0, + "S8" => 1, "N8" => 0, "B8" => 0.0, +) + +const _default_switched_shunt_v35 = merge( + Dict(k => v for (k, v) in pairs(_default_switched_shunt) if k != "SWREM"), + Dict( + "SWREG" => 0, + "ID" => "1", + "NAME" => "", + ), +) + +const _default_gne_device = Dict( + "NTERM" => 1, + "NREAL" => 0, + "NINTG" => 0, + "NCHAR" => 0, + "STATUS" => 1, + "OWNER" => nothing, + "NMETR" => nothing, + "REAL" => 0, + "INTG" => nothing, + "CHAR" => "1", +) + +const _default_gne_device_v35 = _default_gne_device + +const _default_induction_machine = Dict( + "ID" => 1, + "STAT" => 1, + "SCODE" => 1, + "DCODE" => 2, + "AREA" => nothing, + "ZONE" => nothing, + "OWNER" => nothing, + "TCODE" => 1, + "BCODE" => 1, + "MBASE" => nothing, + "RATEKV" => 0.0, + "PCODE" => 1, + "H" => 1.0, + "A" => 1.0, + "B" => 1.0, + "D" => 1.0, + "E" => 1.0, + "RA" => 0.0, + "XA" => 0.0, + "XM" => 2.5, + "R1" => 999.0, + "X1" => 999.0, + "R2" => 999.0, + "X2" => 999.0, + "X3" => 0.0, + "E1" => 1.0, + "SE1" => 0.0, + "E2" => 1.2, + "SE2" => 0.0, + "IA1" => 0.0, + "IA2" => 0.0, + "XAMULT" => 1, +) + +const _default_induction_machine_v35 = _default_induction_machine + +const _default_substation_data_v35 = Dict( + "NAME" => "", + "LATI" => 0.0, + "LONG" => 0.0, + "SGR" => 0.1, +) + +const _pti_defaults = Dict( + "BUS" => _default_bus, + "LOAD" => _default_load, + "FIXED SHUNT" => _default_fixed_shunt, + "GENERATOR" => _default_generator, + "BRANCH" => _default_branch, + "TRANSFORMER" => _default_transformer, + "AREA INTERCHANGE" => _default_area_interchange, + "TWO-TERMINAL DC" => _default_two_terminal_dc, + "VOLTAGE SOURCE CONVERTER" => _default_vsc_dc, + "IMPEDANCE CORRECTION" => _default_impedance_correction, + "MULTI-TERMINAL DC" => _default_multi_term_dc, + "MULTI-SECTION LINE" => _default_multi_section, + "ZONE" => _default_zone, + "INTER-AREA TRANSFER" => _default_interarea, + "OWNER" => _default_owner, + "FACTS CONTROL DEVICE" => _default_facts, + "SWITCHED SHUNT" => _default_switched_shunt, + "CASE IDENTIFICATION" => _default_case_identification, + "GNE DEVICE" => _default_gne_device, + "INDUCTION MACHINE" => _default_induction_machine, +) + +const _pti_defaults_v35 = Dict( + "BUS" => _default_bus, + "LOAD" => _default_load, + "FIXED SHUNT" => _default_fixed_shunt, + "GENERATOR" => _default_generator, + "BRANCH" => _default_branch, + "SWITCHING DEVICE" => _default_switching_device_v35, + "TRANSFORMER" => _default_transformer, + "AREA INTERCHANGE" => _default_area_interchange, + "TWO-TERMINAL DC" => _default_two_terminal_dc, + "VOLTAGE SOURCE CONVERTER" => _default_vsc_dc, + "IMPEDANCE CORRECTION" => _default_impedance_correction, + "MULTI-TERMINAL DC" => _default_multi_term_dc, + "MULTI-SECTION LINE" => _default_multi_section, + "ZONE" => _default_zone, + "INTER-AREA TRANSFER" => _default_interarea, + "OWNER" => _default_owner, + "FACTS CONTROL DEVICE" => _default_facts, + "SWITCHED SHUNT" => _default_switched_shunt, + "CASE IDENTIFICATION" => _default_case_identification, + "GNE DEVICE" => _default_gne_device, + "INDUCTION MACHINE" => _default_induction_machine, + "SUBSTATION DATA" => _default_substation_data_v35, +) + +function _correct_nothing_values!(data::Dict) + if !haskey(data, "BUS") + return + end + + sbase = data["CASE IDENTIFICATION"][1]["SBASE"] + bus_lookup = Dict(bus["I"] => bus for bus in data["BUS"]) + + if haskey(data, "LOAD") + for load in data["LOAD"] + load_bus = bus_lookup[load["I"]] + if load["AREA"] === nothing + load["AREA"] = load_bus["AREA"] + end + if load["ZONE"] === nothing + load["ZONE"] = load_bus["ZONE"] + end + if load["OWNER"] === nothing + load["OWNER"] = load_bus["OWNER"] + end + end + end + + if haskey(data, "GENERATOR") + for gen in data["GENERATOR"] + gen_bus = bus_lookup[gen["I"]] + if haskey(gen, "OWNER") && gen["OWNER"] === nothing + gen["OWNER"] = gen_bus["OWNER"] + end + if gen["MBASE"] === nothing + gen["MBASE"] = sbase + end + end + end + + if haskey(data, "BRANCH") + for branch in data["BRANCH"] + branch_bus = bus_lookup[branch["I"]] + if haskey(branch, "OWNER") && branch["OWNER"] === nothing + branch["OWNER"] = branch_bus["OWNER"] + end + end + end + + if haskey(data, "TRANSFORMER") + for transformer in data["TRANSFORMER"] + transformer_bus = bus_lookup[transformer["I"]] + for base_id in ["SBASE1-2", "SBASE2-3", "SBASE3-1"] + if haskey(transformer, base_id) && transformer[base_id] === nothing + transformer[base_id] = sbase + end + end + for winding_id in ["WINDV1", "WINDV2", "WINDV3"] + if haskey(transformer, winding_id) && transformer[winding_id] === nothing + if transformer["CW"] == 2 + transformer[winding_id] = transformer_bus["BASKV"] + else + transformer[winding_id] = 1.0 + end + end + end + end + end + + #= + # TODO update this default value + if haskey(data, "VOLTAGE SOURCE CONVERTER") + for mdc in data["VOLTAGE SOURCE CONVERTER"] + mdc["O1"] = Expr(:call, :_get_component_property, data["BUS"], "OWNER", "I", get(get(component, "CONVERTER BUSES", [Dict()])[1], "IBUS", 0)) + end + end + =# + + if haskey(data, "GNE DEVICE") + for gne in data["GNE DEVICE"] + gne_bus = bus_lookup[gne["I"]] + if haskey(gne, "OWNER") && gne["OWNER"] === nothing + gne["OWNER"] = gne_bus["OWNER"] + end + if haskey(gne, "NMETR") && gne["NMETR"] === nothing + gne["NMETR"] = gne_bus["NTERM"] + end + end + end + + if haskey(data, "INDUCTION MACHINE") + for indm in data["INDUCTION MACHINE"] + indm_bus = bus_lookup[indm["I"]] + if indm["AREA"] === nothing + indm["AREA"] = indm_bus["AREA"] + end + if indm["ZONE"] === nothing + indm["ZONE"] = indm_bus["ZONE"] + end + if indm["OWNER"] === nothing + indm["OWNER"] = indm_bus["OWNER"] + end + if indm["MBASE"] === nothing + indm["MBASE"] = sbase + end + end + end +end + +""" +This is an experimental method for parsing elements and setting defaults at the same time. +It is not currently working but would reduce memory allocations if implemented correctly. +""" +function _parse_elements( + elements::Array, + dtypes::Array, + defaults::Dict, + section::AbstractString, +) + data = Dict{String, Any}() + + if length(elements) > length(dtypes) + @warn( + "ignoring $(length(elements) - length(dtypes)) extra values in section $section, only $(length(dtypes)) items are defined" + ) + elements = elements[1:length(dtypes)] + end + + for (i, element) in enumerate(elements) + field, dtype = dtypes[i] + + element = strip(element) + + if dtype == String + if startswith(element, "'") && endswith(element, "'") + data[field] = element[2:(end - 1)] + else + data[field] = element + end + else + if length(element) <= 0 + # this will be set to a default in the cleanup phase + data[field] = nothing + else + try + data[field] = parse(dtype, element) + catch message + if isa(message, Meta.ParseError) + data[field] = element + else + @error( + "value '$element' for $field in section $section is not of type $dtype." + ) + end + end + end + end + end + + if length(elements) < length(dtypes) + for (field, dtype) in dtypes[length(elements):end] + data[field] = defaults[field] + #= + if length(missing_fields) > 0 + for field in missing_fields + data[field] = "" + end + missing_str = join(missing_fields, ", ") + if !(section == "SWITCHED SHUNT" && startswith(missing_str, "N")) && + !(section == "MULTI-SECTION LINE" && startswith(missing_str, "DUM")) && + !(section == "IMPEDANCE CORRECTION" && startswith(missing_str, "T")) + @warn("The following fields in $section are missing: $missing_str") + end + end + =# + end + end + + return data +end + +""" + _parse_line_element!(data, elements, section) + +Internal function. Parses a single "line" of data elements from a PTI file, as +given by `elements` which is an array of the line, typically split at `,`. +Elements are parsed into data types given by `section` and saved into `data::Dict`. +""" +function _parse_line_element!( + data::Dict, + elements::Array, + section::AbstractString, + dtypes::Dict{String, Array}, +) + missing_fields = [] + for (i, (field, dtype)) in enumerate(dtypes[section]) + if i > length(elements) + @debug "Have run out of elements in $section at $field" _group = + IS.LOG_GROUP_PARSING + push!(missing_fields, field) + continue + else + element = strip(elements[i]) + end + + try + if dtype != String && element != "" + data[field] = parse(dtype, element) + else + if dtype == String && startswith(element, "'") && endswith(element, "'") + data[field] = chop(element[nextind(element, 1):end]) + else + data[field] = element + end + end + catch message + if isa(message, Meta.ParseError) + data[field] = element + else + error( + "value '$element' for $field in section $section is not of type $dtype.", + ) + end + end + end + + if length(missing_fields) > 0 + for field in missing_fields + data[field] = "" + end + missing_str = join(missing_fields, ", ") + if !(section == "SWITCHED SHUNT" && startswith(missing_str, "N")) && + !(section == "MULTI-SECTION LINE" && startswith(missing_str, "DUM")) && + !(section == "IMPEDANCE CORRECTION" && startswith(missing_str, "T")) + @debug "The following fields in $section are missing: $missing_str" + end + end +end + +const _comment_split = r"(?!\B[\'][^\']*)[\/](?![^\']*[\']\B)" +const _split_string = r",(?=(?:[^']*'[^']*')*[^']*$)" + +""" + _get_line_elements(line) + +Internal function. Uses regular expressions to extract all separate data +elements from a line of a PTI file and populate them into an `Array{String}`. +Comments, typically indicated at the end of a line with a `'/'` character, +are also extracted separately, and `Array{Array{String}, String}` is returned. +""" +function _get_line_elements(line::AbstractString) + if count(i -> (i == "'"), line) % 2 == 1 + throw( + DataFormatError( + "There are an uneven number of single-quotes in \"{line}\", the line cannot be parsed.", + ), + ) + end + + line_comment = split(line, _comment_split; limit = 2) + line = strip(line_comment[1]) + comment = length(line_comment) > 1 ? strip(line_comment[2]) : "" + + elements = split(line, _split_string) + + return (elements, comment) +end + +""" +Process substation data with elements and parse associated nodes +""" +function parse_substation_nodes!( + section_data::Dict{String, Any}, + data_lines::Vector{String}, + start_line_index::Int, +)::Int + """Parse nodes for a substation and return the updated line index""" + section_data["NODES"] = [] + temp_line_index = start_line_index + 1 + + # Look for "BEGIN SUBSTATION NODE DATA" comment + while temp_line_index <= length(data_lines) + temp_line = data_lines[temp_line_index] + if contains(temp_line, "BEGIN SUBSTATION NODE DATA") + temp_line_index += 1 + break + end + temp_line_index += 1 + end + + # Parse node data until we hit the end marker + while temp_line_index <= length(data_lines) + temp_line = data_lines[temp_line_index] + + if startswith(temp_line, "0 /") && ( + contains(temp_line, "END OF SUBSTATION NODE DATA") || + contains(temp_line, "SUBSTATION TERMINAL DATA") + ) + return temp_line_index + end + + if contains(temp_line, "BEGIN SUBSTATION DATA BLOCK") + return temp_line_index - 1 + end + + if !startswith(temp_line, "@!") && !isempty(strip(temp_line)) + (check_elements, check_comment) = _get_line_elements(temp_line) + if length(check_elements) == 5 && + tryparse(Int, strip(check_elements[1])) !== nothing && + occursin('\'', check_elements[2]) && + tryparse(Float64, strip(check_elements[3])) !== nothing && + tryparse(Float64, strip(check_elements[4])) !== nothing && + tryparse(Float64, strip(check_elements[5])) !== nothing + return temp_line_index - 1 + end + end + + if startswith(temp_line, "@!") + temp_line_index += 1 + continue + end + + if !isempty(strip(temp_line)) + (node_elements, node_comment) = _get_line_elements(temp_line) + + if length(node_elements) >= 4 + if length(node_elements) >= 3 && + length(strip(node_elements[3])) == 3 && + startswith(strip(node_elements[3]), "'") && + endswith(strip(node_elements[3]), "'") + return temp_line_index - 1 + end + end + + if length(node_elements) >= 4 && length(node_elements) <= 6 + node_data = Dict{String, Any}() + node_data["NI"] = parse(Int, strip(node_elements[1])) + + name_string = strip(node_elements[2]) + if startswith(name_string, "'") && endswith(name_string, "'") + name_string = name_string[2:(end - 1)] # Remove quotes + end + node_data["NAME"] = strip(name_string) + + i_value = strip(node_elements[3]) + if startswith(i_value, "'") && endswith(i_value, "'") + i_value = i_value[2:(end - 1)] # Remove quotes + end + node_data["I"] = parse(Int, strip(i_value)) + + node_data["STATUS"] = parse(Int, strip(node_elements[4])) + if length(node_elements) >= 5 && !isempty(strip(node_elements[5])) + node_data["VM"] = parse(Float64, strip(node_elements[5])) + end + if length(node_elements) >= 6 && !isempty(strip(node_elements[6])) + node_data["VA"] = parse(Float64, strip(node_elements[6])) + end + push!(section_data["NODES"], node_data) + elseif length(node_elements) > 6 + return temp_line_index - 1 + end + end + temp_line_index += 1 + end + + return temp_line_index +end + +""" +Process substation data with elements and parse associated nodes +""" +function process_substation_data!( + section_data, + elements, + section, + current_dtypes, + data_lines, + line_index, + pti_data, +) + try + _parse_line_element!(section_data, elements, section, current_dtypes) + + if haskey(section_data, "NAME") + section_data["NAME"] = strip(section_data["NAME"]) + end + + # Parse nodes for this substation + updated_line_index = parse_substation_nodes!(section_data, data_lines, line_index) + + if haskey(pti_data, section) + push!(pti_data[section], section_data) + else + pti_data[section] = [section_data] + end + + return updated_line_index + catch message + error("Parsing failed at line $line_index: $(sprint(showerror, message))") + end +end + +""" + _parse_pti_data(data_string, sections) + +Internal function. Parse a PTI raw file into a `Dict`, given the +`data_string` of the file and a list of the `sections` in the PTI +file (typically given by default by `get_pti_sections()`. +""" +function _parse_pti_data(data_io::IO) + sections = deepcopy(_pti_sections) + sections_v35 = deepcopy(_pti_sections_v35) + data_lines = readlines(data_io) + skip_lines = 0 + skip_sublines = 0 + subsection = "" + is_v35 = false + + pti_data = Dict{String, Array{Dict}}() + + section = popfirst!(sections) + section_v35 = popfirst!(sections_v35) + section_data = Dict{String, Any}() + + if any(startswith.(data_lines, "@!")) + is_v35 = true + end + + header_line_start = is_v35 ? 2 : 1 # Start in second line due to @! + # Dynamically handle the start of BUS DATA section + # In v35 files, BUS DATA starts in different lines due to the fields GENERAL,GAUSS,NEWTON,ADJUST,TYSL,SOLVER,RATING + # This fields are optional in the file and when not found, the start of the reading vary a lot + bus_data_start = if is_v35 + found_start = 25 # Default of most files + for i in 3:min(35, length(data_lines)) + line = strip(data_lines[i]) + + # Skip comments and system-wide data + if startswith( + line, + r"@!|GENERAL,|GAUSS,|NEWTON,|ADJUST,|TYSL,|SOLVER,|RATING,", + ) || isempty(line) + continue + end + + # Look for section marker of BUS DATA + if contains(line, "END OF SYSTEM-WIDE DATA") || + ( + tryparse(Int, split(line, ',')[1] |> strip) !== nothing && + contains(line, "'") + ) + found_start = if contains(line, "END OF SYSTEM-WIDE DATA") + (i + (startswith(strip(data_lines[i + 1]), "@!") ? 2 : 1)) + else + i + end + break + end + end + # New updated start section + found_start + else + 4 # Start for all v33 files + end + + current_dtypes = is_v35 ? _pti_dtypes_v35 : _pti_dtypes + + line_index = 1 + while line_index <= length(data_lines) + line = data_lines[line_index] + + if startswith(line, "@!") + line_index += 1 + continue + end + + (elements, comment) = _get_line_elements(line) + + first_element = strip(elements[1]) + + if is_v35 && (line_index == 3 || line_index == 4) && + section_v35 == "CASE IDENTIFICATION" + comment_line = strip(line) + comment_key = line_index == 3 ? "Comment_Line_1" : "Comment_Line_2" + + if haskey(pti_data, "CASE IDENTIFICATION") && + !isempty(pti_data["CASE IDENTIFICATION"]) + pti_data["CASE IDENTIFICATION"][1][comment_key] = comment_line + @debug "Added $comment_key: $comment_line" _group = IS.LOG_GROUP_PARSING + end + line_index += 1 + continue + end + + if is_v35 && line_index >= 3 && line_index < bus_data_start + line_index += 1 + continue + end + + if line_index > (is_v35 ? bus_data_start - 1 : 3) && length(elements) != 0 && + first_element == "Q" + break + elseif line_index > (is_v35 ? bus_data_start - 1 : 3) && length(elements) != 0 && + first_element == "0" + if line_index == bus_data_start + section = is_v35 ? popfirst!(sections_v35) : popfirst!(sections) + end + + if length(elements) > 1 + @info( + "At line $line_index, new section started with '0', but additional non-comment data is present. Pattern '^\\s*0\\s*[/]*.*' is reserved for section start/end.", + ) + elseif length(comment) > 0 + @debug "At line $line_index, switched to $section" _group = + IS.LOG_GROUP_PARSING + end + + current_sections = is_v35 ? sections_v35 : sections + if !isempty(current_sections) + section = popfirst!(current_sections) + end + + line_index += 1 + continue + else + if line_index == bus_data_start + section = is_v35 ? popfirst!(sections_v35) : popfirst!(sections) + section_data = Dict{String, Any}() + end + + if skip_lines > 0 + skip_lines -= 1 + line_index += 1 + continue + end + + if section == "IMPEDANCE CORRECTION" && is_v35 + temporal_ic_elements = Vector{Vector{String}}() + + while line_index <= length(data_lines) + line = data_lines[line_index] + + if startswith(line, "0 /") || startswith(line, "Q") + if !isempty(temporal_ic_elements) + last_entry_elements = temporal_ic_elements[end] + + section_data_final = Dict{String, Any}() + section_data_final["I"] = + parse(Int64, strip(last_entry_elements[1])) + + processing_elements = last_entry_elements[2:end] + + point_index = 1 + element_index = 1 + while element_index <= length(processing_elements) && + element_index + 2 <= length(processing_elements) + t_str = strip(processing_elements[element_index]) + re_str = strip(processing_elements[element_index + 1]) + im_str = strip(processing_elements[element_index + 2]) + + if !isempty(t_str) && !isempty(re_str) && !isempty(im_str) + section_data_final["T$point_index"] = + parse(Float64, t_str) + section_data_final["Re(F$point_index)"] = + parse(Float64, re_str) + section_data_final["Im(F$point_index)"] = + parse(Float64, im_str) + point_index += 1 + end + element_index += 3 + end + + if haskey(pti_data, section) + push!(pti_data[section], section_data_final) + else + pti_data[section] = [section_data_final] + end + end + break + end + + if startswith(line, "@!") + line_index += 1 + continue + end + + if isempty(strip(line)) + line_index += 1 + continue + end + + (elements, comment) = _get_line_elements(line) + first_element = strip(elements[1]) + + if tryparse(Int64, first_element) === nothing + line_index += 1 + if !isempty(temporal_ic_elements) + append!(temporal_ic_elements[end], elements) + end + continue + end + + if !isempty(temporal_ic_elements) + last_entry_elements = temporal_ic_elements[end] + + section_data_prev = Dict{String, Any}() + section_data_prev["I"] = parse(Int64, strip(last_entry_elements[1])) + + processing_elements = last_entry_elements[2:end] + + point_index = 1 + element_index = 1 + while element_index <= length(processing_elements) && + element_index + 2 <= length(processing_elements) + t_str = strip(processing_elements[element_index]) + re_str = strip(processing_elements[element_index + 1]) + im_str = strip(processing_elements[element_index + 2]) + + if !isempty(t_str) && !isempty(re_str) && !isempty(im_str) + section_data_prev["T$point_index"] = parse(Float64, t_str) + section_data_prev["Re(F$point_index)"] = + parse(Float64, re_str) + section_data_prev["Im(F$point_index)"] = + parse(Float64, im_str) + point_index += 1 + end + element_index += 3 + end + + if haskey(pti_data, section) + push!(pti_data[section], section_data_prev) + else + pti_data[section] = [section_data_prev] + end + end + + push!(temporal_ic_elements, elements) + line_index += 1 + end + + elseif !( + section in [ + "CASE IDENTIFICATION", + "SWITCHING DEVICE DATA", + "TRANSFORMER", + "VOLTAGE SOURCE CONVERTER", + "IMPEDANCE CORRECTION", + "MULTI-TERMINAL DC", + "TWO-TERMINAL DC", + "GNE DEVICE", + "SUBSTATION DATA", + ] + ) + section_data = Dict{String, Any}() + + try + _parse_line_element!(section_data, elements, section, current_dtypes) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))" + ) + ) + end + line_index += 1 + + elseif section == "CASE IDENTIFICATION" + if line_index == header_line_start + try + _parse_line_element!( + section_data, + elements, + section, + current_dtypes, + ) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + + if section_data["REV"] != "" && section_data["REV"] < 33 + @info( + "Version $(section_data["REV"]) of PTI format is unsupported, parser may not function correctly.", + ) + end + + if is_v35 + if haskey(pti_data, section) + push!(pti_data[section], section_data) + else + pti_data[section] = [section_data] + end + end + else + if is_v35 + if line_index == 3 + comment_line = strip(line) + if haskey(pti_data, section) && !isempty(pti_data[section]) + pti_data[section][1]["Comment_Line_1"] = comment_line + end + elseif line_index == 4 + comment_line = strip(line) + if haskey(pti_data, section) && !isempty(pti_data[section]) + pti_data[section][1]["Comment_Line_2"] = comment_line + end + end + elseif !is_v35 && line_index > header_line_start + section_data["Comment_Line_$(line_index - 1)"] = strip(line) + end + end + + if line_index < (bus_data_start - 1) + line_index += 1 + continue + end + + line_index += 1 + + elseif section == "SWITCHING DEVICE" + if is_v35 + section_data = Dict{String, Any}() + try + _parse_line_element!( + section_data, + elements, + section, + current_dtypes, + ) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + else + @info("SWITCHING DEVICE DATA section found in non-v35 file, skipping.") + end + line_index += 1 + + elseif section == "TRANSFORMER" + section_data = Dict{String, Any}() + if parse(Int64, _get_line_elements(line)[1][3]) == 0 # two winding transformer + winding = "TWO-WINDING" + skip_lines = 3 + elseif parse(Int64, _get_line_elements(line)[1][3]) != 0 # three winding transformer + winding = "THREE-WINDING" + skip_lines = 4 + else + @error("Cannot detect type of Transformer") + end + + try + for transformer_line in 0:4 + if transformer_line == 0 + temp_section = section + else + temp_section = + join([section, winding, "LINE", transformer_line], " ") + end + + if winding == "TWO-WINDING" && transformer_line == 4 + break + else + elements = _get_line_elements( + data_lines[line_index + transformer_line], + )[1] + _parse_line_element!( + section_data, + elements, + temp_section, + current_dtypes, + ) + end + end + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + line_index += 1 + + elseif section == "VOLTAGE SOURCE CONVERTER" + vsc_line_length = length(_get_line_elements(line)[1]) + # VSC DC LINE DATA can have 5 or 11 elements in all cases possible + # "CSC-VSC ",1, 1.5800, 28,1.0000 + # "CSC-VSC ",1, 1.5800, 28,1.0000,,,,,, + # "CSC-VSC ",1, 1.5800, 28,1.0000,1.0,0.0,1.0,0.0,1.0,0.0 + # This is how originally the parser was written + if vsc_line_length == 5 || vsc_line_length == 11 + section_data = Dict{String, Any}() + try + _parse_line_element!( + section_data, + elements, + section, + current_dtypes, + ) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + skip_sublines = 2 + line_index += 1 + continue + + elseif skip_sublines > 0 + skip_sublines -= 1 + subsection_data = Dict{String, Any}() + + for (field, dtype) in _pti_dtypes["$section SUBLINES"] + element = popfirst!(elements) + if element != "" + subsection_data[field] = parse(dtype, element) + else + line_index += 1 + subsection_data[field] = "" + end + end + + if haskey(section_data, "CONVERTER BUSES") + push!(section_data["CONVERTER BUSES"], subsection_data) + else + section_data["CONVERTER BUSES"] = [subsection_data] + line_index += 1 + continue + end + end + line_index += 1 + + elseif section == "TWO-TERMINAL DC" + section_data = Dict{String, Any}() + if length(_get_line_elements(line)[1]) == 12 + (elements, comment) = _get_line_elements( + join(data_lines[line_index:(line_index + 2)], ','), + ) + skip_lines = 2 + end + + try + _parse_line_element!(section_data, elements, section, current_dtypes) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + line_index += 1 + + elseif section == "IMPEDANCE CORRECTION" && !is_v35 + section_data = Dict{String, Any}() + try + _parse_line_element!(section_data, elements, section, current_dtypes) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + line_index += 1 + + elseif section == "MULTI-TERMINAL DC" + if skip_sublines == 0 + section_data = Dict{String, Any}() + try + _parse_line_element!( + section_data, + elements, + section, + current_dtypes, + ) + catch message + throw( + @error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + + if section_data["NCONV"] > 0 + skip_sublines = section_data["NCONV"] + subsection = "NCONV" + line_index += 1 + continue + elseif section_data["NDCBS"] > 0 + skip_sublines = section_data["NDCBS"] + subsection = "NDCBS" + line_index += 1 + continue + elseif section_data["NDCLN"] > 0 + skip_sublines = section_data["NDCLN"] + subsection = "NDCLN" + line_index += 1 + continue + end + end + + if skip_sublines > 0 + skip_sublines -= 1 + + subsection_data = Dict{String, Any}() + try + _parse_line_element!( + subsection_data, + elements, + "$section $subsection", + current_dtypes, + ) + catch message + throw( + error( + "Parsing failed at line $line_index: $(sprint(showerror, message))", + ), + ) + end + + if haskey(section_data, "$(subsection[2:end])") + section_data["$(subsection[2:end])"] = + push!(section_data["$(subsection[2:end])"], subsection_data) + if skip_sublines > 0 && subsection != "NDCLN" + line_index += 1 + continue + end + else + section_data["$(subsection[2:end])"] = [subsection_data] + if skip_sublines > 0 && subsection != "NDCLN" + line_index += 1 + continue + end + end + + if skip_sublines == 0 && subsection != "NDCLN" + if subsection == "NDCBS" + skip_sublines = section_data["NDCLN"] + subsection = "NDCLN" + line_index += 1 + continue + elseif subsection == "NCONV" + skip_sublines = section_data["NDCBS"] + subsection = "NDCBS" + line_index += 1 + continue + end + elseif skip_sublines == 0 && subsection == "NDCLN" + subsection = "" + else + line_index += 1 + continue + end + end + line_index += 1 + + elseif section == "SUBSTATION DATA" && is_v35 + if startswith(line, "@!") + line_index += 1 + continue + else + if length(elements) == 4 && occursin('\'', elements[1]) + first_part = elements[1] + if occursin(",'", first_part) + comma_quote_pos = findfirst(",'", first_part) + if comma_quote_pos !== nothing + is_part = first_part[1:(comma_quote_pos[1] - 1)] + name_part = first_part[(comma_quote_pos[1] + 1):end] + + corrected_elements = [ + is_part, + name_part, + elements[2], + elements[3], + elements[4], + ] + + if length(corrected_elements) == 5 && + occursin('\'', corrected_elements[2]) && + tryparse(Float64, strip(corrected_elements[3])) !== + nothing && + tryparse(Float64, strip(corrected_elements[4])) !== + nothing && + tryparse(Float64, strip(corrected_elements[5])) !== + nothing + @debug "Parsing substation data line: $line" _group = + IS.LOG_GROUP_PARSING + section_data = Dict{String, Any}() + line_index = process_substation_data!( + section_data, + corrected_elements, + section, + current_dtypes, + data_lines, + line_index, + pti_data, + ) + end + end + end + + elseif length(elements) == 5 && + occursin('\'', elements[2]) && + tryparse(Float64, strip(elements[3])) !== nothing && + tryparse(Float64, strip(elements[4])) !== nothing && + tryparse(Float64, strip(elements[5])) !== nothing + section_data = Dict{String, Any}() + line_index = process_substation_data!( + section_data, + elements, + section, + current_dtypes, + data_lines, + line_index, + pti_data, + ) + end + + line_index += 1 + continue + end + line_index += 1 + + elseif section == "GNE DEVICE" + # TODO: handle multiple lines of GNE Device + @info("GNE DEVICE parsing is not supported.") + line_index += 1 + else + line_index += 1 + end + end + if subsection != "" + @debug "appending data" _group = IS.LOG_GROUP_PARSING + end + + if haskey(pti_data, section) + if section == "IMPEDANCE CORRECTION" && + pti_data["CASE IDENTIFICATION"][1]["REV"] == 35 + continue + else + push!(pti_data[section], section_data) + end + else + pti_data[section] = [section_data] + end + end + + _split_breakers_and_branches!(pti_data) + _populate_defaults!(pti_data) + _correct_nothing_values!(pti_data) + + return pti_data +end + +""" + parse_pti(filename::String) + +Open PTI raw file given by `filename`, returning a `Dict` of the data parsed +into the proper types. +""" +function parse_pti(filename::String)::Dict + pti_data = open(filename) do f + parse_pti(f) + end + + return pti_data +end + +""" + parse_pti(io::IO) + +Reads PTI data in `io::IO`, returning a `Dict` of the data parsed into the +proper types. +""" +function parse_pti(io::IO)::Dict + pti_data = _parse_pti_data(io) + try + pti_data["CASE IDENTIFICATION"][1]["NAME"] = match( + r"^\$", + lowercase(io.name), + ).captures[1] + catch + throw(error("This file is unrecognized and cannot be parsed")) + end + + return pti_data +end + +function _split_breakers_and_branches!(data::Dict) + if !haskey(data, "BRANCH") + @info "No BRANCH section found in the system." + return data + end + breakers = sizehint!(eltype(data["BRANCH"])[], length(data["BRANCH"])) + delete_ixs = Int[] + for (ix, item) in enumerate(data["BRANCH"]) + if first(item["CKT"]) == '@' || first(item["CKT"]) == '*' + push!(breakers, item) + push!(delete_ixs, ix) + end + end + if isempty(delete_ixs) + @info "No breakers modeled as branches using @ or * found in the system." + return data + else + @info "Found $(length(breakers)) breakers in the system modeled as branches." + end + deleteat!(data["BRANCH"], delete_ixs) + data["SWITCHES_AS_BRANCHES"] = breakers + return data +end + +""" + _populate_defaults!(pti_data) + +Internal function. Populates empty fields with PSS(R)E PTI v33 default values +""" +function _populate_defaults!(data::Dict) + for section in _pti_sections + if haskey(data, section) + component_defaults = _pti_defaults[section] + for component in data[section] + for (field, field_value) in component + if isa(field_value, Array) + sub_component_defaults = component_defaults[field] + for sub_component in field_value + for (sub_field, sub_field_value) in sub_component + if sub_field_value == "" + try + sub_component[sub_field] = + sub_component_defaults[sub_field] + catch msg + if isa(msg, KeyError) + @warn( + "'$sub_field' in '$field' in '$section' has no default value", + ) + else + rethrow(msg) + end + end + end + end + end + elseif field_value == "" && + !(field in ["Comment_Line_1", "Comment_Line_2"]) && + !startswith(field, "DUM") + try + component[field] = component_defaults[field] + catch msg + if isa(msg, KeyError) + @warn("'$field' in '$section' has no default value",) + else + rethrow(msg) + end + end + end + end + end + end + end +end diff --git a/src/power_models_data.jl b/src/power_models_data.jl new file mode 100644 index 0000000..2cdb13c --- /dev/null +++ b/src/power_models_data.jl @@ -0,0 +1,2143 @@ +"""Container for data parsed by PowerModels""" +struct PowerModelsData + data::Dict{String, Any} +end + +""" +Constructs PowerModelsData from a raw file. +Currently Supports MATPOWER and PSSE data files parsed by PowerModels. +""" +function PowerModelsData(file::Union{String, IO}; kwargs...) + validate = get(kwargs, :pm_data_corrections, true) + import_all = get(kwargs, :import_all, false) + correct_branch_rating = get(kwargs, :correct_branch_rating, true) + pm_dict = parse_file( + file; + import_all = import_all, + validate = validate, + correct_branch_rating = correct_branch_rating, + ) + pm_data = PowerModelsData(pm_dict) + correct_pm_transformer_status!(pm_data) + return pm_data +end + +""" +Constructs a System from PowerModelsData. + +# Arguments +- `pm_data::Union{PowerModelsData, Union{String, IO}}`: PowerModels data object or supported +load flow case (*.m, *.raw) + +# Keyword arguments +- `ext::Dict`: Contains user-defined parameters. Should only contain standard types. +- `runchecks::Bool`: Run available checks on input fields and when add_component! is called. + Throws InvalidValue if an error is found. +- `time_series_in_memory::Bool=false`: Store time series data in memory instead of HDF5. +- `config_path::String`: specify path to validation config file +- `pm_data_corrections::Bool=true` : Run the PowerModels data corrections (aka :validate in PowerModels) +- `import_all:Bool=false` : Import all fields from PTI files + +# Examples +```julia +sys = System( + pm_data, config_path = "ACTIVSg25k_validation.json", + bus_name_formatter = x->string(x["name"]*"-"*string(x["index"])), + load_name_formatter = x->strip(join(x["source_id"], "_")) +) +``` +""" +function System(pm_data::PowerModelsData; kwargs...) + runchecks = get(kwargs, :runchecks, true) + data = pm_data.data + if length(data["bus"]) < 1 + throw(DataFormatError("There are no buses in this file.")) + end + + @info "Constructing System from Power Models" data["name"] data["source_type"] + + sys = System(data["baseMVA"]; kwargs...) + source_type = data["source_type"] + + bus_number_to_bus = read_bus!(sys, data; kwargs...) + read_loads!(sys, data, bus_number_to_bus; kwargs...) + read_loadzones!(sys, data, bus_number_to_bus; kwargs...) + read_gen!(sys, data, bus_number_to_bus; kwargs...) + for component_type in ["switch", "breaker"] + read_switch_breaker!(sys, data, bus_number_to_bus, component_type; kwargs...) + end + read_branch!(sys, data, bus_number_to_bus; kwargs...) + read_switched_shunt!(sys, data, bus_number_to_bus; kwargs...) + read_shunt!(sys, data, bus_number_to_bus; kwargs...) + read_dcline!(sys, data, bus_number_to_bus, source_type; kwargs...) + read_vscline!(sys, data, bus_number_to_bus; kwargs...) + read_facts!(sys, data, bus_number_to_bus; kwargs...) + read_storage!(sys, data, bus_number_to_bus; kwargs...) + read_3w_transformer!(sys, data, bus_number_to_bus; kwargs...) + if runchecks + check(sys) + end + + substation_data = get(data, "substation_data", []) + add_geographic_info_to_buses!(sys, substation_data) + + return sys +end + +function correct_pm_transformer_status!(pm_data::PowerModelsData) + for (k, branch) in pm_data.data["branch"] + f_bus_bvolt = pm_data.data["bus"][branch["f_bus"]]["base_kv"] + t_bus_bvolt = pm_data.data["bus"][branch["t_bus"]]["base_kv"] + percent_difference = + abs(f_bus_bvolt - t_bus_bvolt) / ((f_bus_bvolt + t_bus_bvolt) / 2) + if !branch["transformer"] && + percent_difference > BRANCH_BUS_VOLTAGE_DIFFERENCE_TOL + branch["transformer"] = true + branch["base_power"] = pm_data.data["baseMVA"] + branch["ext"] = Dict{String, Any}() + @warn "Branch $(branch["f_bus"]) - $(branch["t_bus"]) has different voltage levels endpoints (from: $(f_bus_bvolt)kV, to: $(t_bus_bvolt)kV) which exceed the $(BRANCH_BUS_VOLTAGE_DIFFERENCE_TOL*100)% threshold; converting to transformer." + if !haskey(branch, "base_voltage_from") + branch["base_voltage_from"] = f_bus_bvolt + branch["base_voltage_to"] = t_bus_bvolt + end + end + end +end + +""" +Internal component name retrieval from pm2ps_dict +""" +function _get_pm_dict_name(device_dict::Dict)::String + if haskey(device_dict, "shunt_bus") + # With shunts, we have FixedAdmittance and SwitchedAdmittance types. + # To avoid potential name collision, we add the connected bus number to the name. + name = join(strip.(string.((device_dict["shunt_bus"], device_dict["name"]))), "-") + elseif haskey(device_dict, "name") + name = string(device_dict["name"]) + elseif haskey(device_dict, "source_id") + name = strip(join(string.(device_dict["source_id"]), "-")) + else + name = string(device_dict["index"]) + end + return name +end + +function _get_pm_bus_name(device_dict::Dict, unique_names::Bool) + if haskey(device_dict, "name") + if unique_names + name = strip(device_dict["name"]) + else + name = strip(device_dict["name"]) * "_" * string(device_dict["bus_i"]) + end + else + name = strip(join(string.(device_dict["source_id"]), "-")) + end + return name +end + +""" +Internal branch name retrieval from pm2ps_dict +""" +function _get_pm_branch_name(device_dict, bus_f::ACBus, bus_t::ACBus) + # Additional if-else are used to catch line id in PSSe parsing cases + if haskey(device_dict, "name") + index = device_dict["name"] + elseif device_dict["source_id"][1] == "branch" && length(device_dict["source_id"]) > 2 + index = strip(device_dict["source_id"][4]) + elseif ( + device_dict["source_id"][1] == "switch" || device_dict["source_id"][1] == "breaker" + ) && length(device_dict["source_id"]) > 2 + index = string(device_dict["source_id"][4][2]) + elseif device_dict["source_id"][1] == "transformer" && + length(device_dict["source_id"]) > 3 + index = strip(device_dict["source_id"][5]) + else + index = device_dict["index"] + end + return "$(get_name(bus_f))-$(get_name(bus_t))-i_$index" +end + +""" +Internal 3WT name retrieval from pm2ps_dict +""" +function _get_pm_3w_name( + device_dict, + bus_primary::ACBus, + bus_secondary::ACBus, + bus_tertiary::ACBus, +) + ckt = device_dict["circuit"] + return "$(get_name(bus_primary))-$(get_name(bus_secondary))-$(get_name(bus_tertiary))-i_$ckt" +end + +"""Add geographic coordinates to all buses using pre-built lookup""" +function add_geographic_info_to_buses!(sys, substation_data) + if isempty(substation_data) + @warn "No substation data found" + return + end + + bus_coords_lookup = Dict{Int, GeographicInfo}() + + for (_, substation) in substation_data + if haskey(substation, "nodes") && haskey(substation, "latitude") && + haskey(substation, "longitude") + lat, lon = substation["latitude"], substation["longitude"] + + geo_info = GeographicInfo(; + geo_json = Dict( + "type" => "Point", + "coordinates" => [lon, lat], + ), + ) + for node in substation["nodes"] + if haskey(node, "I") + bus_coords_lookup[node["I"]] = geo_info + end + end + end + end + + begin_supplemental_attributes_update(sys) do + buses_with_coords = 0 + buses_without_coords = 0 + + for bus in get_components(ACBus, sys) + bus_number = get_number(bus) + + if haskey(bus_coords_lookup, bus_number) + geo_info = bus_coords_lookup[bus_number] + add_supplemental_attribute!(sys, bus, geo_info) + buses_with_coords += 1 + else + buses_without_coords += 1 + end + end + + @info "Added coordinates to $(buses_with_coords) buses, $(buses_without_coords) buses without coordinates" + end +end + +""" +Parses ITC data from a dictionary and constructs a lookup table +of piecewise linear scaling functions. +""" +function _impedance_correction_table_lookup(data::Dict) + ict_instances = Dict{Tuple{Int64, WindingCategory}, ImpedanceCorrectionData}() + + @info "Reading Impedance Correction Table data" + if !haskey(data, "impedance_correction") + @info "There is no Impedance Correction Table data in this file" + return ict_instances + end + + for (_, table_data) in data["impedance_correction"] + table_number = table_data["table_number"] + x = table_data["tap_or_angle"] + y = table_data["scaling_factor"] + + if length(x) == length(y) + if length(x) < 2 + @warn "Skipping impedance correction entry due to insufficient data points ($(length(x)) < 2): $(x)" + continue + end + pwl_data = PiecewiseLinearData([(x[i], y[i]) for i in eachindex(x)]) + table_type = + if ( + x[1] >= PSSE_PARSER_TAP_RATIO_LBOUND && + x[1] <= PSSE_PARSER_TAP_RATIO_UBOUND + ) + ImpedanceCorrectionTransformerControlMode.TAP_RATIO + else + ImpedanceCorrectionTransformerControlMode.PHASE_SHIFT_ANGLE + end + + for winding_index in instances(WindingCategory) + ict_instances[(table_number, winding_index)] = ImpedanceCorrectionData(; + table_number = table_number, + impedance_correction_curve = pwl_data, + transformer_winding = winding_index, + transformer_control_mode = table_type, + ) + end + else + throw( + DataFormatError( + "Impedance correction mismatch at table $table_number: tap/angle and scaling count differs.", + ), + ) + end + end + + return ict_instances +end + +""" +Function to attach ICTs to a single Transformer component. +""" +function _attach_single_ict!( + sys::System, + transformer::Union{TwoWindingTransformer, ThreeWindingTransformer}, + name::String, + d::Dict, + table_key::String, + winding_idx::WindingCategory, + ict_instances::Dict{Tuple{Int64, WindingCategory}, ImpedanceCorrectionData}, +) + if isempty(ict_instances) + return + end + if haskey(d, table_key) + table_number = d[table_key] + cache_key = (table_number, winding_idx) + if haskey(ict_instances, cache_key) + ict = ict_instances[cache_key] + add_supplemental_attribute!(sys, transformer, ict) + else + @debug "No correction table associated with transformer $name for winding $winding_idx." + end + end + return +end + +""" +Attaches the corresponding ICT data to a Transformer2W component. +""" +function _attach_impedance_correction_tables!( + sys::System, + transformer::TwoWindingTransformer, + name::String, + d::Dict, + ict_instances::Dict{Tuple{Int64, WindingCategory}, ImpedanceCorrectionData}, +) + _attach_single_ict!( + sys, + transformer, + name, + d, + "correction_table", + WindingCategory.TR2W_WINDING, + ict_instances, + ) + return +end + +""" +Attaches the corresponding ICT data to a Transformer3W component. +""" +function _attach_impedance_correction_tables!( + sys::System, + transformer::ThreeWindingTransformer, + name::String, + d::Dict, + ict_instances::Dict{Tuple{Int64, WindingCategory}, ImpedanceCorrectionData}, +) + if isempty(ict_instances) + return + end + for winding_category in instances(WindingCategory) + winding_category == WindingCategory.TR2W_WINDING && continue + key = "$(WINDING_NAMES[winding_category])_correction_table" + _attach_single_ict!(sys, transformer, name, d, key, winding_category, ict_instances) + end + return +end + +""" +Creates a PowerSystems.ACBus from a PowerSystems bus dictionary +""" +function make_bus(bus_dict::Dict{String, Any}) + bus = ACBus( + bus_dict["number"], + bus_dict["name"], + bus_dict["available"], + bus_dict["bustype"], + bus_dict["angle"], + bus_dict["voltage"], + bus_dict["voltage_limits"], + bus_dict["base_voltage"], + bus_dict["area"], + bus_dict["zone"], + ) + return bus +end + +function make_bus( + bus_name::Union{String, SubString{String}}, + bus_number::Int, + d, + bus_types, + area::Area, +) + bus = make_bus( + Dict{String, Any}( + "name" => bus_name, + "number" => bus_number, + "available" => d["bus_status"], + "bustype" => bus_types[d["bus_type"]], + "angle" => d["va"], + "voltage" => d["vm"], + "voltage_limits" => (min = d["vmin"], max = d["vmax"]), + "base_voltage" => d["base_kv"], + "area" => area, + "zone" => nothing, + ), + ) + return bus +end + +# Disabling this because not all matpower files define areas even when bus definitions +# contain area references. +#function read_area!(sys::System, data::Dict; kwargs...) +# if !haskey(data, "areas") +# @info "There are no Areas in this file" +# return +# end +# +# for (key, val) in data["areas"] +# area = Area(string(val["col_1"])) +# add_component!(sys, area; skip_validation = SKIP_PM_VALIDATION) +# end +#end + +function read_bus!(sys::System, data::Dict; kwargs...) + @info "Reading bus data" + + bus_number_to_bus = Dict{Int, ACBus}() + + bus_types = instances(ACBusTypes) + unique_bus_names = true + bus_data = SortedDict{Int, Any}() + # Bus name uniqueness is not enforced by PSSE. This loop avoids forcing the users to have to + # pass the bus formatter always for larger datasets. + bus_names = Set{String}() + for (k, b) in data["bus"] + # If buses aren't unique stop searching and growing the set + if unique_bus_names && haskey(b, "name") + if b["name"] ∈ bus_names + unique_bus_names = false + end + push!(bus_names, b["name"]) + end + bus_data[k] = b + end + if isempty(bus_data) + @error "No bus data found" # TODO : need for a model without a bus + end + + default_bus_naming = x -> _get_pm_bus_name(x, unique_bus_names) + + _get_name = get(kwargs, :bus_name_formatter, default_bus_naming) + + default_area_naming = string + # The formatter for area_name should be a function that transform the Area Int to a String + _get_name_area = get(kwargs, :area_name_formatter, default_area_naming) + + for (i, (d_key, d)) in enumerate(bus_data) + # d id the data dict for each bus + # d_key is bus key + bus_name = strip(_get_name(d)) + bus_number = Int(d["bus_i"]) + + area_name = _get_name_area(d["area"]) + area = get_component(Area, sys, area_name) + if isnothing(area) + area = Area(area_name) + add_component!(sys, area; skip_validation = SKIP_PM_VALIDATION) + end + + # Store area data into ext dictionary + ext = Dict{String, Any}( + "ARNAME" => "", + "I" => "", + "ISW" => "", + "PDES" => "", + "PTOL" => "", + ) + if data["source_type"] == "pti" && haskey(data, "area_interchange") + for (_, area_data) in data["area_interchange"] + if haskey(area_data, "area_number") && + string(area_data["area_number"]) == area_name + ext["ARNAME"] = strip(get(area_data, "area_name", "")) + ext["I"] = string(get(area_data, "area_number", "")) + ext["ISW"] = string(get(area_data, "bus_number", "")) + ext["PDES"] = get(area_data, "net_interchange", "") + ext["PTOL"] = get(area_data, "tol_interchange", "") + break # Only one match is allowed + end + end + end + set_ext!(area, ext) + if !haskey(d, "bus_status") + d["bus_status"] = true + end + bus = make_bus(bus_name, bus_number, d, bus_types, area) + has_component(ACBus, sys, bus_name) && throw( + DataFormatError( + "Found duplicate bus names for $(get_name(bus)), consider reviewing your `bus_name_formatter` function", + ), + ) + + bus_number_to_bus[bus.number] = bus + add_component!(sys, bus; skip_validation = SKIP_PM_VALIDATION) + end + + if data["source_type"] == "pti" && haskey(data, "interarea_transfer") + # get Inter-area Transfers as AreaInterchange + for (k, d) in data["interarea_transfer"] + area_from_name = _get_name_area(d["area_from"]) + area_to_name = _get_name_area(d["area_to"]) + transfer_id = get(d, "transfer_id", "1") # 1 by default + + from_area = get_component(Area, sys, area_from_name) + to_area = get_component(Area, sys, area_to_name) + + name = "$(area_from_name)_$(area_to_name)_$(transfer_id)" + available = true + active_power_flow = d["power_transfer"] + flow_limits = (from_to = -INFINITE_BOUND, to_from = INFINITE_BOUND) + + ext = Dict{String, Any}( + "index" => d["index"], + "source_id" => ["interarea_transfer", k], + ) + + interarea_inter = AreaInterchange(; + name = name, + available = available, + active_power_flow = active_power_flow, + from_area = from_area, + to_area = to_area, + flow_limits = flow_limits, + ext = ext, + ) + + add_component!(sys, interarea_inter; skip_validation = SKIP_PM_VALIDATION) + end + end + + return bus_number_to_bus +end + +function make_interruptible_powerload(d::Dict, bus::ACBus, sys_mbase::Float64; kwargs...) + operation_cost = LoadCost(; + variable = zero(CostCurve), + fixed = 0.0, + ) + + _get_name = get(kwargs, :load_name_formatter, x -> strip(join(x["source_id"]))) + return InterruptiblePowerLoad(; + name = _get_name(d), + available = d["status"], + bus = bus, + active_power = d["pd"], + reactive_power = d["qd"], + max_active_power = d["pd"], + max_reactive_power = d["qd"], + base_power = sys_mbase, + operation_cost = operation_cost, + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function make_interruptible_standardload(d::Dict, bus::ACBus, sys_mbase::Float64; kwargs...) + operation_cost = LoadCost(; + variable = zero(CostCurve), + fixed = 0.0, + ) + + _get_name = get(kwargs, :load_name_formatter, x -> strip(join(x["source_id"]))) + return InterruptibleStandardLoad(; + name = _get_name(d), + available = d["status"], + bus = bus, + base_power = sys_mbase, + conformity = d["conformity"], + operation_cost = operation_cost, + constant_active_power = d["pd"], + constant_reactive_power = d["qd"], + current_active_power = d["pi"], + current_reactive_power = d["qi"], + impedance_active_power = d["py"], + impedance_reactive_power = d["qy"], + max_constant_active_power = d["pd"], + max_constant_reactive_power = d["qd"], + max_current_active_power = d["pi"], + max_current_reactive_power = d["qi"], + max_impedance_active_power = d["py"], + max_impedance_reactive_power = d["qy"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function make_power_load(d::Dict, bus::ACBus, sys_mbase::Float64; kwargs...) + _get_name = get(kwargs, :load_name_formatter, x -> strip(join(x["source_id"]))) + return PowerLoad(; + name = _get_name(d), + available = d["status"], + bus = bus, + active_power = d["pd"], + reactive_power = d["qd"], + max_active_power = d["pd"], + max_reactive_power = d["qd"], + base_power = sys_mbase, + conformity = d["conformity"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function make_standard_load(d::Dict, bus::ACBus, sys_mbase::Float64; kwargs...) + _get_name = get(kwargs, :load_name_formatter, x -> strip(join(x["source_id"]))) + return StandardLoad(; + name = _get_name(d), + available = d["status"], + bus = bus, + constant_active_power = d["pd"], + constant_reactive_power = d["qd"], + current_active_power = d["pi"], + current_reactive_power = d["qi"], + impedance_active_power = d["py"], + impedance_reactive_power = d["qy"], + max_constant_active_power = d["pd"], + max_constant_reactive_power = d["qd"], + max_current_active_power = d["pi"], + max_current_reactive_power = d["qi"], + max_impedance_active_power = d["py"], + max_impedance_reactive_power = d["qy"], + base_power = sys_mbase, + conformity = d["conformity"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function read_loads!(sys::System, data, bus_number_to_bus::Dict{Int, ACBus}; kwargs...) + @info "Reading Load data in PowerModels dict to populate System ..." + + if !haskey(data, "load") + @error "There are no loads in this file" + return + end + + sys_mbase = data["baseMVA"] + for d_key in keys(data["load"]) + d = data["load"][d_key] + bus = bus_number_to_bus[d["load_bus"]] + is_interruptible = haskey(d, "interruptible") + if data["source_type"] == "pti" && is_interruptible && d["interruptible"] != 1 + load = make_standard_load(d, bus, sys_mbase; kwargs...) + has_component(StandardLoad, sys, get_name(load)) && throw( + DataFormatError( + "Found duplicate load names of $(summary(load)), consider formatting names with `load_name_formatter` kwarg", + ), + ) + elseif data["source_type"] == "pti" && is_interruptible && d["interruptible"] == 1 + load = make_interruptible_standardload(d, bus, sys_mbase; kwargs...) + has_component(InterruptibleStandardLoad, sys, get_name(load)) && throw( + DataFormatError( + "Found duplicate interruptible load names of $(summary(load)), consider formatting names with `load_name_formatter` kwarg", + ), + ) + else + load = make_power_load(d, bus, sys_mbase; kwargs...) + has_component(PowerLoad, sys, get_name(load)) && throw( + DataFormatError( + "Found duplicate load names of $(summary(load)), consider formatting names with `load_name_formatter` kwarg", + ), + ) + end + add_component!(sys, load; skip_validation = SKIP_PM_VALIDATION) + end +end + +function make_loadzone( + name::String, + active_power::Float64, + reactive_power::Float64; + kwargs..., +) + return LoadZone(; + name = name, + peak_active_power = active_power, + peak_reactive_power = reactive_power, + ) +end + +function read_loadzones!( + sys::System, + data::Dict{String, Any}, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading LoadZones data in PowerModels dict to populate System ..." + zones = Set{Int}() + zone_bus_map = Dict{Int, Vector}() + for (_, bus) in data["bus"] + push!(zones, bus["zone"]) + push!(get!(zone_bus_map, bus["zone"], Vector()), bus) + end + + load_zone_map = + Dict{Int, Dict{String, Float64}}(i => Dict("pd" => 0.0, "qd" => 0.0) for i in zones) + for (key, load) in data["load"] + zone = data["bus"][load["load_bus"]]["zone"] + load_zone_map[zone]["pd"] += load["pd"] + load_zone_map[zone]["qd"] += load["qd"] + # Use get with defaults because matpower data doesn't have other load representations + load_zone_map[zone]["pd"] += get(load, "pi", 0.0) + load_zone_map[zone]["qd"] += get(load, "qi", 0.0) + load_zone_map[zone]["pd"] += get(load, "py", 0.0) + load_zone_map[zone]["qd"] += get(load, "qy", 0.0) + end + + default_loadzone_naming = string + # The formatter for loadzone_name should be a function that transform the LoadZone Int to a String + _get_name = get(kwargs, :loadzone_name_formatter, default_loadzone_naming) + + @info "Reading Zone data" + if !haskey(data, "zone") + @info "There is no Zone data in this file" + else + for (_, v) in data["zone"] + zone_number = v["zone_number"] + if !(zone_number in zones) + @warn "Skipping empty LoadZone $(zone_number)-$(v["zone_name"])" + end + end + end + + for zone in zones + name = _get_name(zone) + load_zone = make_loadzone( + name, + load_zone_map[zone]["pd"], + load_zone_map[zone]["qd"]; + kwargs..., + ) + add_component!(sys, load_zone; skip_validation = SKIP_PM_VALIDATION) + for bus in zone_bus_map[zone] + set_load_zone!(bus_number_to_bus[bus["bus_i"]], load_zone) + end + end +end + +function make_hydro_dispatch( + gen_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, + sys_mbase::Float64, +) + curtailcost = HydroGenerationCost(zero(CostCurve), 0.0) + + if d["mbase"] != 0.0 + mbase = d["mbase"] + else + @warn "Generator $gen_name has base power equal to zero: $(d["mbase"]). Changing it to system base: $sys_mbase" _group = + IS.LOG_GROUP_PARSING + mbase = sys_mbase + end + + base_conversion = sys_mbase / mbase + return HydroDispatch(; # No way to define storage parameters for gens in PM so can only make HydroDispatch + name = gen_name, + available = Bool(d["gen_status"]), + bus = bus, + active_power = d["pg"] * base_conversion, + reactive_power = d["qg"] * base_conversion, + rating = calculate_gen_rating(d["pmax"], d["qmax"], base_conversion), + prime_mover_type = parse_enum_mapping(PrimeMovers, d["type"]), + active_power_limits = ( + min = d["pmin"] * base_conversion, + max = d["pmax"] * base_conversion, + ), + reactive_power_limits = ( + min = d["qmin"] * base_conversion, + max = d["qmax"] * base_conversion, + ), + ramp_limits = calculate_ramp_limit(d, gen_name), + time_limits = nothing, + operation_cost = curtailcost, + base_power = mbase, + ) +end + +function make_hydro_reservoir( + gen_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, + sys_mbase::Float64, +) + curtailcost = HydroGenerationCost(zero(CostCurve), 0.0) + + if d["mbase"] != 0.0 + mbase = d["mbase"] + else + @warn "Generator $gen_name has base power equal to zero: $(d["mbase"]). Changing it to system base: $sys_mbase" _group = + IS.LOG_GROUP_PARSING + mbase = sys_mbase + end + + base_conversion = sys_mbase / mbase + return HydroDispatch(; # No way to define storage parameters for gens in PM so can only make HydroDispatch + name = gen_name, + available = Bool(d["gen_status"]), + bus = bus, + active_power = d["pg"] * base_conversion, + reactive_power = d["qg"] * base_conversion, + rating = calculate_gen_rating(d["pmax"], d["qmax"], base_conversion), + prime_mover_type = parse_enum_mapping(PrimeMovers, d["type"]), + active_power_limits = ( + min = d["pmin"] * base_conversion, + max = d["pmax"] * base_conversion, + ), + reactive_power_limits = ( + min = d["qmin"] * base_conversion, + max = d["qmax"] * base_conversion, + ), + ramp_limits = calculate_ramp_limit(d, gen_name), + time_limits = nothing, + operation_cost = curtailcost, + base_power = mbase, + ) +end + +function make_renewable_dispatch( + gen_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, + sys_mbase::Float64, +) + cost = RenewableGenerationCost(zero(CostCurve)) + + if d["mbase"] != 0.0 + mbase = d["mbase"] + else + @warn "Generator $gen_name has base power equal to zero: $(d["mbase"]). Changing it to system base: $sys_mbase" _group = + IS.LOG_GROUP_PARSING + mbase = sys_mbase + end + + base_conversion = sys_mbase / mbase + + rating = calculate_gen_rating(d["pmax"], d["qmax"], base_conversion) + if rating > mbase + @warn "rating is larger than base power for $gen_name, setting to $mbase" _group = + IS.LOG_GROUP_PARSING + rating = mbase + end + + generator = RenewableDispatch(; + name = gen_name, + available = Bool(d["gen_status"]), + bus = bus, + active_power = d["pg"] * base_conversion, + reactive_power = d["qg"] * base_conversion, + rating = rating * base_conversion, + prime_mover_type = parse_enum_mapping(PrimeMovers, d["type"]), + reactive_power_limits = ( + min = d["qmin"] * base_conversion, + max = d["qmax"] * base_conversion, + ), + power_factor = 1.0, + operation_cost = cost, + base_power = mbase, + ) + + return generator +end + +function make_renewable_fix( + gen_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, + sys_mbase::Float64, +) + if d["mbase"] != 0.0 + mbase = d["mbase"] + else + @warn "Generator $gen_name has base power equal to zero: $(d["mbase"]). Changing it to system base: $sys_mbase" _group = + IS.LOG_GROUP_PARSING + mbase = sys_mbase + end + + base_conversion = sys_mbase / mbase + generator = RenewableNonDispatch(; + name = gen_name, + available = Bool(d["gen_status"]), + bus = bus, + active_power = d["pg"] * base_conversion, + reactive_power = d["qg"] * base_conversion, + rating = float(d["pmax"]) * base_conversion, + prime_mover_type = parse_enum_mapping(PrimeMovers, d["type"]), + power_factor = 1.0, + base_power = mbase, + ) + + return generator +end + +function make_generic_battery( + storage_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, +) + energy_rating = iszero(d["energy_rating"]) ? d["energy"] : d["energy_rating"] + storage = EnergyReservoirStorage(; + name = storage_name, + available = Bool(d["status"]), + bus = bus, + prime_mover_type = PrimeMovers.BA, + storage_technology_type = StorageTech.OTHER_CHEM, + storage_capacity = energy_rating, + storage_level_limits = (min = 0.0, max = energy_rating), + initial_storage_capacity_level = d["energy"] / energy_rating, + rating = d["thermal_rating"], + active_power = d["ps"], + input_active_power_limits = (min = 0.0, max = d["charge_rating"]), + output_active_power_limits = (min = 0.0, max = d["discharge_rating"]), + efficiency = (in = d["charge_efficiency"], out = d["discharge_efficiency"]), + reactive_power = d["qs"], + reactive_power_limits = (min = d["qmin"], max = d["qmax"]), + base_power = d["thermal_rating"], + ) + return storage +end + +function _is_likely_motor_load(d::Dict, gen_name::Union{SubString{String}, String}) + # A motor load is likely if it has a negative active power and a non-zero reactive power. + # This is a heuristic and may not be accurate for all cases. + # likely_motor_load + if d["pmin"] < 0 && d["pmax"] < 0 && d["pg"] < 0 + @warn "Generator $gen_name is likely a motor load with negative active power: $(d["pg"]) and negative power limits: (min = $(d["pmin"]), max = $(d["pmax"])) \ + this component will be parsed as a thermal generator with negative active power limits. You can convert the device to a MotorLoad for more accurate modeling." + end + + if d["pmin"] == 0 && d["pmax"] == 0 && d["pg"] < 0 + @warn "Generator $gen_name is likely a motor load with negative active power: $(d["pg"]) and undefined active power limits \ + this component will be parsed as a thermal generator with negative active power injection. You can convert the device to a MotorLoad for more accurate modeling." + end + + if d["pmin"] < 0 && d["pmax"] == 0 + @warn "Generator $gen_name is likely something that is not a ThermalGenerators with negative power limits: (min = $(d["pmin"]), max = $(d["pmax"])) \ + this component will be parsed as a thermal generator with negative active power limits. Check this entry for more accurate modeling." + end + return +end + +# TODO test this more directly? +""" +The polynomial term follows the convention that for an n-degree polynomial, at least n + 1 components are needed. + c(p) = c_n*p^n+...+c_1p+c_0 + c_o is stored in the field in of the Econ Struct +""" +function make_thermal_gen( + gen_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, + sys_mbase::Float64, +) + if haskey(d, "model") + model = GeneratorCostModels(d["model"]) + # Input data layout: table B-4 of https://matpower.org/docs/MATPOWER-manual.pdf + if model == GeneratorCostModels.PIECEWISE_LINEAR + # For now, we make the fixed cost the y-intercept of the first segment of the + # piecewise curve and the variable cost a PiecewiseLinearData representing + # the data minus this fixed cost; in a future update, there will be no + # separation between the PiecewiseLinearData and the fixed cost. + cost_component = d["cost"] + power_p = [i for (ix, i) in enumerate(cost_component) if isodd(ix)] + cost_p = [i for (ix, i) in enumerate(cost_component) if iseven(ix)] + points = collect(zip(power_p, cost_p)) + (first_x, first_y) = first(points) + fixed = max(0.0, + first_y - first(get_slopes(PiecewiseLinearData(points))) * first_x, + ) + cost = PiecewiseLinearData([(x, y - fixed) for (x, y) in points]) + elseif model == GeneratorCostModels.POLYNOMIAL + # For now, we make the variable cost a QuadraticFunctionData with all but the + # constant term and make the fixed cost the constant term; in a future update, + # there will be no separation between the QuadraticFunctionData and the fixed + # cost. + # This transforms [3.0, 1.0, 4.0, 2.0] into [(1, 4.0), (2, 1.0), (3, 3.0)] + coeffs = enumerate(reverse(d["cost"][1:(end - 1)])) + coeffs = Dict((i, c / sys_mbase^i) for (i, c) in coeffs) + quadratic_degrees = [2, 1, 0] + (keys(coeffs) <= Set(quadratic_degrees)) || throw( + ArgumentError( + "Can only handle polynomials up to degree two; given coefficients $coeffs", + ), + ) + cost = QuadraticFunctionData(get.(Ref(coeffs), quadratic_degrees, 0)...) + fixed = (d["ncost"] >= 1) ? last(d["cost"]) : 0.0 + end + cost = CostCurve(InputOutputCurve((cost)), UnitSystem.DEVICE_BASE) + startup = d["startup"] + shutdn = d["shutdown"] + else + @warn "Generator cost data not included for Generator: $gen_name" + tmpcost = ThermalGenerationCost(nothing) + cost = tmpcost.variable + fixed = tmpcost.fixed + startup = tmpcost.start_up + shutdn = tmpcost.shut_down + end + + operation_cost = ThermalGenerationCost(; + variable = cost, + fixed = fixed, + start_up = startup, + shut_down = shutdn, + ) + + if !haskey(d, "ext") + d["ext"] = Dict{String, Float64}() + end + + if haskey(d, "r_source") && haskey(d, "x_source") + d["ext"]["r"] = d["r_source"] + d["ext"]["x"] = d["x_source"] + end + + if haskey(d, "rt_source") && haskey(d, "xt_source") + d["ext"]["rt"] = d["rt_source"] + d["ext"]["xt"] = d["xt_source"] + end + + if d["mbase"] != 0.0 + mbase = d["mbase"] + else + @warn "Generator $gen_name has base power equal to zero: $(d["mbase"]). Changing it to system base: $sys_mbase" _group = + IS.LOG_GROUP_PARSING + mbase = sys_mbase + end + + base_conversion = sys_mbase / mbase + _is_likely_motor_load(d, gen_name) + thermal_gen = ThermalStandard(; + name = gen_name, + status = Bool(d["gen_status"]), + available = Bool(d["gen_status"]), + bus = bus, + active_power = d["pg"] * base_conversion, + reactive_power = d["qg"] * base_conversion, + rating = calculate_gen_rating(d["pmax"], d["qmax"], base_conversion), + prime_mover_type = parse_enum_mapping(PrimeMovers, d["type"]), + fuel = parse_enum_mapping(ThermalFuels, d["fuel"]), + active_power_limits = ( + min = d["pmin"] * base_conversion, + max = d["pmax"] * base_conversion, + ), + reactive_power_limits = ( + min = d["qmin"] * base_conversion, + max = d["qmax"] * base_conversion, + ), + ramp_limits = calculate_ramp_limit(d, gen_name), + time_limits = nothing, + operation_cost = operation_cost, + base_power = mbase, + ext = get(d, "ext", Dict{String, Any}()), + ) + + return thermal_gen +end + +function make_synchronous_condenser( + gen_name::Union{SubString{String}, String}, + d::Dict, + bus::ACBus, + sys_mbase::Float64, +) + ext = get(d, "ext", Dict{String, Any}()) + if haskey(d, "r_source") && haskey(d, "x_source") + ext["r"] = d["r_source"] + ext["x"] = d["x_source"] + end + + if haskey(d, "rt_source") && haskey(d, "xt_source") + ext["rt"] = d["rt_source"] + ext["xt"] = d["xt_source"] + end + + if d["mbase"] != 0.0 + mbase = d["mbase"] + else + @warn "Generator $gen_name has base power equal to zero: $(d["mbase"]). Changing it to system base: $sys_mbase" _group = + IS.LOG_GROUP_PARSING + mbase = sys_mbase + end + + # NOTE: qmax and qmin can be both negatives, so this approach is taken for the rating. + base_conversion = sys_mbase / mbase + synchronous_condenser = SynchronousCondenser(; + name = gen_name, + available = Bool(d["gen_status"]), + bus = bus, + reactive_power = d["qg"] * base_conversion, + rating = max(abs(d["qmax"]), abs(d["qmin"])) * base_conversion, + reactive_power_limits = ( + min = d["qmin"] * base_conversion, + max = d["qmax"] * base_conversion, + ), + base_power = mbase, + ext = ext, + ) + + return synchronous_condenser +end + +""" +Transfer generators to ps_dict according to their classification +""" +function read_gen!(sys::System, data::Dict, bus_number_to_bus::Dict{Int, ACBus}; kwargs...) + @info "Reading generator data" + + if !haskey(data, "gen") + @error "There are no Generators in this file" + return nothing + end + + generator_mapping = get(kwargs, :generator_mapping, GENERATOR_MAPPING_FILE_PM) + try + generator_mapping = get_generator_mapping(generator_mapping) + catch e + @error "Error loading generator mapping $(generator_mapping)" + rethrow(e) + end + + sys_mbase = data["baseMVA"] + + _get_name = get(kwargs, :gen_name_formatter, _get_pm_dict_name) + for (name, pm_gen) in data["gen"] + gen_name = _get_name(pm_gen) + + bus = bus_number_to_bus[pm_gen["gen_bus"]] + pm_gen["fuel"] = get(pm_gen, "fuel", "OTHER") + pm_gen["type"] = get(pm_gen, "type", "OT") + @debug "Found generator" _group = IS.LOG_GROUP_PARSING gen_name bus pm_gen["fuel"] pm_gen["type"] + + gen_type = get_generator_type(pm_gen["fuel"], pm_gen["type"], generator_mapping) + if gen_type == ThermalStandard + generator = make_thermal_gen(gen_name, pm_gen, bus, sys_mbase) + elseif gen_type == HydroDispatch + generator = make_hydro_dispatch(gen_name, pm_gen, bus, sys_mbase) + elseif gen_type == HydroTurbine + # This method adds a + generator = make_hydro_reservoir(gen_name, pm_gen, bus, sys_mbase) + elseif gen_type == RenewableDispatch + generator = make_renewable_dispatch(gen_name, pm_gen, bus, sys_mbase) + elseif gen_type == RenewableNonDispatch + generator = make_renewable_fix(gen_name, pm_gen, bus, sys_mbase) + elseif gen_type == SynchronousCondenser + generator = make_synchronous_condenser(gen_name, pm_gen, bus, sys_mbase) + elseif gen_type == EnergyReservoirStorage + @warn "EnergyReservoirStorage should be defined as a PowerModels storage... Skipping" + continue + else + @error "Skipping unsupported generator" gen_type + continue + end + + has_component(typeof(generator), sys, get_name(generator)) && throw( + DataFormatError( + "Found duplicate $(typeof(generator)) names of $(get_name(generator)), consider formatting names with `gen_name_formatter` kwarg", + ), + ) + add_component!(sys, generator; skip_validation = SKIP_PM_VALIDATION) + end +end + +const _SHIFT_TO_GROUP_MAP = Dict{Float64, WindingGroupNumber}( + 0.0 => WindingGroupNumber.GROUP_0, + -30.0 => WindingGroupNumber.GROUP_1, + -150.0 => WindingGroupNumber.GROUP_5, + 180.0 => WindingGroupNumber.GROUP_6, + 150.0 => WindingGroupNumber.GROUP_7, + 30.0 => WindingGroupNumber.GROUP_11, +) + +function _add_vector_control_group(d::Dict, angle_key::String, group_key::String) + angle = d[angle_key] + for (angle_key_deg, group) in _SHIFT_TO_GROUP_MAP + if isapprox(rad2deg(angle), angle_key_deg) + d[group_key] = group + return + end + end + d[group_key] = WindingGroupNumber.UNDEFINED + return +end + +function get_branch_type_matpower( + d::Dict, +) + tap = d["tap"] + shift = d["shift"] + is_transformer = d["transformer"] + if !is_transformer + is_transformer = (tap != 0.0) && (tap != 1.0) || (shift != 0.0) + end + + is_transformer || return Line + + _add_vector_control_group(d, "shift", "group_number") + + if d["group_number"] == WindingGroupNumber.UNDEFINED + return PhaseShiftingTransformer + elseif tap != 1.0 + return TapTransformer + else + return Transformer2W + end +end + +function get_branch_type_psse( + d::Dict, +) + if d["br_r"] == 0.0 && d["br_x"] == 0.0 + return DiscreteControlledACBranch + end + + is_transformer = d["transformer"] + tap = d["tap"] + + if !is_transformer + if (tap != 0.0) && (tap != 1.0) + @warn "Transformer $d has tap ratio $tap, which is not 0.0 or 1.0; this is not a valid value for a Line. Parsing entry as a Transformer" + is_transformer = true + _add_vector_control_group(d, "shift", "group_number") + else + return Line + end + end + + _add_vector_control_group(d, "shift", "group_number") + is_tap_controllable, is_alpha_controllable = _determine_control_modes(d, "COD1", "tap") + if d["group_number"] == WindingGroupNumber.UNDEFINED || is_alpha_controllable + return PhaseShiftingTransformer + elseif (is_tap_controllable || (tap != 1.0)) && + d["group_number"] != WindingGroupNumber.UNDEFINED + return TapTransformer + elseif !is_tap_controllable && d["group_number"] != WindingGroupNumber.UNDEFINED + return Transformer2W + else + error("Couldn't infer the branch type for branch $d") + end +end + +function make_branch( + name::String, + d::Dict, + bus_f::ACBus, + bus_t::ACBus, + source_type::String; + kwargs..., +) + if source_type == "matpower" + branch_type = get_branch_type_matpower(d) + elseif source_type == "pti" + branch_type = get_branch_type_psse(d) + else + error("Source Type $source_type not supported") + end + + if d["transformer"] && branch_type == Line + throw( + DataFormatError( + "Branch data mismatched, cannot build the branch correctly for $d", + ), + ) + elseif branch_type == DiscreteControlledACBranch + value = _make_switch_from_zero_impedance_line(name, d, bus_f, bus_t) + elseif branch_type == Transformer2W + value = make_transformer_2w(name, d, bus_f, bus_t; kwargs...) + elseif branch_type == TapTransformer + value = make_tap_transformer(name, d, bus_f, bus_t; kwargs...) + elseif branch_type == PhaseShiftingTransformer + value = make_phase_shifting_transformer(name, d, bus_f, bus_t; kwargs...) + elseif branch_type == Line + value = make_line(name, d, bus_f, bus_t) + else + error("Unsupported branch type $branch_type") + end + return value +end + +function _make_switch_from_zero_impedance_line( + name::String, + d::Dict, + bus_f::ACBus, + bus_t::ACBus, +) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + available_value = d["br_status"] == 1 + if get_bustype(bus_f) == ACBusTypes.ISOLATED || + get_bustype(bus_t) == ACBusTypes.ISOLATED + available_value = false + end + if available_value == true + status_value = DiscreteControlledBranchStatus.CLOSED + else + status_value = DiscreteControlledBranchStatus.OPEN + end + @warn "Branch $name has zero impedance and available = $available_value; converting to a DiscreteControlledACBranch of type SWITCH with available = $available_value and branch_status = $status_value" + return DiscreteControlledACBranch(; + name = name, + available = Bool(available_value), + active_power_flow = pf, + reactive_power_flow = qf, + arc = Arc(bus_f, bus_t), + r = d["br_r"], + x = d["br_x"], + rating = _get_rating("Line", name, d, "rate_a"), + discrete_branch_type = DiscreteControlledBranchType.SWITCH, + branch_status = status_value, + ) +end + +function _get_rating( + branch_type::String, + name::AbstractString, + line_data::Dict, + key::String, +) + haskey(line_data, key) || return key == "rate_a" ? INFINITE_BOUND : nothing + + if isapprox(line_data[key], 0.0) + @info( + "$branch_type $name rating value: $(line_data[key]). Unbounded value implied as per PSSe Manual" + ) + return INFINITE_BOUND + else + return line_data[key] + end +end + +function make_line(name::String, d::Dict, bus_f::ACBus, bus_t::ACBus) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + available_value = d["br_status"] == 1 + if get_bustype(bus_f) == ACBusTypes.ISOLATED || + get_bustype(bus_t) == ACBusTypes.ISOLATED + available_value = false + end + + ext = haskey(d, "ext") ? d["ext"] : Dict{String, Any}() + + return Line(; + name = name, + available = available_value, + active_power_flow = pf, + reactive_power_flow = qf, + arc = Arc(bus_f, bus_t), + r = d["br_r"], + x = d["br_x"], + b = (from = d["b_fr"], to = d["b_to"]), + rating = _get_rating("Line", name, d, "rate_a"), + angle_limits = (min = d["angmin"], max = d["angmax"]), + rating_b = _get_rating("Line", name, d, "rate_b"), + rating_c = _get_rating("Line", name, d, "rate_c"), + ext = ext, + ) +end + +function make_switch_breaker(name::String, d::Dict, bus_f::ACBus, bus_t::ACBus) + return DiscreteControlledACBranch(; + name = name, + available = Bool(d["state"]), + active_power_flow = d["active_power_flow"], + reactive_power_flow = d["reactive_power_flow"], + arc = Arc(bus_f, bus_t), + r = d["r"], + x = d["x"], + rating = d["rating"], + discrete_branch_type = d["discrete_branch_type"], + branch_status = d["state"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function read_switch_breaker!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}, + device_type::String; + kwargs..., +) + @info "Reading $device_type data" + if !haskey(data, device_type) + @info "There is no $device_type data in this file" + return + end + + _get_name = get(kwargs, :branch_name_formatter, _get_pm_branch_name) + + for (_, d) in data[device_type] + bus_f = bus_number_to_bus[d["f_bus"]] + bus_t = bus_number_to_bus[d["t_bus"]] + name = _get_name(d, bus_f, bus_t) + value = make_switch_breaker(name, d, bus_f, bus_t) + + add_component!(sys, value; skip_validation = SKIP_PM_VALIDATION) + end +end + +function make_transformer_2w( + name::String, + d::Dict, + bus_f::ACBus, + bus_t::ACBus; + kwargs..., +) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + available_value = d["br_status"] == 1 + if get_bustype(bus_f) == ACBusTypes.ISOLATED || + get_bustype(bus_t) == ACBusTypes.ISOLATED + available_value = false + end + + resistance_formatter = get(kwargs, :transformer_resistance_formatter, nothing) + r = resistance_formatter !== nothing ? resistance_formatter(name) : d["br_r"] + reactance_formatter = get(kwargs, :transformer_reactance_formatter, nothing) + x = reactance_formatter !== nothing ? reactance_formatter(name) : d["br_x"] + + return Transformer2W(; + name = name, + available = available_value, + active_power_flow = pf, + reactive_power_flow = qf, + arc = Arc(bus_f, bus_t), + r = r, + x = x, + primary_shunt = d["g_fr"] + im * d["b_fr"], + winding_group_number = d["group_number"], + rating = _get_rating("Transformer2W", name, d, "rate_a"), + rating_b = _get_rating("Transformer2W", name, d, "rate_b"), + rating_c = _get_rating("Transformer2W", name, d, "rate_c"), + base_power = d["base_power"], + # for psse inputs, these numbers may be different than the buses' base voltages + base_voltage_primary = d["base_voltage_from"], + base_voltage_secondary = d["base_voltage_to"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function make_3w_transformer( + name::String, + d::Dict, + bus_primary::ACBus, + bus_secondary::ACBus, + bus_tertiary::ACBus, + star_bus::ACBus, +) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + return Transformer3W(; + name = name, + available = d["available"], + primary_star_arc = Arc(bus_primary, star_bus), + secondary_star_arc = Arc(bus_secondary, star_bus), + tertiary_star_arc = Arc(bus_tertiary, star_bus), + star_bus = star_bus, + active_power_flow_primary = pf, + reactive_power_flow_primary = qf, + active_power_flow_secondary = pf, + reactive_power_flow_secondary = qf, + active_power_flow_tertiary = pf, + reactive_power_flow_tertiary = qf, + r_primary = d["r_primary"], + x_primary = d["x_primary"], + r_secondary = d["r_secondary"], + x_secondary = d["x_secondary"], + r_tertiary = d["r_tertiary"], + x_tertiary = d["x_tertiary"], + rating = d["rating"], + r_12 = d["r_12"], + x_12 = d["x_12"], + r_23 = d["r_23"], + x_23 = d["x_23"], + r_13 = d["r_13"], + x_13 = d["x_13"], + base_power_12 = d["base_power_12"], + base_power_23 = d["base_power_23"], + base_power_13 = d["base_power_13"], + base_voltage_primary = d["base_voltage_primary"], + base_voltage_secondary = d["base_voltage_secondary"], + base_voltage_tertiary = d["base_voltage_tertiary"], + g = d["g"], + b = d["b"], + primary_turns_ratio = d["primary_turns_ratio"], + secondary_turns_ratio = d["secondary_turns_ratio"], + tertiary_turns_ratio = d["tertiary_turns_ratio"], + available_primary = d["available_primary"], + available_secondary = d["available_secondary"], + available_tertiary = d["available_tertiary"], + rating_primary = _get_rating("Transformer3W", name, d, "rating_primary"), + rating_secondary = _get_rating("Transformer3W", name, d, "rating_secondary"), + rating_tertiary = _get_rating("Transformer3W", name, d, "rating_tertiary"), + primary_group_number = d["primary_group_number"], + secondary_group_number = d["secondary_group_number"], + tertiary_group_number = d["tertiary_group_number"], + control_objective_primary = get(d, "COD1", -99), + control_objective_secondary = get(d, "COD2", -99), + control_objective_tertiary = get(d, "COD3", -99), + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function make_3w_phase_shifting_transformer( + name::String, + d::Dict, + bus_primary::ACBus, + bus_secondary::ACBus, + bus_tertiary::ACBus, + star_bus::ACBus, +) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + return PhaseShiftingTransformer3W(; + name = name, + available = d["available"], + primary_star_arc = Arc(bus_primary, star_bus), + secondary_star_arc = Arc(bus_secondary, star_bus), + tertiary_star_arc = Arc(bus_tertiary, star_bus), + star_bus = star_bus, + active_power_flow_primary = pf, + reactive_power_flow_primary = qf, + active_power_flow_secondary = pf, + reactive_power_flow_secondary = qf, + active_power_flow_tertiary = pf, + reactive_power_flow_tertiary = qf, + r_primary = d["r_primary"], + x_primary = d["x_primary"], + r_secondary = d["r_secondary"], + x_secondary = d["x_secondary"], + r_tertiary = d["r_tertiary"], + x_tertiary = d["x_tertiary"], + rating = d["rating"], + r_12 = d["r_12"], + x_12 = d["x_12"], + r_23 = d["r_23"], + x_23 = d["x_23"], + r_13 = d["r_13"], + x_13 = d["x_13"], + α_primary = d["primary_phase_shift_angle"], + α_secondary = d["secondary_phase_shift_angle"], + α_tertiary = d["tertiary_phase_shift_angle"], + base_power_12 = d["base_power_12"], + base_power_23 = d["base_power_23"], + base_power_13 = d["base_power_13"], + base_voltage_primary = d["base_voltage_primary"], + base_voltage_secondary = d["base_voltage_secondary"], + base_voltage_tertiary = d["base_voltage_tertiary"], + g = d["g"], + b = d["b"], + primary_turns_ratio = d["primary_turns_ratio"], + secondary_turns_ratio = d["secondary_turns_ratio"], + tertiary_turns_ratio = d["tertiary_turns_ratio"], + available_primary = d["available_primary"], + available_secondary = d["available_secondary"], + available_tertiary = d["available_tertiary"], + rating_primary = _get_rating( + "PhaseShiftingTransformer3W", + name, + d, + "rating_primary", + ), + rating_secondary = _get_rating( + "PhaseShiftingTransformer3W", + name, + d, + "rating_secondary", + ), + rating_tertiary = _get_rating( + "PhaseShiftingTransformer3W", + name, + d, + "rating_tertiary", + ), + control_objective_primary = get(d, "COD1", -99), + control_objective_secondary = get(d, "COD2", -99), + control_objective_tertiary = get(d, "COD3", -99), + ext = d["ext"], + ) +end + +function make_tap_transformer( + name::String, + d::Dict, + bus_f::ACBus, + bus_t::ACBus; + kwargs..., +) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + available_value = d["br_status"] == 1 + if get_bustype(bus_f) == ACBusTypes.ISOLATED || + get_bustype(bus_t) == ACBusTypes.ISOLATED + available_value = false + end + + ext = haskey(d, "ext") ? d["ext"] : Dict{String, Any}() + control_objective_formatter = + get(kwargs, :transformer_control_objective_formatter, nothing) + control_objective = if control_objective_formatter !== nothing + result = control_objective_formatter(name) + result !== nothing ? result : get(d, "COD1", -99) + else + get(d, "COD1", -99) + end + resistance_formatter = get(kwargs, :transformer_resistance_formatter, nothing) + r = if resistance_formatter !== nothing + result = resistance_formatter(name) + result !== nothing ? result : d["br_r"] + else + d["br_r"] + end + reactance_formatter = get(kwargs, :transformer_reactance_formatter, nothing) + x = if reactance_formatter !== nothing + result = reactance_formatter(name) + result !== nothing ? result : d["br_x"] + else + d["br_x"] + end + tap_formatter = get(kwargs, :transformer_tap_formatter, nothing) + tap = if tap_formatter !== nothing + result = tap_formatter(name) + result !== nothing ? result : d["tap"] + else + d["tap"] + end + + return TapTransformer(; + name = name, + available = available_value, + active_power_flow = pf, + reactive_power_flow = qf, + arc = Arc(bus_f, bus_t), + r = r, + x = x, + tap = tap, + primary_shunt = d["g_fr"] + im * d["b_fr"], + winding_group_number = d["group_number"], + base_power = d["base_power"], + rating = _get_rating("TapTransformer", name, d, "rate_a"), + rating_b = _get_rating("TapTransformer", name, d, "rate_b"), + rating_c = _get_rating("TapTransformer", name, d, "rate_c"), + # for psse inputs, these numbers may be different than the buses' base voltages + base_voltage_primary = d["base_voltage_from"], + base_voltage_secondary = d["base_voltage_to"], + control_objective = control_objective, + ext = ext, + ) +end + +function make_phase_shifting_transformer( + name::String, + d::Dict, + bus_f::ACBus, + bus_t::ACBus; + kwargs..., +) + pf = get(d, "pf", 0.0) + qf = get(d, "qf", 0.0) + available_value = d["br_status"] == 1 + if get_bustype(bus_f) == ACBusTypes.ISOLATED || + get_bustype(bus_t) == ACBusTypes.ISOLATED + available_value = false + end + + ext = haskey(d, "ext") ? d["ext"] : Dict{String, Any}() + control_objective_formatter = + get(kwargs, :transformer_control_objective_formatter, nothing) + control_objective = if control_objective_formatter !== nothing + result = control_objective_formatter(name) + result !== nothing ? result : get(d, "COD1", -99) + else + get(d, "COD1", -99) + end + resistance_formatter = get(kwargs, :transformer_resistance_formatter, nothing) + r = resistance_formatter !== nothing ? resistance_formatter(name) : d["br_r"] + reactance_formatter = get(kwargs, :transformer_reactance_formatter, nothing) + x = reactance_formatter !== nothing ? reactance_formatter(name) : d["br_x"] + tap_formatter = get(kwargs, :transformer_tap_formatter, nothing) + tap = tap_formatter !== nothing ? tap_formatter(name) : d["tap"] + + return PhaseShiftingTransformer(; + name = name, + available = available_value, + active_power_flow = pf, + reactive_power_flow = qf, + arc = Arc(bus_f, bus_t), + r = r, + x = x, + tap = tap, + primary_shunt = d["g_fr"] + im * d["b_fr"], + α = d["shift"], + base_power = d["base_power"], + rating = _get_rating("PhaseShiftingTransformer", name, d, "rate_a"), + rating_b = _get_rating("PhaseShiftingTransformer", name, d, "rate_b"), + rating_c = _get_rating("PhaseShiftingTransformer", name, d, "rate_c"), + # for psse inputs, these numbers may be different than the buses' base voltages + base_voltage_primary = d["base_voltage_from"], + base_voltage_secondary = d["base_voltage_to"], + control_objective = control_objective, + ext = ext, + ) +end + +function read_branch!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; kwargs..., +) + @info "Reading branch data" + if !haskey(data, "branch") + @info "There is no Branch data in this file" + return + end + + _get_name = get(kwargs, :branch_name_formatter, _get_pm_branch_name) + ict_instances = _impedance_correction_table_lookup(data) + + source_type = data["source_type"] + for d in values(data["branch"]) + bus_f = bus_number_to_bus[d["f_bus"]] + bus_t = bus_number_to_bus[d["t_bus"]] + name = _get_name(d, bus_f, bus_t) + value = make_branch(name, d, bus_f, bus_t, source_type; kwargs...) + + if !isnothing(value) + add_component!(sys, value; skip_validation = SKIP_PM_VALIDATION) + else + continue + end + + if isa(value, TwoWindingTransformer) + _attach_impedance_correction_tables!( + sys, + value, + name, + d, + ict_instances, + ) + end + end + return +end + +function read_3w_transformer!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading 3W transformer data" + if !haskey(data, "3w_transformer") + @info "There is no 3W transformer data in this file" + return + end + + _get_name = get(kwargs, :xfrm_3w_name_formatter, _get_pm_3w_name) + + ict_instances = _impedance_correction_table_lookup(data) + + for (_, d) in data["3w_transformer"] + bus_primary = bus_number_to_bus[d["bus_primary"]] + bus_secondary = bus_number_to_bus[d["bus_secondary"]] + bus_tertiary = bus_number_to_bus[d["bus_tertiary"]] + star_bus = bus_number_to_bus[d["star_bus"]] + + name = _get_name(d, bus_primary, bus_secondary, bus_tertiary) + three_winding_transformer_type = get_three_winding_transformer_type(d) + if three_winding_transformer_type == PhaseShiftingTransformer3W + value = make_3w_phase_shifting_transformer( + name, + d, + bus_primary, + bus_secondary, + bus_tertiary, + star_bus, + ) + elseif three_winding_transformer_type == Transformer3W + value = make_3w_transformer( + name, + d, + bus_primary, + bus_secondary, + bus_tertiary, + star_bus, + ) + else + error( + "Unsupported three winding transformer type $three_winding_transformer_type", + ) + end + + add_component!(sys, value; skip_validation = SKIP_PM_VALIDATION) + + _attach_impedance_correction_tables!(sys, value, name, d, ict_instances) + end +end + +function _determine_control_modes(d::Dict, control_flag::String, tap_key::String) + control_code = get(d, control_flag, -99) + tap = d[tap_key] + + is_tap_controllable = false + is_alpha_controllable = false + + # There is no control + if control_code == 0 + is_tap_controllable = false + is_alpha_controllable = false + # Reactive Power Control + elseif control_code ∈ [1, -1] + is_tap_controllable = true + is_alpha_controllable = false + # Voltage Control + elseif control_code ∈ [2, -2] + is_tap_controllable = true + is_alpha_controllable = false + # Active Power Control + elseif control_code ∈ [3, -3] + is_tap_controllable = true + is_alpha_controllable = true + # DC Line Control + elseif control_code ∈ [4, -4] + is_tap_controllable = true + is_alpha_controllable = true + # Asymmetric Active Power Control + elseif control_code ∈ [5, -5] + is_tap_controllable = true + is_alpha_controllable = true + elseif control_code == -99 + @warn "Can't determine control objective for the transformer from the $(control_flag) field for $d" + if d["shift"] != 0.0 + is_alpha_controllable = true + elseif (tap != 0.0) || (tap != 1.0) + is_tap_controllable = true + else + @warn "Can't determine control objective for the other fields. Will return a Transformer2W" + end + else + error(d) + end + return is_tap_controllable, is_alpha_controllable +end + +function get_three_winding_transformer_type(d::Dict) + _add_vector_control_group(d, "primary_phase_shift_angle", "primary_group_number") + _add_vector_control_group(d, "secondary_phase_shift_angle", "secondary_group_number") + _add_vector_control_group(d, "tertiary_phase_shift_angle", "tertiary_group_number") + # NOTE: with current three winding transformer type hierarchy, tap controllable and not controllable three winding transformers are Transformer3W + _, primary_is_alpha_controllable = + _determine_control_modes(d, "COD1", "primary_turns_ratio") + _, secondary_is_alpha_controllable = + _determine_control_modes(d, "COD2", "secondary_turns_ratio") + _, tertiary_is_alpha_controllable = + _determine_control_modes(d, "COD3", "tertiary_turns_ratio") + if d["primary_group_number"] == WindingGroupNumber.UNDEFINED || + d["secondary_group_number"] == WindingGroupNumber.UNDEFINED || + d["tertiary_group_number"] == WindingGroupNumber.UNDEFINED || + primary_is_alpha_controllable || secondary_is_alpha_controllable || + tertiary_is_alpha_controllable + return PhaseShiftingTransformer3W + else + return Transformer3W + end +end + +function make_dcline(name::String, d::Dict, bus_f::ACBus, bus_t::ACBus, source_type::String) + if source_type == "pti" + return TwoTerminalLCCLine(; + name = name, + available = d["available"], + arc = Arc(bus_f, bus_t), + active_power_flow = get(d, "pf", 0.0), + r = d["r"], + transfer_setpoint = d["transfer_setpoint"], + scheduled_dc_voltage = d["scheduled_dc_voltage"], + rectifier_bridges = d["rectifier_bridges"], + rectifier_delay_angle_limits = d["rectifier_delay_angle_limits"], + rectifier_rc = d["rectifier_rc"], + rectifier_xc = d["rectifier_xc"], + rectifier_base_voltage = d["rectifier_base_voltage"], + inverter_bridges = d["inverter_bridges"], + inverter_extinction_angle_limits = d["inverter_extinction_angle_limits"], + inverter_rc = d["inverter_rc"], + inverter_xc = d["inverter_xc"], + inverter_base_voltage = d["inverter_base_voltage"], + power_mode = d["power_mode"], + switch_mode_voltage = d["switch_mode_voltage"], + compounding_resistance = d["compounding_resistance"], + min_compounding_voltage = d["min_compounding_voltage"], + rectifier_transformer_ratio = d["rectifier_transformer_ratio"], + rectifier_tap_setting = d["rectifier_tap_setting"], + rectifier_tap_limits = d["rectifier_tap_limits"], + rectifier_tap_step = d["rectifier_tap_step"], + rectifier_delay_angle = d["rectifier_delay_angle"], + rectifier_capacitor_reactance = d["rectifier_capacitor_reactance"], + inverter_transformer_ratio = d["inverter_transformer_ratio"], + inverter_tap_setting = d["inverter_tap_setting"], + inverter_tap_limits = d["inverter_tap_limits"], + inverter_tap_step = d["inverter_tap_step"], + inverter_extinction_angle = d["inverter_extinction_angle"], + inverter_capacitor_reactance = d["inverter_capacitor_reactance"], + active_power_limits_from = d["active_power_limits_from"], + active_power_limits_to = d["active_power_limits_to"], + reactive_power_limits_from = d["reactive_power_limits_from"], + reactive_power_limits_to = d["reactive_power_limits_to"], + loss = LinearCurve(d["loss1"], d["loss0"]), + ext = get(d, "ext", Dict{String, Any}()), + ) + elseif source_type == "matpower" + return TwoTerminalGenericHVDCLine(; + name = name, + available = d["br_status"] == 1, + active_power_flow = get(d, "pf", 0.0), + arc = Arc(bus_f, bus_t), + active_power_limits_from = (min = d["pminf"], max = d["pmaxf"]), + active_power_limits_to = (min = d["pmint"], max = d["pmaxt"]), + reactive_power_limits_from = (min = d["qminf"], max = d["qmaxf"]), + reactive_power_limits_to = (min = d["qmint"], max = d["qmaxt"]), + loss = LinearCurve(d["loss1"], d["loss0"]), + ) + else + error("Not supported source type for DC lines: $source_type") + end +end + +function read_dcline!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}, + source_type::String; + kwargs..., +) + @info "Reading DC Line data" + if !haskey(data, "dcline") + @info "There is no DClines data in this file" + return + end + + _get_name = get(kwargs, :dcline_name_formatter, _get_pm_branch_name) + + for (d_key, d) in data["dcline"] + d["name"] = get(d, "name", d_key) + bus_f = bus_number_to_bus[d["f_bus"]] + bus_t = bus_number_to_bus[d["t_bus"]] + name = _get_name(d, bus_f, bus_t) + dcline = make_dcline(name, d, bus_f, bus_t, source_type) + add_component!(sys, dcline; skip_validation = SKIP_PM_VALIDATION) + end +end + +function make_vscline(name::String, d::Dict, bus_f::ACBus, bus_t::ACBus) + return TwoTerminalVSCLine(; + name = name, + available = d["available"], + arc = Arc(bus_f, bus_t), + active_power_flow = get(d, "pf", 0.0), + rating = d["rating"], + active_power_limits_from = (min = d["pminf"], max = d["pmaxf"]), + active_power_limits_to = (min = d["pmint"], max = d["pmaxt"]), + g = d["r"] == 0.0 ? 0.0 : 1.0 / d["r"], + dc_current = get(d, "if", 0.0), + reactive_power_from = get(d, "qf", 0.0), + dc_voltage_control_from = d["dc_voltage_control_from"], + ac_voltage_control_from = d["ac_voltage_control_from"], + dc_setpoint_from = d["dc_setpoint_from"], + ac_setpoint_from = d["ac_setpoint_from"], + converter_loss_from = d["converter_loss_from"], + max_dc_current_from = d["max_dc_current_from"], + rating_from = d["rating_from"], + reactive_power_limits_from = (min = d["qminf"], max = d["qmaxf"]), + power_factor_weighting_fraction_from = d["power_factor_weighting_fraction_from"], + reactive_power_to = get(d, "qt", 0.0), + dc_voltage_control_to = d["dc_voltage_control_to"], + ac_voltage_control_to = d["ac_voltage_control_to"], + dc_setpoint_to = d["dc_setpoint_to"], + ac_setpoint_to = d["ac_setpoint_to"], + converter_loss_to = d["converter_loss_to"], + max_dc_current_to = d["max_dc_current_to"], + rating_to = d["rating_to"], + reactive_power_limits_to = (min = d["qmint"], max = d["qmaxt"]), + power_factor_weighting_fraction_to = d["power_factor_weighting_fraction_to"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function read_vscline!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading VSC Line data" + if !haskey(data, "vscline") + @info "There is no VSC lines data in this file" + return + end + + _get_name = get(kwargs, :vsc_line_name_formatter, _get_pm_branch_name) + + for (d_key, d) in data["vscline"] + d["name"] = get(d, "name", d_key) + bus_f = bus_number_to_bus[d["f_bus"]] + bus_t = bus_number_to_bus[d["t_bus"]] + name = _get_name(d, bus_f, bus_t) + vscline = make_vscline(name, d, bus_f, bus_t) + add_component!(sys, vscline; skip_validation = SKIP_PM_VALIDATION) + end +end + +function make_switched_shunt(name::String, d::Dict, bus::ACBus) + params = Dict( + :name => name, + :available => Bool(d["status"]), + :bus => bus, + :Y => (d["gs"] + d["bs"]im), + :number_of_steps => d["step_number"], + :Y_increase => d["y_increment"], + :admittance_limits => d["admittance_limits"], + :ext => d["ext"], + ) + + if haskey(d, "initial_status") + params[:initial_status] = d["initial_status"] + end + + return SwitchedAdmittance(; params...) +end + +function read_switched_shunt!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading switched shunt data" + if !haskey(data, "switched_shunt") + @info "There is no switched shunt data in this file" + return + end + + _get_name = get(kwargs, :switched_shunt_name_formatter, _get_pm_dict_name) + + for (d_key, d) in data["switched_shunt"] + d["name"] = get(d, "name", d_key) + name = _get_name(d) + bus = bus_number_to_bus[d["shunt_bus"]] + shunt = make_switched_shunt(name, d, bus) + + add_component!(sys, shunt; skip_validation = SKIP_PM_VALIDATION) + end +end + +function make_shunt(name::String, d::Dict, bus::ACBus) + return FixedAdmittance(; + name = name, + available = Bool(d["status"]), + bus = bus, + Y = (d["gs"] + d["bs"]im), + ) +end + +function make_facts(name::String, d::Dict, bus::ACBus) + if d["tbus"] != 0 + @warn "Series FACTs not supported." + end + + if d["control_mode"] > 3 + throw(DataFormatError("Operation mode not supported.")) + end + + if d["reactive_power_required"] < 0 + throw(DataFormatError("% MVAr required must me positive.")) + end + + return FACTSControlDevice(; + name = name, + available = Bool(d["available"]), + bus = bus, + control_mode = d["control_mode"], + voltage_setpoint = d["voltage_setpoint"], + max_shunt_current = d["max_shunt_current"], + reactive_power_required = d["reactive_power_required"], + ext = get(d, "ext", Dict{String, Any}()), + ) +end + +function read_facts!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading FACTS data" + if !haskey(data, "facts") + @info "There is no facts data in this file" + return + end + + _get_name = get(kwargs, :bus_name_formatter, _get_pm_dict_name) + + for (d_key, d) in data["facts"] + d["name"] = get(d, "name", d_key) + name = _get_name(d) + bus = bus_number_to_bus[d["bus"]] + full_name = "$(d["bus"])_$(name)" + facts = make_facts(full_name, d, bus) + + add_component!(sys, facts; skip_validation = SKIP_PM_VALIDATION) + end +end + +function read_shunt!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading shunt data" + if !haskey(data, "shunt") + @info "There is no shunt data in this file" + return + end + + _get_name = get(kwargs, :shunt_name_formatter, _get_pm_dict_name) + + for (d_key, d) in data["shunt"] + d["name"] = get(d, "name", d_key) + name = _get_name(d) + bus = bus_number_to_bus[d["shunt_bus"]] + shunt = make_shunt(name, d, bus) + + add_component!(sys, shunt; skip_validation = SKIP_PM_VALIDATION) + end +end + +function read_storage!( + sys::System, + data::Dict, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading storage data" + if !haskey(data, "storage") + @info "There is no storage data in this file" + return + end + + _get_name = get(kwargs, :gen_name_formatter, _get_pm_dict_name) + + for (d_key, d) in data["storage"] + d["name"] = get(d, "name", d_key) + name = _get_name(d) + bus = bus_number_to_bus[d["storage_bus"]] + storage = make_generic_battery(name, d, bus) + + add_component!(sys, storage; skip_validation = SKIP_PM_VALIDATION) + end +end diff --git a/src/powerflowdata_data.jl b/src/powerflowdata_data.jl new file mode 100644 index 0000000..73a4a08 --- /dev/null +++ b/src/powerflowdata_data.jl @@ -0,0 +1,822 @@ +"""Container for data parsed by PowerFlowData""" +struct PowerFlowDataNetwork + data::PowerFlowData.Network +end + +""" +Constructs PowerFlowDataNetwork from a raw file. +Currently Supports PSSE data files v30, v32 and v33 +""" +function PowerFlowDataNetwork(file::Union{String, IO}; kwargs...) + return PowerFlowDataNetwork(PowerFlowData.parse_network(file)) +end + +""" +Constructs a System from PowerModelsData. + +# Arguments +- `pfd_data::Union{PowerFlowDataNetwork, Union{String, IO}}`: PowerModels data object or supported +load flow case (*.m, *.raw) + +# Keyword arguments +- `ext::Dict`: Contains user-defined parameters. Should only contain standard types. +- `runchecks::Bool`: Run available checks on input fields and when add_component! is called. + Throws InvalidValue if an error is found. +- `time_series_in_memory::Bool=false`: Store time series data in memory instead of HDF5. +- `config_path::String`: specify path to validation config file +- `pm_data_corrections::Bool=true` : Run the PowerModels data corrections (aka :validate in PowerModels) +- `import_all:Bool=false` : Import all fields from PTI files + +# Examples +```julia +sys = System( + pm_data, config_path = "ACTIVSg25k_validation.json", + bus_name_formatter = x->string(x["name"]*"-"*string(x["index"])), + load_name_formatter = x->strip(join(x["source_id"], "_")) +) +``` +""" +function System(net_data::PowerFlowDataNetwork; kwargs...) + runchecks = get(kwargs, :runchecks, true) + data = net_data.data + if length(data.buses) < 1 + throw(DataFormatError("There are no buses in this file.")) + end + + @info "Constructing System from PowerFlowData version v$(data.caseid.rev)" + + if isa(data.caseid.sbase, Missing) + error("Base power not specified in .raw file. Data parsing can not continue") + end + + if isa(data.caseid.basfrq, Missing) + @warn "Frequency value missing from .raw file. Using default 60 Hz" + frequency = 60.0 + else + frequency = data.caseid.basfrq + end + + sys = System(data.caseid.sbase; frequency = frequency, kwargs...) + bus_number_to_bus = read_bus!(sys, data.buses, data; kwargs...) + read_loads!(sys, data.loads, data.caseid.sbase, bus_number_to_bus; kwargs...) + read_gen!(sys, data.generators, data.caseid.sbase, bus_number_to_bus; kwargs...) + read_branch!( + sys, + data.branches, + data.caseid.sbase, + bus_number_to_bus; + kwargs..., + ) + read_branch!(sys, data.transformers, data.caseid.sbase, bus_number_to_bus; kwargs...) + read_shunt!(sys, data.fixed_shunts, data.caseid.sbase, bus_number_to_bus; kwargs...) + read_switched_shunt!( + sys, + data.switched_shunts, + data.caseid.sbase, + bus_number_to_bus; + kwargs..., + ) + read_dcline!( + sys, + data.two_terminal_dc, + data.caseid.sbase, + bus_number_to_bus; + kwargs..., + ) + read_dcline!( + sys, + data.multi_terminal_dc, + data.caseid.sbase, + bus_number_to_bus; + kwargs..., + ) + + read_dcline!( + sys, + data.vsc_dc, + data.caseid.sbase, + bus_number_to_bus; + kwargs..., + ) + + if runchecks + check(sys) + end + return sys +end + +function read_bus!( + sys::System, + buses::PowerFlowData.Buses33, + data::PowerFlowData.Network; + kwargs..., +) + bus_number_to_bus = Dict{Int, ACBus}() + bus_types = instances(ACBusTypes) + + for ix in eachindex(buses.i) + # d id the data dict for each bus + # d_key is bus key + bus_name = strip(buses.name[ix]) * "_$(buses.i[ix])" * "_$(buses.ide[ix])" + has_component(ACBus, sys, bus_name) && throw( + DataFormatError( + "Found duplicate bus names of $bus_name, consider formatting names with `bus_name_formatter` kwarg", + ), + ) + bus_number = buses.i[ix] + if isempty(data.area_interchanges) + area_name = string(buses.area[ix]) + @debug "File doesn't contain area names" + else + area_ix = findfirst(data.area_interchanges.i .== buses.area[ix]) + area_name = data.area_interchanges.arname[area_ix] + end + + area = get_component(Area, sys, area_name) + if isnothing(area) + area = Area(area_name) + add_component!(sys, area; skip_validation = SKIP_PM_VALIDATION) + end + + # TODO: LoadZones need to be created and populated here + if isempty(data.zones) + zone_name = string(buses.zone[ix]) + @debug "File doesn't contain load zones" + else + zone_ix = findfirst(data.zones.i .== buses.zone[ix]) + zone_name = "$(data.zones.zoname[zone_ix])_$(data.zones.i[zone_ix])" + end + zone = get_component(LoadZone, sys, zone_name) + if isnothing(zone) + zone = LoadZone(zone_name, 0.0, 0.0) + add_component!(sys, zone; skip_validation = SKIP_PM_VALIDATION) + end + + zone = get_component(LoadZone, sys, zone_name) + if isnothing(zone) + zone = LoadZone(zone_name, 0.0, 0.0) + add_component!(sys, zone; skip_validation = SKIP_PM_VALIDATION) + end + + bus = ACBus( + bus_number, + bus_name, + true, + bus_types[buses.ide[ix]], + clamp(buses.va[ix] * (π / 180), -π / 2, π / 2), + buses.vm[ix], + (min = buses.nvlo[ix], max = buses.nvhi[ix]), + buses.basekv[ix], + area, + zone, + ) + + bus_number_to_bus[bus_number] = bus + + add_component!(sys, bus; skip_validation = SKIP_PM_VALIDATION) + end + # TODO: Checking for surplus Areas or LoadZones in the data which don't get populated in the sys above + # but are available in the raw file + if ~isempty(data.area_interchanges) + for area_name in data.area_interchanges.arname + area = get_component(Area, sys, area_name) + if isnothing(area) + area = Area(area_name) + add_component!(sys, area; skip_validation = SKIP_PM_VALIDATION) + end + end + end + + if ~isempty(data.zones) + for (i, name) in zip(data.zones.i, data.zones.zoname) + zone_name = "$(name)_$(i)" + zone = get_component(LoadZone, sys, zone_name) + if isnothing(zone) + zone = LoadZone(zone_name, 0.0, 0.0) + add_component!(sys, zone; skip_validation = SKIP_PM_VALIDATION) + end + end + end + return bus_number_to_bus +end + +function read_bus!( + sys::System, + buses::PowerFlowData.Buses30, + data::PowerFlowData.Network; + kwargs..., +) + bus_number_to_bus = Dict{Int, ACBus}() + + bus_types = instances(ACBusTypes) + + for ix in 1:length(buses) + # d id the data dict for each bus + # d_key is bus key + bus_name = strip(buses.name[ix]) * "_$(buses.i[ix])" * "_$(buses.ide[ix])" + has_component(ACBus, sys, bus_name) && throw( + DataFormatError( + "Found duplicate bus names of $bus_name, consider formatting names with `bus_name_formatter` kwarg", + ), + ) + bus_number = buses.i[ix] + if isempty(data.area_interchanges) + area_name = string(buses.area[ix]) + @debug "File doesn't contain area names" + else + area_ix = data.area_interchanges.i .== buses.area[ix] + if all(.!area_ix) + error("Area numbering is incorrectly specified in PSSe file") + end + area_name = first(data.area_interchanges.arname[area_ix]) + end + area = get_component(Area, sys, area_name) + if isnothing(area) + area = Area(area_name) + add_component!(sys, area; skip_validation = SKIP_PM_VALIDATION) + end + + # TODO: LoadZones need to be created and populated here + + bus = ACBus( + bus_number, + bus_name, + true, + bus_types[buses.ide[ix]], + clamp(buses.va[ix] * (π / 180), -π / 2, π / 2), + buses.vm[ix], + nothing, # PSSe 30 data doesn't have magnitude limits + buses.basekv[ix], + area, + ) + + bus_number_to_bus[bus_number] = bus + + add_component!(sys, bus; skip_validation = SKIP_PM_VALIDATION) + + if buses.bl[ix] > 0 || buses.gl[ix] > 0 + shunt = FixedAdmittance(bus_name, true, bus, buses.gl[ix] + 1im * buses.bl[ix]) + add_component!(sys, shunt; skip_validation = SKIP_PM_VALIDATION) + end + end + + return bus_number_to_bus +end + +function read_loads!( + sys::System, + loads::PowerFlowData.Loads, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + if isempty(loads) + @error "There are no loads in this file" + return + end + for ix in eachindex(loads.i) + bus = bus_number_to_bus[loads.i[ix]] + load_name = "load-$(get_name(bus))~$(loads.id[ix])" + if has_component(StandardLoad, sys, load_name) + throw(DataFormatError("Found duplicate load names of $(load_name)")) + end + + load = StandardLoad(; + name = load_name, + available = loads.status[ix], + bus = bus, + constant_active_power = loads.pl[ix] / sys_mbase, + constant_reactive_power = loads.ql[ix] / sys_mbase, + impedance_active_power = loads.yp[ix] / sys_mbase, + impedance_reactive_power = loads.yq[ix] / sys_mbase, + current_active_power = loads.ip[ix] / sys_mbase, + current_reactive_power = loads.iq[ix] / sys_mbase, + max_constant_active_power = loads.pl[ix] / sys_mbase, + max_constant_reactive_power = loads.ql[ix] / sys_mbase, + max_impedance_active_power = loads.yp[ix] / sys_mbase, + max_impedance_reactive_power = loads.yq[ix] / sys_mbase, + max_current_active_power = loads.ip[ix] / sys_mbase, + max_current_reactive_power = loads.iq[ix] / sys_mbase, + base_power = sys_mbase, + ) + + add_component!(sys, load; skip_validation = SKIP_PM_VALIDATION) + end + # Populate Areas and LoadZones with peak active and reactive power + areas = get_components(Area, sys) + if ~isnothing(areas) + for area in areas + area_comps = get_components_in_aggregation_topology(StandardLoad, sys, area) + if (isempty(area_comps)) + set_peak_active_power!(area, 0.0) + set_peak_reactive_power!(area, 0.0) + else + set_peak_active_power!( + area, + sum(get.(get_ext.(area_comps), "active_power_load", 0.0)), + ) + set_peak_reactive_power!( + area, + sum(get.(get_ext.(area_comps), "reactive_power_load", 0.0)), + ) + end + end + end + zones = get_components(LoadZone, sys) + if ~isnothing(zones) + for zone in zones + zone_comps = get_components_in_aggregation_topology(StandardLoad, sys, zone) + if (isempty(zone_comps)) + set_peak_active_power!(zone, 0.0) + set_peak_reactive_power!(zone, 0.0) + else + set_peak_active_power!( + zone, + sum(get.(get_ext.(zone_comps), "active_power_load", 0.0)), + ) + set_peak_reactive_power!( + zone, + sum(get.(get_ext.(zone_comps), "reactive_power_load", 0.0)), + ) + end + end + end + return nothing +end + +function _get_active_power_limits( + pt::Float64, + pb::Float64, + machine_base::Float64, + system_base::Float64, +) + min_p = 0.0 + if pb < 0.0 + @info "Min power in dataset is negative, active_power_limits minimum set to 0.0" + else + min_p = pb + end + + if machine_base != system_base && pt >= machine_base + @info "Max active power limit is $(pt/machine_base) than the generator base. Check the data" + end + + return (min = min_p / machine_base, max = pt / machine_base) +end + +function read_gen!( + sys::System, + gens::PowerFlowData.Generators, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading generator data" + + if isempty(gens) + @error "There are no generators in this file" + return + end + + for ix in eachindex(gens.i) + bus = get(bus_number_to_bus, gens.i[ix], nothing) + if isnothing(bus) + error("Incorrect bus id for generator $(gens.i[ix])-$(gens.id[ix])") + end + + gen_name = "gen-$(get_name(bus))~$(gens.id[ix])" + if has_component(ThermalStandard, sys, gen_name) + throw(DataFormatError("Found duplicate load names of $(gen_name)")) + end + ireg_bus_num = gens.ireg[ix] == 0 ? gens.i[ix] : gens.ireg[ix] + + thermal_gen = ThermalStandard(; + name = gen_name, + available = gens.stat[ix] > 0 ? true : false, + status = gens.stat[ix] > 0 ? true : false, + bus = bus, + active_power = gens.pg[ix] / gens.mbase[ix], + reactive_power = gens.qg[ix] / gens.mbase[ix], + active_power_limits = _get_active_power_limits( + gens.pt[ix], + gens.pb[ix], + gens.mbase[ix], + sys_mbase, + ), + reactive_power_limits = ( + min = gens.qb[ix] / sys_mbase, + max = gens.qt[ix] / sys_mbase, + ), + base_power = gens.mbase[ix], + rating = gens.mbase[ix], + ramp_limits = nothing, + time_limits = nothing, + operation_cost = ThermalGenerationCost(nothing), + ext = Dict( + "IREG" => ireg_bus_num, + "WMOD" => gens.wmod[ix], + "WPF" => gens.wpf[ix], + ), + ) + + add_component!(sys, thermal_gen; skip_validation = SKIP_PM_VALIDATION) + end + return nothing +end + +function read_branch!( + sys::System, + branches::PowerFlowData.Branches30, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading line data" + + if isempty(branches) + @error "There are no lines in this file" + return + end + + for ix in eachindex(branches.i) + if branches.i[ix] < 0 + i_ix = abs(branches.i[ix]) + @warn "Branch index $(branches.i[ix]) corrected to $i_ix" + end + + if branches.j[ix] < 0 + j_ix = abs(branches.i[ix]) + @warn "Branch index $(branches.j[ix]) corrected to $j_ix" + end + + bus_from = bus_number_to_bus[abs(branches.i[ix])] + bus_to = bus_number_to_bus[abs(branches.j[ix])] + branch_name = "line-$(get_name(bus_from))-$(get_name(bus_to))-$(branches.ckt[ix])" + max_rate = max(branches.rate_a[ix], branches.rate_b[ix], branches.rate_c[ix]) + if max_rate == 0.0 + max_rate = abs(1 / (branches.r[ix] + 1im * branches.x[ix])) * sys_mbase + end + branch = Line(; + name = branch_name, + available = branches.st[ix] > 0 ? true : false, + active_power_flow = 0.0, + reactive_power_flow = 0.0, + arc = Arc(bus_from, bus_to), + r = branches.r[ix], + x = branches.x[ix], + b = (from = branches.bi[ix], to = branches.bj[ix]), + angle_limits = (min = -π / 2, max = π / 2), + rating = max_rate, + ) + + add_component!(sys, branch; skip_validation = SKIP_PM_VALIDATION) + end + + return nothing +end + +function read_branch!( + sys::System, + branches::PowerFlowData.Branches33, + sys_mbase::Float64, + rating_flag::Int8, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading line data" + + if isempty(branches) + @error "There are no lines in this file" + return + end + + for ix in eachindex(branches.i) + bus_from = bus_number_to_bus[branches.i[ix]] + bus_to = bus_number_to_bus[branches.j[ix]] + branch_name = "line-$(get_name(bus_from))-$(get_name(bus_to))~$(branches.ckt[ix])" + + max_rate = max(branches.rate_a[ix], branches.rate_b[ix], branches.rate_c[ix]) + + if get_base_voltage(bus_from) != get_base_voltage(bus_to) + @warn("bad line data $branch_name. Transforming this Line to Transformer2W.") + # Method needed for NTPS to make this data into a transformer + transformer_name = "transformer-$(get_name(bus_from))-$(get_name(bus_to))~$(branches.ckt[ix])" + transformer = Transformer2W(; + name = transformer_name, + available = branches.st[ix] > 0 ? true : false, + active_power_flow = 0.0, + reactive_power_flow = 0.0, + arc = Arc(bus_from, bus_to), + r = branches.r[ix], + x = branches.x[ix], + primary_shunt = 0.0, + winding_group_number = WindingGroupNumber(0), + rating = max_rate, + base_power = get_base_power(sys), # add system base power + ext = Dict( + "line_to_xfr" => true, + ), + ) + add_component!(sys, transformer; skip_validation = SKIP_PM_VALIDATION) + + continue + end + + rated_current = 0.0 + if (rating_flag > 0) + rated_current = (max_rate / (sqrt(3) * get_base_voltage(bus_from))) * 10^3 + end + + branch = Line(; + name = branch_name, + available = branches.st[ix] > 0 ? true : false, + active_power_flow = 0.0, + reactive_power_flow = 0.0, + arc = Arc(bus_from, bus_to), + r = branches.r[ix], + x = branches.x[ix], + b = (from = branches.bi[ix], to = branches.bj[ix]), + angle_limits = (min = -π / 2, max = π / 2), + rating = max_rate, + ext = Dict( + "length" => branches.len[ix], + "rated_current(A)" => rated_current, + ), + ) + + add_component!(sys, branch; skip_validation = SKIP_PM_VALIDATION) + end + + return nothing +end + +function read_branch!( + sys::System, + transformers::PowerFlowData.Transformers, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading transformer data" + + if isempty(transformers) + @error "There are no transformers in this file" + return + end + + for ix in eachindex(transformers.i) + bus_i = bus_number_to_bus[transformers.i[ix]] + bus_j = bus_number_to_bus[transformers.j[ix]] + if transformers.k[ix] > 0 + @error "Three-winding transformer from PowerFlowData inputs not implemented. Data will be ignored" + continue + else + to_from_name = "$(get_name(bus_i))-$(get_name(bus_j))" + end + + if transformers.ang1[ix] != 0 + @error "Phase Shifting transformer from PowerFlowData inputs not implemented. Data will be ignored" + continue + end + + transformer_name = "transformer-$to_from_name-$(transformers.ckt[ix])" + + if !(transformers.cz[ix] in [1, 2, 3]) + @warn( + "transformer CZ value outside of valid bounds assuming the default value of 1. Given $(transformer["CZ"]), should be 1, 2 or 3", + ) + transformers.cz[ix] = 1 + end + + if !(transformers.cw[ix] in [1, 2, 3]) + @warn( + "transformer CW value outside of valid bounds assuming the default value of 1. Given $(transformer["CW"]), should be 1, 2 or 3", + ) + transformers.cw[ix] = 1 + end + + if !(transformers.cm[ix] in [1, 2]) + @warn( + "transformer CM value outside of valid bounds assuming the default value of 1. Given $(transformer["CM"]), should be 1 or 2", + ) + transformers.cm[ix] = 1 + end + + # Unit Transformations + if transformers.cz[ix] == 1 # "for resistance and reactance in pu on system MVA base and winding voltage base" + br_r, br_x = transformers.r1_2[ix], transformers.x1_2[ix] + else # NOT "for resistance and reactance in pu on system MVA base and winding voltage base" + if transformers.cz[ix] == 3 # "for transformer load loss in watts and impedance magnitude in pu on a specified MVA base and winding voltage base." + br_r = 1e-6 * transformers.r1_2[ix] / transformers.sbase1_2[ix] + br_x = sqrt(transformers.x1_2[ix]^2 - br_r^2) + else + br_r, br_x = transformers.r1_2[ix], transformers.x1_2[ix] + end + per_unit_factor = + ( + transformers.nomv1[ix]^2 / + get_base_voltage(bus_i)^2 + ) * (sys_mbase / transformers.sbase1_2[ix]) + if per_unit_factor == 0 + @warn "Per unit conversion for transformer $to_from_name couldn't be done, assuming system base instead. Check field NOMV1 is valid" + per_unit_factor = 1 + end + br_r *= per_unit_factor + br_x *= per_unit_factor + end + + # Zeq scaling for tap2 (see eq (4.21b) in PROGRAM APPLICATION GUIDE 1 in PSSE installation folder) + # Unit Transformations + if transformers.cw[ix] == 1 # "for off-nominal turns ratio in pu of winding bus base voltage" + br_r *= transformers.windv2[ix]^2 + br_x *= transformers.windv2[ix]^2 + else # NOT "for off-nominal turns ratio in pu of winding bus base voltage" + if transformers.cw[ix] == 2 # "for winding voltage in kV" + br_r *= + ( + transformers.windv2[ix] / + get_base_voltage(bus_j) + )^2 + br_x *= + ( + transformers.windv2[ix] / + get_base_voltage(bus_j) + )^2 + else # "for off-nominal turns ratio in pu of nominal winding voltage, NOMV1, NOMV2 and NOMV3." + br_r *= + ( + transformers.windv2[ix] * ( + transformers.nomv2[ix] / + get_base_voltage(bus_j) + ) + )^2 + br_x *= + ( + transformers.windv2[ix] * ( + transformers.nomv2[ix] / + get_base_voltage(bus_j) + ) + )^2 + end + end + + max_rate = + max(transformers.rata1[ix], transformers.ratb1[ix], transformers.ratc1[ix]) + + tap_value = transformers.windv1[ix] / transformers.windv2[ix] + + # Unit Transformations + if transformers.cw[ix] != 1 # NOT "for off-nominal turns ratio in pu of winding bus base voltage" + tap_value *= get_base_voltage(bus_j) / get_base_voltage(bus_i) + if transformers.cw[ix] == 3 # "for off-nominal turns ratio in pu of nominal winding voltage, NOMV1, NOMV2 and NOMV3." + tap_value *= transformers.nomv1[ix] / transformers.nomv2[ix] + end + end + + transformer = TapTransformer(; + name = transformer_name, + available = transformers.stat[ix] > 0 ? true : false, + active_power_flow = 0.0, + reactive_power_flow = 0.0, + arc = Arc(bus_i, bus_j), + r = br_r, + x = br_x, + tap = tap_value, + primary_shunt = transformers.mag2[ix], + winding_group_number = WindingGroupNumber(0), + base_power = get_base_power(sys), + rating = max_rate, + ) + add_component!(sys, transformer; skip_validation = SKIP_PM_VALIDATION) + end + + return nothing +end + +function read_shunt!( + ::System, + ::Nothing, + ::Float64, + ::Dict{Int, ACBus}; + kwargs..., +) + @debug "No data for Fixed Shunts" + return +end + +function read_shunt!( + sys::System, + data::PowerFlowData.FixedShunts, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @error "FixedShunts parsing from PowerFlowData inputs not implemented. Data will be ignored" + return +end + +function read_switched_shunt!( + sys::System, + ::Nothing, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @debug "No data for Switched Shunts" + return +end + +function read_switched_shunt!( + sys::System, + ::PowerFlowData.SwitchedShunts30, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @error "SwitchedShunts parsing from PSS/e v30 files not implemented. Data will be ignored" + return +end + +function read_switched_shunt!( + sys::System, + data::PowerFlowData.SwitchedShunts33, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @info "Reading line data" + @warn "All switched shunts will be converted to fixed shunts" + + if isempty(data) + @error "There are no lines in this file" + return + end + + @error "SwitchedShunts parsing from PSS/e v33 files not implemented. Data will be ignored" + return +end + +function read_dcline!( + sys::System, + ::Nothing, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @debug "No data for HVDC Line" + return +end + +function read_dcline!( + sys::System, + data::PowerFlowData.TwoTerminalDCLines30, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @error "TwoTerminalDCLines parsing from PSS/e v30 files not implemented. Data will be ignored" + return +end + +function read_dcline!( + sys::System, + data::PowerFlowData.TwoTerminalDCLines33, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + return +end + +function read_dcline!( + sys::System, + data::PowerFlowData.VSCDCLines, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @error "VSCDCLines parsing from PSS/e files not implemented. Data will be ignored" + return +end + +function read_dcline!( + sys::System, + data::PowerFlowData.MultiTerminalDCLines{PowerFlowData.DCLineID30}, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @error "MultiTerminalDCLines parsing from PSS/e files v30 not implemented. Data will be ignored" + return +end + +function read_dcline!( + sys::System, + data::PowerFlowData.MultiTerminalDCLines{PowerFlowData.DCLineID33}, + sys_mbase::Float64, + bus_number_to_bus::Dict{Int, ACBus}; + kwargs..., +) + @error "MultiTerminalDCLines parsing from PSS/e files v30 not implemented. Data will be ignored" + return +end diff --git a/test/Project.toml b/test/Project.toml index 153933c..95493f6 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,5 +1,9 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1" +PowerFlowFileParser = "bed98974-b02a-5e2f-9ee0-a103f5c450dd" +PowerSystemCaseBuilder = "f00506e0-b84f-492a-93c2-c0a9afc4364e" +PowerSystems = "bcd98974-b02a-5e2f-9ee0-a103f5c450dd" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] diff --git a/test/modified_14bus_system.raw b/test/modified_14bus_system.raw new file mode 100644 index 0000000..9781eb0 --- /dev/null +++ b/test/modified_14bus_system.raw @@ -0,0 +1,129 @@ +0, 100.00, 33, 0, 1, 60.00 / PSS(R)E 33 RAW created by rawd33 THU, APR 03 2025 15:09 + + + 101,'BUS 101 ', 138.0000,3, 1, 1, 1,1.00000, 0.0000,1.10000,0.90000,1.10000,0.90000 + 102,'BUS 102 ', 138.0000,1, 1, 1, 1,0.99896, -0.3060,1.10000,0.90000,1.10000,0.90000 + 103,'BUS 103 ', 138.0000,1, 1, 1, 1,1.00307, -1.7958,1.10000,0.90000,1.10000,0.90000 + 104,'BUS 104 ', 138.0000,1, 1, 1, 1,0.99365, -2.8783,1.10000,0.90000,1.10000,0.90000 + 105,'BUS 105 ', 138.0000,1, 1, 1, 1,0.99365, -2.8783,1.10000,0.90000,1.10000,0.90000 + 106,'BUS 106 ', 230.0000,1, 1, 1, 1,0.99394, -2.9090,1.10000,0.90000,1.10000,0.90000 + 107,'BUS 107 ', 65.0000,1, 1, 1, 1,0.99476, -2.8576,1.10000,0.90000,1.10000,0.90000 + 108,'BUS 108 ', 65.0000,3, 1, 1, 1,1.00000, 11.7593,1.10000,0.90000,1.10000,0.90000 + 109,'BUS 109 ', 500.0000,1, 1, 1, 1,0.99627, -2.9203,1.10000,0.90000,1.10000,0.90000 + 110,'BUS 110 ', 230.0000,2, 1, 1, 1,1.00000, -2.9926,1.10000,0.90000,1.10000,0.90000 + 111,'BUS 111 ', 230.0000,1, 1, 1, 1,0.98996, -3.0408,1.10000,0.90000,1.10000,0.90000 + 112,'BUS 112 ', 230.0000,1, 1, 1, 1,0.99941, -3.0972,1.10000,0.90000,1.10000,0.90000 + 113,'BUS 113 ', 230.0000,1, 1, 1, 1,0.99941, -3.0972,1.10000,0.90000,1.10000,0.90000 + 114,'BUS 114 ', 500.0000,1, 1, 1, 1,0.99959, -3.1078,1.10000,0.90000,1.10000,0.90000 + 200,' BUS 200 ', 500.0000,1, 1, 1, 1,0.99965, -3.0851,1.10000,0.90000,1.10000,0.90000 + 201,'BUS 201 ', 500.0000,1, 1, 1, 1,0.99920, -3.1048,1.10000,0.90000,1.10000,0.90000 + 301,'BUS 301 ', 500.0000,1, 1, 1, 1,0.99912, -3.1502,1.10000,0.90000,1.10000,0.90000 + 401,'BUS 401 ', 500.0000,1, 1, 1, 1,0.99889, -2.9786,1.10000,0.90000,1.10000,0.90000 + 501,'BUS 501 ', 500.0000,1, 1, 1, 1,0.99894, -3.0975,1.10000,0.90000,1.10000,0.90000 + 601,'BUS 601 ', 500.0000,1, 1, 1, 1,0.99662, -2.9299,1.10000,0.90000,1.10000,0.90000 +0 / END OF BUS DATA, BEGIN LOAD DATA + 101,'1 ',1, 1, 1, 200.000, 500.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 102,'1 ',1, 1, 1, 200.000, -350.000, 20.000, 0.000, 0.000, 0.000, 1,1,0 + 103,'1 ',1, 1, 1, 105.000, 65.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 104,'1 ',1, 1, 1, 500.000, 180.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 105,'1 ',1, 1, 1, 500.000, -350.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 106,'1 ',1, 1, 1, 80.000, 20.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 109,'1 ',1, 1, 1, 300.000, 100.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 110,'1 ',1, 1, 1, 100.000, -500.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 110,'2 ',1, 1, 1, 150.000, -500.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 111,'1 ',1, 1, 1, 250.000, 100.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 112,'1 ',1, 1, 1, 325.000, 23.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 113,'1 ',1, 1, 1, 250.000, 100.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 + 114,'1 ',1, 1, 1, 500.000, -250.000, 0.000, 0.000, 0.000, 0.000, 1,1,0 +0 / END OF LOAD DATA, BEGIN FIXED SHUNT DATA + 102,'1 ',1, 50.000, -300.000 + 106,'1 ',1, 80.000, 30.000 + 110,'1 ',1, 20.000, 400.000 + 111,'1 ',1, 100.000, 200.000 +0 / END OF FIXED SHUNT DATA, BEGIN GENERATOR DATA + 101,'1 ', 2088.167, -2446.801, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 + 101,'2 ', 1089.478, -1276.592, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 + 102,'1 ', 300.000, -50.000, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 + 106,'1 ', 200.000, -50.000, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 + 108,'1 ', 500.273, -4.997, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 + 110,'1 ', 300.000, 2708.623, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 + 111,'1 ', 90.000, 784.759, 9999.000, -9999.000,1.00000, 0, 100.000, 0.00000E+0, 1.00000E+0, 0.00000E+0, 0.00000E+0,1.00000,1, 100.0, 9999.000, -9999.000, 1,1.0000 +0 / END OF GENERATOR DATA, BEGIN BRANCH DATA + 101, 102,'1 ', 1.25000E-4, 1.00000E-4, 0.00120, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 101, 105,'1 ', 2.50000E-3, 1.00000E-4, 0.00230, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 102, 103,'1 ', 1.28000E-3, 8.10000E-3, 0.00196, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 102, 104,'1 ', 3.40000E-3, 4.10000E-3, 0.00026, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 102, 105,'1 ', 1.88000E-3, 1.10400E-3, 0.00315, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 103, 104,'1 ', 7.45000E-3, 3.15000E-3, 0.00342, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 104, 105,'*1',-0.00000E+0, 1.00000E-4, 0.00000, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 106, 111,'1 ', 1.34000E-3, 1.34500E-4, 0.00024, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 106, 111,'2 ', 1.25000E-2, 8.10000E-3, 0.00535, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 106, 112,'1 ', 1.45600E-3, 4.10000E-3, 0.00734, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 107, 108,'1 ', 7.98000E-3, 5.01000E-2, 0.00056, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 109, 401,'1 ', 1.23000E-3, 7.10000E-3, 0.00340, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 109, 501,'1 ', 7.53000E-2, 4.10000E-3, 0.00350, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 109, 601,'1 ', 1.88000E-3, 2.10000E-3, 0.05634, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,2, 0.00, 1,1.0000 + 110, 111,'1 ', 3.56000E-3, 2.41000E-2, 0.00502, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 112, 113,'@1',-0.00000E+0, 1.00000E-4, 0.00000, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 114, 200,'1 ', 1.24000E-3, 1.00000E-4, 0.04400, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 114, 201,'1 ', 6.44300E-3, 5.10000E-3, 0.00235, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 114, 301,'1 ', 2.30000E-3, 2.01000E-2, 0.06400, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 200, 401,'1 ', 4.53000E-3, 3.10000E-3, 0.04580, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 201, 501,'1 ', 5.34300E-3, 2.31000E-3, 0.00556, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 + 301, 601,'1 ', 5.23000E-2, 6.01000E-3, 0.05340, 0.00, 0.00, 0.00, 0.00000, 0.00000, 0.00000, 0.00000,1,1, 0.00, 1,1.0000 +0 / END OF BRANCH DATA, BEGIN TRANSFORMER DATA + 109, 104, 0,'1 ',1,1,1, 0.00000E+0, 0.00000E+0,2,'TRAFO 2W 3 ',1, 1,1.0000, 0,1.0000, 0,1.0000, 0,1.0000,' ' + 0.00000E+0, 1.00000E-4, 100.00 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 4, 0.00000, 0.00000, 0.000 +1.00000, 0.000 + 106, 105, 0,'1 ',1,1,1, 0.00000E+0, 0.00000E+0,2,'TRAFO 2W 1 ',1, 1,1.0000, 0,1.0000, 0,1.0000, 0,1.0000,' ' + 0.00000E+0, 1.00000E-4, 100.00 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 7, 0.00000, 0.00000, 0.000 +1.00000, 0.000 + 110, 109, 0,'1 ',1,1,1, 0.00000E+0, 0.00000E+0,2,'TRAFO 2W 2 ',1, 1,1.0000, 0,1.0000, 0,1.0000, 0,1.0000,' ' + 0.00000E+0, 1.00000E-4, 100.00 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 3, 0.00000, 0.00000, 0.000 +1.00000, 0.000 + 109, 104, 107,'1 ',1,1,1, 0.00000E+0, 0.00000E+0,2,'TRAFO 3W 2 ',1, 1,1.0000, 0,1.0000, 0,1.0000, 0,1.0000,' ' + 0.00000E+0, 2.00000E-4, 100.00, 0.00000E+0, 2.00000E-4, 100.00, 0.00000E+0, 2.00000E-4, 100.00,0.99489, -2.8854 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 1, 0.00000, 0.00000, 0.000 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 8, 0.00000, 0.00000, 0.000 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 4, 0.00000, 0.00000, 0.000 + 113, 110, 114,'1 ',1,1,1, 0.00000E+0, 0.00000E+0,2,'TRAFO 3W 1 ',1, 1,1.0000, 0,1.0000, 0,1.0000, 0,1.0000,' ' + 0.00000E+0, 2.00000E-4, 100.00, 0.00000E+0, 2.00000E-4, 100.00, 0.00000E+0, 2.00000E-4, 100.00,0.99967, -3.0658 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 9, 0.00000, 0.00000, 0.000 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 8, 0.00000, 0.00000, 0.000 +1.00000, 0.000, 0.000, 0.00, 0.00, 0.00, 0, 0, 1.10000, 0.90000, 1.10000, 0.90000, 33, 9, 0.00000, 0.00000, 0.000 +0 / END OF TRANSFORMER DATA, BEGIN AREA DATA +0 / END OF AREA DATA, BEGIN TWO-TERMINAL DC DATA +'LINE 1',1, 20.0000, 500.00, 1.00, 0.00, 0.0000, 0.00000,'I', 0.00, 20, 1.00000 + 111, 0,90.00, 5.00, 0.0000, 0.1780, 230.0,1.00000,0.51000,1.50000,0.51000,0.00625, 0, 0, 0,'1 ', 0.0000 + 104, 0,90.00, 5.00, 0.0000, 0.1720, 230.0,1.00000,0.51000,1.50000,0.51000,0.00625, 0, 0, 0,'1 ', 0.0000 +0 / END OF TWO-TERMINAL DC DATA, BEGIN VSC DC LINE DATA +0 / END OF VSC DC LINE DATA, BEGIN IMPEDANCE CORRECTION DATA + 1, 0.88,1.12000, 1.00,1.00000, 1.02,0.97900, 1.17,0.89500 + 2, 0.90,1.30000, 1.00,1.00000, 1.10,0.83000 + 3, -24.10,1.27000, -18.10,1.16600, -12.20,1.08200, -6.00,1.06100, 0.00,1.00000, 6.00,1.06100, 12.20,1.08200, 18.10,1.16600, 24.10,1.27000 + 4, 0.88,1.09300, 0.95,1.03000, 1.00,1.00000, 1.10,0.95000, 1.17,0.91600 + 5, -45.00,1.74300, -36.00,1.42800, -27.00,1.26700, -18.00,1.15700, -9.00,1.07400, 0.00,1.00000, 9.00,1.13800, 18.00,1.30500, 27.00,1.49800, 36.00,1.71900, 45.00,1.97200 + 6, -45.00,1.72500, -36.00,1.41800, -27.00,1.26000, -18.00,1.15300, -9.00,1.07300, 0.00,1.00000, 9.00,1.13500, 18.00,1.30000, 27.00,1.48900, 36.00,1.70500, 45.00,1.95200 + 7, -45.00,2.07300, 0.00,1.00000, 45.00,2.07300 + 8, -60.00,1.57180, -50.30,1.47940, -36.00,1.34310, -24.40,1.23250, -12.40,1.11820, 0.00,1.00000, 12.40,1.11820, 24.40,1.23250, 36.00,1.34310, 50.30,1.47940, 60.00,1.57180 + 9, -40.00,1.40000, -30.00,1.30000, -20.00,1.20000, -10.00,1.10000, 0.00,1.00000, 10.00,1.10000, 20.00,1.20000, 30.00,1.30000, 40.00,1.40000 +0 / END OF IMPEDANCE CORRECTION DATA, BEGIN MULTI-TERMINAL DC DATA +0 / END OF MULTI-TERMINAL DC DATA, BEGIN MULTI-SECTION LINE DATA + 109, 114,'&1',2, 401, 200 + 109, 114,'&2',2, 501, 201 + 109, 114,'&3',2, 601, 301 +0 / END OF MULTI-SECTION LINE DATA, BEGIN ZONE DATA +0 / END OF ZONE DATA, BEGIN INTER-AREA TRANSFER DATA +0 / END OF INTER-AREA TRANSFER DATA, BEGIN OWNER DATA +0 / END OF OWNER DATA, BEGIN FACTS DEVICE DATA +'FACTS 1', 108, 0,1, 50.000, 10.000,1.00000, 9999.000, 9999.000,0.90000,1.10000,1.00000, 0.000, 0.05000, 100.0, 1, 0.00000, 0.00000,0, 0," " +0 / END OF FACTS DEVICE DATA, BEGIN SWITCHED SHUNT DATA + 101,1,0,1,1.00000,1.00000, 0, 100.0,' ', 50.00, 5, 100.00 + 113,1,0,1,1.00000,1.00000, 0, 100.0,' ', 10.00, 2, 5.00 +0 / END OF SWITCHED SHUNT DATA, BEGIN GNE DATA +0 / END OF GNE DATA, BEGIN INDUCTION MACHINE DATA +0 / END OF INDUCTION MACHINE DATA +Q diff --git a/test/runtests.jl b/test/runtests.jl index e720095..dfd17ea 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,12 +1,28 @@ using Test -import Logging +using Logging +import PowerSystemCaseBuilder as PSB +import InfrastructureSystems as IS +import InfrastructureSystems: DataFormatError +import PowerSystems as PSY +import PowerSystems: + System, + get_components, + Generator, + get_ext + +using PowerFlowFileParser import Aqua -Aqua.test_unbound_args(SiennaTemplate) -Aqua.test_undefined_exports(SiennaTemplate) -Aqua.test_ambiguities(SiennaTemplate) -Aqua.test_stale_deps(SiennaTemplate) -Aqua.test_deps_compat(SiennaTemplate) +Aqua.test_unbound_args(PowerFlowFileParser) +Aqua.test_undefined_exports(PowerFlowFileParser) +Aqua.test_ambiguities(PowerFlowFileParser) +Aqua.test_stale_deps(PowerFlowFileParser) +Aqua.test_deps_compat(PowerFlowFileParser) + +const DATA_DIR = PSB.DATA_DIR +const MATPOWER_DIR = joinpath(DATA_DIR, "matpower") +const PSSE_RAW_DIR = joinpath(DATA_DIR, "psse_raw") +const BAD_DATA = joinpath(DATA_DIR, "bad_data_for_tests") LOG_FILE = "power-systems.log" LOG_LEVELS = Dict( diff --git a/test/test_file.jl b/test/test_file.jl deleted file mode 100644 index ab9151b..0000000 --- a/test/test_file.jl +++ /dev/null @@ -1,2 +0,0 @@ -# Put your tests here -@test 1 == 1 diff --git a/test/test_parse_matpower.jl b/test/test_parse_matpower.jl new file mode 100644 index 0000000..7fe61cd --- /dev/null +++ b/test/test_parse_matpower.jl @@ -0,0 +1,109 @@ +# TODO: Reviewers: Is this a correct list of keys to verify? +POWER_MODELS_KEYS = [ + "baseMVA", + "branch", + "bus", + "dcline", + "gen", + "load", + "name", + "per_unit", + "shunt", + "source_type", + "source_version", + "storage", +] + +badfiles = Dict("case30.m" => PSY.InvalidValue) +voltage_inconsistent_files = ["RTS_GMLC_original.m", "case5_re.m", "case5_re_uc.m"] +error_log_files = ["ACTIVSg2000.m", "case_ACTIVSg10k.m"] + +@testset "Parse Matpower data files" begin + files = [x for x in readdir(joinpath(MATPOWER_DIR)) if splitext(x)[2] == ".m"] + if length(files) == 0 + @error "No test files in the folder" + end + + for f in files + @info "Parsing $f..." + path = joinpath(MATPOWER_DIR, f) + + if f in voltage_inconsistent_files + continue + else + pm_dict = parse_file(path) + end + + for key in POWER_MODELS_KEYS + @test haskey(pm_dict, key) + end + @info "Successfully parsed $path to PowerModels dict" + + if f in keys(badfiles) + @test_logs( + (:error, r"cannot create Line"), + match_mode = :any, + @test_throws(badfiles[f], PowerFlowFileParser.System(PowerModelsData(pm_dict))) + ) + else + sys = PowerFlowFileParser.System(PowerModelsData(pm_dict)) + @info "Successfully parsed $path to System struct" + end + end +end + +@testset "Parse PowerModel Matpower data files" begin + files = [ + x for x in readdir(MATPOWER_DIR) if + splitext(x)[2] == ".m" + ] + if length(files) == 0 + @error "No test files in the folder" + end + + for f in files + @info "Parsing $f..." + path = joinpath(MATPOWER_DIR, f) + + if f in voltage_inconsistent_files + continue + else + pm_dict = parse_file(path) + end + + for key in POWER_MODELS_KEYS + @test haskey(pm_dict, key) + end + @info "Successfully parsed $path to PowerModels dict" + + if f in keys(badfiles) + @test_logs( + (:error, r"cannot create Line"), + match_mode = :any, + @test_throws(badfiles[f], PowerFlowFileParser.System(PowerModelsData(pm_dict))) + ) + else + sys = PowerFlowFileParser.System(PowerModelsData(pm_dict)) + @info "Successfully parsed $path to System struct" + end + end +end + +@testset "Parse Matpower files with voltage inconsistencies" begin + test_parse = (path) -> begin + pm_dict = parse_file(path) + + for key in POWER_MODELS_KEYS + @test haskey(pm_dict, key) + end + @info "Successfully parsed $path to PowerModels dict" + sys = PowerFlowFileParser.System(PowerModelsData(pm_dict)) + @info "Successfully parsed $path to System struct" + @test isa(sys, PowerFlowFileParser.System) + end + for f in voltage_inconsistent_files + @info "Parsing $f..." + path = joinpath(BAD_DATA, f) + @test_logs (:error,) match_mode = :any test_parse(path) + end +end diff --git a/test/test_parse_psse.jl b/test/test_parse_psse.jl new file mode 100644 index 0000000..887e215 --- /dev/null +++ b/test/test_parse_psse.jl @@ -0,0 +1,23 @@ +@testset "PSSE Parsing" begin + files = readdir(PSSE_RAW_DIR) + if length(files) == 0 + error("No test files in the folder") + end + + for f in files[1:1] + @info "Parsing $f ..." + pm_data = PowerModelsData(joinpath(PSSE_RAW_DIR, f)) + @info "Successfully parsed $f to PowerModelsData" + sys = PowerFlowFileParser.System(pm_data) + for g in get_components(Generator, sys) + @test haskey(get_ext(g), "r") + @test haskey(get_ext(g), "x") + end + @info "Successfully parsed $f to System struct" + end + + # Test bad input + pm_data = PowerModelsData(joinpath(PSSE_RAW_DIR, files[1])) + pm_data.data["bus"] = Dict{String, Any}() + @test_throws DataFormatError PowerFlowFileParser.System(pm_data) +end