Import tool fixes

docs/counters.md

@@ -281,7 +281,7 @@ Column referred to in TMCF is missing from CSV header.
 Found an unknown statType value.
 
 StatTypes values either:
-- end with one of {`value`, `estimate`, `stderror`, `samplesize`, `growthrate`}, or
+- end with one of {`value`, `estimate`, `stderror`, `samplesize`, `growthrate`, `ratio`}, or
 - start with `percentile`, or
 - equal any one of {`marginoferror`, `measurementResult`}.
 

pipeline/differ/src/test/resources/current/schema.mcf

@@ -1,68 +1,68 @@
-dcid: "InterestRate_TreasuryBill_6Month"
+Node: dcid:InterestRate_TreasuryBill_6Month
 name: "InterestRate_TreasuryBill_6Month"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBill
 maturity: [6 Month]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBill_1Year"
+Node: dcid:InterestRate_TreasuryBill_1Year
 name: "InterestRate_TreasuryBill_1Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBill
 maturity: [1 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_2Year"
+Node: dcid:InterestRate_TreasuryNote_2Year
 name: "InterestRate_TreasuryNote_2Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [2 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_3Year"
+Node: dcid:InterestRate_TreasuryNote_3Year
 name: "InterestRate_TreasuryNote_3Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [3 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_5Year"
+Node: dcid:InterestRate_TreasuryNote_5Year
 name: "InterestRate_TreasuryNote_5Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [5 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_7Year"
+Node: dcid:InterestRate_TreasuryNote_7Year
 name: "InterestRate_TreasuryNote_7Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [7 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_10Year"
+Node: dcid:InterestRate_TreasuryNote_10Year
 name: "InterestRate_TreasuryNote_10Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [10 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBond_20Year"
+Node: dcid:InterestRate_TreasuryBond_20Year
 name: "InterestRate_TreasuryBond_20Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBond
 maturity: [20 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBond_30Year"
+Node: dcid:InterestRate_TreasuryBond_30Year
 name: "InterestRate_TreasuryBond_30Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate

pipeline/differ/src/test/resources/obs-diff.csv

@@ -1,4 +1,4 @@
-dcid:InterestRate_TreasuryBill_3Month;dcid:country/USA;"2025-01-31";;dcid:ConstantMaturityRate;dcid:Percent;,,4.31,DELETED
-dcid:InterestRate_TreasuryBond_20Year;dcid:country/USA;"2025-01-30";;dcid:ConstantMaturityRate;dcid:Percent;,4.85,4.81,MODIFIED
-dcid:InterestRate_TreasuryBill_3Month;dcid:country/USA;"2025-01-30";;dcid:ConstantMaturityRate;dcid:Percent;,,4.30,DELETED
-dcid:InterestRate_TreasuryNote_10Year;dcid:country/USA;"2025-01-31";;dcid:ConstantMaturityRate;dcid:Percent;,4.58,,ADDED
+InterestRate_TreasuryBill_3Month;country/USA;2025-01-31;;ConstantMaturityRate;Percent;,,4.31,DELETED
+InterestRate_TreasuryBond_20Year;country/USA;2025-01-30;;ConstantMaturityRate;Percent;,4.85,4.81,MODIFIED
+InterestRate_TreasuryBill_3Month;country/USA;2025-01-30;;ConstantMaturityRate;Percent;,,4.30,DELETED
+InterestRate_TreasuryNote_10Year;country/USA;2025-01-31;;ConstantMaturityRate;Percent;,4.58,,ADDED

pipeline/differ/src/test/resources/previous/schema.mcf

@@ -1,60 +1,60 @@
-dcid: "InterestRate_TreasuryBill_1Month"
+Node: dcid:InterestRate_TreasuryBill_1Month
 name: "InterestRate_TreasuryBill_1Month"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBill
 maturity: [1 Month]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBill_3Month"
+Node: dcid:InterestRate_TreasuryBill_3Month
 name: "InterestRate_TreasuryBill_3Month"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBill
 maturity: [3 Month]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBill_6Month"
+Node: dcid:InterestRate_TreasuryBill_6Month
 name: "InterestRate_TreasuryBill_6Month"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBill
 maturity: [6 Month]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBill_1Year"
+Node: dcid:InterestRate_TreasuryBill_1Year
 name: "InterestRate_TreasuryBill_01Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBill
 maturity: [1 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_2Year"
+Node: dcid:InterestRate_TreasuryNote_2Year
 name: "InterestRate_TreasuryNote_02Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [2 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryNote_10Year"
+Node: dcid:InterestRate_TreasuryNote_10Year
 name: "InterestRate_TreasuryNote_10Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryNote
 maturity: [10 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBond_20Year"
+Node: dcid:InterestRate_TreasuryBond_20Year
 name: "InterestRate_TreasuryBond_20Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate
 populationType: dcs:TreasuryBond
 maturity: [20 Year]
 statType: dcs:measuredValue
 
-dcid: "InterestRate_TreasuryBond_30Year"
+Node: dcid:InterestRate_TreasuryBond_30Year
 name: "InterestRate_TreasuryBond_30Year"
 typeOf: dcs:StatisticalVariable
 measuredProperty: dcs:interestRate

pipeline/differ/src/test/resources/schema-diff.csv

@@ -1,7 +1,7 @@
-"InterestRate_TreasuryBill_1Year",maturity:[1 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_1Year";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,maturity:[1 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_01Year";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,MODIFIED
-"InterestRate_TreasuryBill_3Month",,maturity:[3 Month];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_3Month";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,DELETED
-"InterestRate_TreasuryNote_3Year",maturity:[3 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_3Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
-"InterestRate_TreasuryNote_7Year",maturity:[7 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_7Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
-"InterestRate_TreasuryNote_5Year",maturity:[5 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_5Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
-"InterestRate_TreasuryBill_1Month",,maturity:[1 Month];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_1Month";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,DELETED
-"InterestRate_TreasuryNote_2Year",maturity:[2 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_2Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,maturity:[2 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_02Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,MODIFIED
+dcid:InterestRate_TreasuryBill_1Year,maturity:[1 Year];measuredProperty:interestRate;name:InterestRate_TreasuryBill_1Year;populationType:TreasuryBill;statType:measuredValue;typeOf:StatisticalVariable,maturity:[1 Year];measuredProperty:interestRate;name:InterestRate_TreasuryBill_01Year;populationType:TreasuryBill;statType:measuredValue;typeOf:StatisticalVariable,MODIFIED
+dcid:InterestRate_TreasuryBill_3Month,,maturity:[3 Month];measuredProperty:interestRate;name:InterestRate_TreasuryBill_3Month;populationType:TreasuryBill;statType:measuredValue;typeOf:StatisticalVariable,DELETED
+dcid:InterestRate_TreasuryNote_3Year,maturity:[3 Year];measuredProperty:interestRate;name:InterestRate_TreasuryNote_3Year;populationType:TreasuryNote;statType:measuredValue;typeOf:StatisticalVariable,,ADDED
+dcid:InterestRate_TreasuryNote_7Year,maturity:[7 Year];measuredProperty:interestRate;name:InterestRate_TreasuryNote_7Year;populationType:TreasuryNote;statType:measuredValue;typeOf:StatisticalVariable,,ADDED
+dcid:InterestRate_TreasuryNote_5Year,maturity:[5 Year];measuredProperty:interestRate;name:InterestRate_TreasuryNote_5Year;populationType:TreasuryNote;statType:measuredValue;typeOf:StatisticalVariable,,ADDED
+dcid:InterestRate_TreasuryBill_1Month,,maturity:[1 Month];measuredProperty:interestRate;name:InterestRate_TreasuryBill_1Month;populationType:TreasuryBill;statType:measuredValue;typeOf:StatisticalVariable,DELETED
+dcid:InterestRate_TreasuryNote_2Year,maturity:[2 Year];measuredProperty:interestRate;name:InterestRate_TreasuryNote_2Year;populationType:TreasuryNote;statType:measuredValue;typeOf:StatisticalVariable,maturity:[2 Year];measuredProperty:interestRate;name:InterestRate_TreasuryNote_02Year;populationType:TreasuryNote;statType:measuredValue;typeOf:StatisticalVariable,MODIFIED

util/src/main/java/org/datacommons/util/FileGroup.java

@@ -66,7 +66,7 @@ public static FileGroup build(
     int nTsv = 0;
     for (File file : files) {
       String lowerPath = file.getPath().toLowerCase();
-      if (lowerPath.endsWith(".mcf")) {
+      if (lowerPath.contains(".mcf")) {
         mcfFiles.add(file);
       } else if (lowerPath.endsWith(".tmcf")) {
         tmcfFiles.add(file);

util/src/main/java/org/datacommons/util/GraphUtils.java

@@ -3,6 +3,7 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -301,28 +302,7 @@ public static McfStatVarObsSeries convertMcfGraphToMcfStatVarObsSeries(
    * @return An McfGraph proto representing the input MCF string.
    */
   public static McfGraph convertToGraph(String input) {
-    McfGraph.Builder g = McfGraph.newBuilder();
-    g.setType(McfType.INSTANCE_MCF);
-    String node_id = "";
-    McfGraph.PropertyValues.Builder base_node = McfGraph.PropertyValues.newBuilder();
-    String[] lines = input.split("\n");
-    for (String line : lines) {
-      line = line.trim();
-      if (line.isEmpty() || line.startsWith("//") || line.startsWith("#")) {
-        continue;
-      }
-      int colon = line.indexOf(":");
-      String lhs = line.substring(0, colon).trim();
-      String rhs = line.substring(colon + 1).trim();
-      if (lhs.equals(Property.dcid.name()) || lhs.equals("Node")) {
-        node_id = rhs;
-      }
-      setPropVal(lhs, ValueType.TEXT, rhs, base_node);
-    }
-    if (!node_id.isEmpty()) {
-      g.putNodes(node_id, base_node.build());
-    }
-    return g.build();
+    return McfParser.parseInstanceMcfString(input, true, null);
   }
 
   /**
@@ -334,8 +314,25 @@ public static McfGraph convertToGraph(String input) {
    */
   public static List<McfGraph> readMcfFile(String fileName) throws IOException {
     List<McfGraph> graphList = new ArrayList<>();
-    File file = new File(fileName);
-    Scanner scanner = new Scanner(file);
+    try (Scanner scanner = new Scanner(new File(fileName), StandardCharsets.UTF_8)) {
+      scanner.useDelimiter("\n\n");
+      while (scanner.hasNext()) {
+        graphList.add(convertToGraph(scanner.next()));
+      }
+    }
+    return graphList;
+  }
+
+  /**
+   * Converts an MCF file text into a list of McfGraph protos.
+   *
+   * @param mcfString Contents of MCF file.
+   * @return A list of McfGraph protos, where each proto represents a node or block from the MCF
+   *     file.
+   */
+  public static List<McfGraph> readMcfString(String mcfString) {
+    List<McfGraph> graphList = new ArrayList<>();
+    Scanner scanner = new Scanner(mcfString);
     scanner.useDelimiter("\n\n");
     while (scanner.hasNext()) {
       String token = scanner.next();