[gephi-lite] Improves field types inference

Details:
- Adds possibility in parseFile to specify known field types
- Specifies field types for nodes labels and z values and edge label in
  parseFile's GEXF section
- Separates heuristics from category and keyword fields, to improve
  keyword fields detection

Author: jacomyal

packages/gephi-lite/src/core/file/index.ts

@@ -65,7 +65,7 @@ export const open = asyncAction(async (file: FileTypeWithoutFormat) => {
 
   try {
     // Parse the file
-    const { data, format } = await parseFile(file);
+    const { data, metadata, format } = await parseFile(file);
 
     // Do the import
     resetStates(false);
@@ -76,7 +76,7 @@ export const open = asyncAction(async (file: FileTypeWithoutFormat) => {
       const { mergeState } = appearanceActions;
       data.setAttribute("title", file.filename);
 
-      const graphDataset = initializeGraphDataset(data);
+      const graphDataset = initializeGraphDataset(data, metadata);
       setGraphDataset(graphDataset);
 
       const appearanceState = inferAppearanceState(graphDataset);

packages/gephi-lite/src/core/file/utils.ts

@@ -1,4 +1,4 @@
-import { gephiLiteParse } from "@gephi/gephi-lite-sdk";
+import { FieldModel, gephiLiteParse } from "@gephi/gephi-lite-sdk";
 import Graph from "graphology";
 import gexf from "graphology-gexf/browser";
 import graphml from "graphology-graphml/browser";
@@ -51,10 +51,13 @@ async function getFileContent(file: FileTypeWithoutFormat): Promise<string> {
 /**
  * Parse the content of the given file and returns its data and its type.
  */
-export async function parseFile(
-  file: FileTypeWithoutFormat,
-): Promise<
-  { format: "gexf" | "graphml" | "graphology"; data: Graph } | { format: "gephi-lite"; data: GephiLiteFileFormat }
+export async function parseFile(file: FileTypeWithoutFormat): Promise<
+  | {
+      format: "gexf" | "graphml" | "graphology";
+      data: Graph;
+      metadata?: { nodeFields?: FieldModel<"nodes">[]; edgeFields?: FieldModel<"edges">[] };
+    }
+  | { format: "gephi-lite"; data: GephiLiteFileFormat; metadata?: undefined }
 > {
   const content = await getFileContent(file);
   const extension = (file.filename.split(".").pop() || "").toLowerCase();
@@ -65,6 +68,27 @@ export async function parseFile(
       return {
         format: "gexf",
         data: gexf.parse(Graph, content, { allowUndeclaredAttributes: true, addMissingNodes: true }),
+        metadata: {
+          nodeFields: [
+            {
+              id: "label",
+              itemType: "nodes",
+              type: "text",
+            },
+            {
+              id: "z",
+              itemType: "nodes",
+              type: "number",
+            },
+          ],
+          edgeFields: [
+            {
+              id: "label",
+              itemType: "edges",
+              type: "text",
+            },
+          ],
+        },
       };
     case "graphml":
       return {

packages/gephi-lite/src/core/graph/fieldModel.ts

@@ -14,7 +14,7 @@ import {
   toStringArray,
 } from "@gephi/gephi-lite-sdk";
 import guessFormat from "@gristlabs/moment-guess";
-import { isNumber, sortBy, take, toPairs, uniq } from "lodash";
+import { countBy, isNumber, mean, size, sortBy, take, toPairs, uniq } from "lodash";
 import { DateTime } from "luxon";
 
 import { isValidColor } from "../../utils/colors";
@@ -38,14 +38,15 @@ export function guessSeparator(values: string[]): string | null {
   values.forEach((value) =>
     SEPARATORS.forEach((sep) => {
       const split = value.split(sep);
-      if (split.length > 1 && split.every((s) => !!s && !s.match(/(^ | $)/))) separatorsFrequencies[sep]++;
+      if (split.length > 1 && split.every((s) => !!s.trim())) separatorsFrequencies[sep]++;
     }),
   );
 
   const bestSeparator = sortBy(
-    SEPARATORS.filter((sep) => !!separatorsFrequencies[sep]),
+    SEPARATORS.filter((sep) => separatorsFrequencies[sep] >= values.length / 10),
     (sep) => -separatorsFrequencies[sep],
   )[0];
+
   return bestSeparator || null;
 }
 
@@ -85,6 +86,7 @@ export function inferFieldType(fieldName: string, values: Scalar[], itemsCount:
     values.every((v) => {
       try {
         const _dateFormat = guessFormat("" + v, "");
+
         // format guesser can return multiple choices, we just pick one
         const dateFormat = Array.isArray(_dateFormat) ? _dateFormat[0] : _dateFormat;
         const correctedDateFormat = dateFormat.replaceAll("Y", "y").replaceAll("D", "d");
@@ -100,24 +102,27 @@ export function inferFieldType(fieldName: string, values: Scalar[], itemsCount:
     return { type: "date", format };
   }
 
-  // KEYWORDS and CATEGORY
+  // KEYWORDS
   const separator = guessSeparator(
     take(
       values.map((v) => "" + v),
       100,
     ),
   );
-  const uniqValues = uniq(separator ? values.flatMap((v) => (v + "").split(separator)) : values);
-  const uniqValuesCount = uniqValues.length;
+  if (separator) {
+    const splitValuesCounts = countBy(values.flatMap((v) => (v + "").split(separator)));
+    const uniqSplitValuesCount = size(splitValuesCounts);
+    const averageValuesCount = mean(Object.values(splitValuesCounts));
 
-  if (
-    uniqValuesCount > 1 &&
-    uniqValuesCount < 50 &&
-    uniqValuesCount < Math.max(separator ? itemsCount : Math.pow(itemsCount, 0.75), 5)
-  ) {
-    // category and keywords
-    if (separator) return { type: "keywords", separator };
-    else return { type: "category" };
+    if (averageValuesCount > 2 && uniqSplitValuesCount > 1 && uniqSplitValuesCount < itemsCount) {
+      return { type: "keywords", separator };
+    }
+  }
+
+  // CATEGORIES
+  const uniqValuesCount = uniq(values).length;
+  if (uniqValuesCount > 1 && uniqValuesCount < Math.max(Math.pow(itemsCount, 0.75), 5)) {
+    return { type: "category" };
   }
 
   // TEXT

packages/gephi-lite/src/core/graph/utils.ts

@@ -1,7 +1,7 @@
 import { NodeCoordinates, getEmptyGraphDataset, toNumber, toScalar } from "@gephi/gephi-lite-sdk";
 import Graph, { MultiGraph } from "graphology";
 import { Attributes } from "graphology-types";
-import { flatMap, forEach, isNil, isNumber, keys, mapValues, omit, sortBy, uniq, values } from "lodash";
+import { flatMap, forEach, isNil, isNumber, keyBy, keys, mapValues, omit, sortBy, uniq, values } from "lodash";
 
 import { ItemType, Scalar } from "../types";
 import { inferFieldType } from "./fieldModel";
@@ -21,7 +21,7 @@ import {
 export { datasetToString, getEmptyGraphDataset, parseDataset } from "@gephi/gephi-lite-sdk";
 
 export function getRandomNodeCoordinate(): number {
-  return Math.random() * 100;
+  return Math.random() * 1000;
 }
 
 export function cleanNode(_node: string, attributes: Attributes): { data: ItemData; position: NodeCoordinates } {
@@ -48,7 +48,10 @@ export function cleanEdge(_edge: string, attributes: Attributes): { data: ItemDa
  * This function takes any graphology instance (like returned by any graphology
  * importer basically), and returns a properly shaped graph dataset:
  */
-export function initializeGraphDataset(graph: Graph): GraphDataset {
+export function initializeGraphDataset(
+  graph: Graph,
+  { nodeFields, edgeFields }: { nodeFields?: FieldModel<"nodes">[]; edgeFields?: FieldModel<"edges">[] } = {},
+): GraphDataset {
   const dataset = getEmptyGraphDataset();
 
   // setting graph meta data
@@ -86,20 +89,26 @@ export function initializeGraphDataset(graph: Graph): GraphDataset {
   });
 
   // Infer model:
+  const nodeFieldsDict = keyBy(nodeFields, "id");
+  const edgeFieldsDict = keyBy(edgeFields, "id");
+
   forEach(nodeAttributeValues, (values, key) => {
-    dataset.nodeFields.push({
-      id: key,
-      itemType: "nodes",
-      ...inferFieldType(key, values, graph.order),
-    });
+    dataset.nodeFields.push(
+      nodeFieldsDict[key] || {
+        id: key,
+        itemType: "nodes",
+        ...inferFieldType(key, values, graph.order),
+      },
+    );
   });
-
   forEach(edgeAttributeValues, (values, key) => {
-    dataset.edgeFields.push({
-      id: key,
-      itemType: "edges",
-      ...inferFieldType(key, values, graph.size),
-    });
+    dataset.edgeFields.push(
+      edgeFieldsDict[key] || {
+        id: key,
+        itemType: "edges",
+        ...inferFieldType(key, values, graph.size),
+      },
+    );
   });
 
   const labelsOrder = ["label", "size", "weight", "color"];
@@ -253,23 +262,3 @@ export function uniqFieldValuesAsStrings(items: Record<string, ItemData>, field:
     }),
   ) as string[];
 }
-
-// /**
-//  * Generate the original graph from the graphDataset.
-//  */
-// export function graphDatasetToGraphData(graphDataset: GraphDataset): DataGraph {
-//   const graph = graphDataset.fullGraph.copy();
-//   Object.entries(graphDataset.nodeData).map(([key, value]) => {
-//     graph.updateNodeAttributes(key, (attrs) => ({ ...attrs, ...value }));
-//   });
-//   Object.entries(graphDataset.edgeData).map(([key, value]) => {
-//     graph.updateEdgeAttributes(key, (attrs) => ({ ...attrs, ...value }));
-//   });
-//   Object.entries(graphDataset.nodeRenderingData).map(([key, value]) => {
-//     graph.updateNodeAttributes(key, (attrs) => ({ ...attrs, ...value }));
-//   });
-//   Object.entries(graphDataset.edgeRenderingData).map(([key, value]) => {
-//     graph.updateEdgeAttributes(key, (attrs) => ({ ...attrs, ...value }));
-//   });
-//   return graph;
-// }