Skip to content

Commit 88909f9

Browse files
authored
Serialization: rename for clarity: we're dealing with kinds here (#348)
Prior to this we'd see rather confusing log entries, e.g. ``` SL_LOGGING_LEVEL=debug ./c2cpg.sh ~/tmp/cpgtesting/linux-kernel/linux-6.1.3/fs/ext4 [DEBUG] wrote 43 nodes with 58 edges and 167 properties Cpg[Graph[183630 nodes]] ```
1 parent b9e4798 commit 88909f9

File tree

2 files changed

+15
-13
lines changed

2 files changed

+15
-13
lines changed

core/src/main/scala/flatgraph/storage/Serialization.scala

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import scala.collection.mutable
1717
import java.util.concurrent
1818

1919
class WriterContext(val fileChannel: FileChannel, val executor: concurrent.ExecutorService) {
20-
2120
val compressCtx = new ZstdWrapper.ZstdCtx
2221
var fileOffset: Long = 16L
2322
val compressQueue = mutable.ArrayDeque[concurrent.Future[(OutlineStorage, ByteBuffer)]]()
@@ -220,7 +219,7 @@ class WriterContext(val fileChannel: FileChannel, val executor: concurrent.Execu
220219

221220
object Serialization {
222221
val logger = LoggerFactory.getLogger(getClass)
223-
case class Counts(nodes: Int, edges: Int, properties: Int)
222+
case class Counts(nodeKinds: Int, edgeKinds: Int, propertyKinds: Int)
224223

225224
def writeGraph(graph: Graph, storagePath: Path, requestedExecutor: Option[concurrent.ExecutorService] = None): Counts = {
226225
logger.info(s"writing to storage at `$storagePath`")
@@ -241,9 +240,9 @@ object Serialization {
241240
}
242241

243242
private def innerWriteGraph(graph: Graph, writer: WriterContext): Counts = {
244-
val nodes = mutable.ArrayBuffer.empty[NodeItem]
245-
val edges = mutable.ArrayBuffer.empty[EdgeItem]
246-
val properties = mutable.ArrayBuffer.empty[PropertyItem]
243+
val nodeKinds = mutable.ArrayBuffer.empty[NodeItem]
244+
val edgeKinds = mutable.ArrayBuffer.empty[EdgeItem]
245+
val propertyKinds = mutable.ArrayBuffer.empty[PropertyItem]
247246
for (nodeKind <- graph.schema.nodeKinds) {
248247
val nodeLabel = graph.schema.getNodeLabel(nodeKind)
249248
val deletions = graph
@@ -252,7 +251,7 @@ object Serialization {
252251
case deleted: GNode if AccessHelpers.isDeleted(deleted) => deleted.seq()
253252
}
254253
val size = graph.nodeCountByKind(nodeKind)
255-
nodes.addOne(new Manifest.NodeItem(nodeLabel, size, deletions))
254+
nodeKinds.addOne(new Manifest.NodeItem(nodeLabel, size, deletions))
256255
}
257256
for {
258257
nodeKind <- graph.schema.nodeKinds
@@ -264,7 +263,7 @@ object Serialization {
264263
val nodeLabel = graph.schema.getNodeLabel(nodeKind)
265264
val edgeLabel = graph.schema.getEdgeLabel(nodeKind, edgeKind)
266265
val edgeItem = new Manifest.EdgeItem(nodeLabel, edgeLabel, direction.encoding)
267-
edges.addOne(edgeItem)
266+
edgeKinds.addOne(edgeItem)
268267
writer.encodeAny(graph.neighbors(pos), edgeItem.qty, delta = graph.nodeCountByKind(nodeKind))
269268
writer.encodeAny(graph.neighbors(pos + 1), edgeItem.neighbors)
270269
writer.encodeAny(graph.neighbors(pos + 2), edgeItem.property)
@@ -279,16 +278,16 @@ object Serialization {
279278
val nodeLabel = graph.schema.getNodeLabel(nodeKind)
280279
val propertyLabel = graph.schema.getPropertyLabel(nodeKind, propertyKind)
281280
val propertyItem = new Manifest.PropertyItem(nodeLabel, propertyLabel)
282-
properties.addOne(propertyItem)
281+
propertyKinds.addOne(propertyItem)
283282
writer.encodeAny(graph.properties(pos).asInstanceOf[Array[Int]], propertyItem.qty, delta = graph.nodeCountByKind(nodeKind))
284283
writer.encodeAny(graph.properties(pos + 1), propertyItem.property)
285284
}
286285
}
287286

288-
val manifest = new GraphItem(nodes.toArray, edges.toArray, properties.toArray)
287+
val manifest = new GraphItem(nodeKinds.toArray, edgeKinds.toArray, propertyKinds.toArray)
289288
writer.finish(manifest)
290-
logger.debug(s"wrote ${nodes.size} nodes with ${edges.size} edges and ${properties.size} properties")
291-
Counts(nodes.size, edges.size, properties.size)
289+
logger.debug(s"wrote ${nodeKinds.size} node kinds, {edges.size} edge kinds and ${propertyKinds.size} property kinds")
290+
Counts(nodeKinds.size, edgeKinds.size, propertyKinds.size)
292291
}
293292

294293
private[flatgraph] def write(bytes: Array[Byte], res: OutlineStorage, filePtr: AtomicLong, fileChannel: FileChannel): OutlineStorage = {

core/src/test/scala/flatgraph/SerializationTests.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package flatgraph
22

33
import flatgraph.misc.DebugDump.debugDump
44
import flatgraph.storage.Deserialization.DeserializationException
5+
import flatgraph.storage.Serialization.Counts
56
import flatgraph.storage.{Deserialization, Serialization}
67
import org.scalatest.matchers.should.Matchers
78
import org.scalatest.wordspec.AnyWordSpec
@@ -38,7 +39,8 @@ class SerializationTests extends AnyWordSpec with Matchers {
3839
| V0_1 [0] <- V0_0
3940
|""".stripMargin
4041

41-
Serialization.writeGraph(graph, storagePath)
42+
val counts = Serialization.writeGraph(graph, storagePath)
43+
counts shouldBe Counts(nodeKinds = 1, edgeKinds = 2, propertyKinds = 0)
4244

4345
val deserialized = Deserialization.readGraph(storagePath, Option(graph.schema))
4446
val newDump = debugDump(deserialized)
@@ -57,7 +59,8 @@ class SerializationTests extends AnyWordSpec with Matchers {
5759
DiffGraphApplier.applyDiff(graph, diff)
5860

5961
val storagePath = Files.createTempFile(s"flatgraph-${getClass.getSimpleName}", "fg")
60-
Serialization.writeGraph(graph, storagePath)
62+
val counts = Serialization.writeGraph(graph, storagePath)
63+
counts shouldBe Counts(nodeKinds = 1, edgeKinds = 0, propertyKinds = 0)
6164
patchFile(storagePath)
6265

6366
// when the vulnerability was reported, the following line raised a:

0 commit comments

Comments
 (0)