LSM trees initial integration

.github/workflows/ci.yml

@@ -55,6 +55,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Check CDDLs up-to-date with Blueprints
+      continue-on-error: true
+      run: |
+        ./scripts/ci/diff-cddls.sh
+
     - name: Install base libraries
       uses: input-output-hk/actions/base@latest
       with:
@@ -138,6 +143,17 @@ jobs:
     - name: Build projects [build]
       run: cabal build all -j
 
+    - name: Install Ruby for CDDL
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4'
+        bundler-cache: true
+
+    - name: Install cddl tools
+      run: |
+        gem install cddlc
+        cabal install cuddle-0.5.0.0
+
     - name: Test
       if: matrix.test-set == 'all'
       run: cabal test all -j --test-show-details=streaming

cabal.project

@@ -14,7 +14,7 @@ repository cardano-haskell-packages
 -- update either of these.
 index-state:
   -- Bump this if you need newer packages from Hackage
-  , hackage.haskell.org 2025-06-03T21:29:34Z
+  , hackage.haskell.org 2025-06-17T07:53:04Z
   -- Bump this if you need newer packages from CHaP
   , cardano-haskell-packages 2025-06-03T13:42:38Z
 
@@ -25,6 +25,7 @@ packages:
   ouroboros-consensus-diffusion
   sop-extras
   strict-sop-core
+  ../lsm-tree
 
 -- We want to always build the test-suites and benchmarks
 tests: true
@@ -57,13 +58,8 @@ if impl (ghc >= 9.12)
     -- https://github.com/kapralVV/Unique/issues/11
     , Unique:hashable
 
-source-repository-package
-   type: git
-   location: https://github.com/IntersectMBO/ouroboros-network
-   tag: 3e8d3b4b8c87ead794876c62d7fe25f32efb5142
-   --sha256: 08fpkx3iagj83nn413h9a865zjcj3lrf7017a756qd2wg2jg3amq
-   subdir:
-     ouroboros-network-api
+    -- https://github.com/phadej/regression-simple/pull/14
+    , regression-simple:base
 
 source-repository-package
    type: git
@@ -97,3 +93,33 @@ source-repository-package
       libs/set-algebra
       libs/small-steps
       libs/vector-map
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-network
+  tag: dd45510b6c5a935e0f330d7b180800a27ebe2fd8
+  --sha256: sha256-Xk+DbtqQwadtmRoZNVzDUWM8tnV+RECXYRe74IOkL0s=
+  subdir:
+          network-mux
+          ouroboros-network
+          ouroboros-network-api
+          ouroboros-network-framework
+          ouroboros-network-mock
+          ouroboros-network-protocols
+          ouroboros-network-testing
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/fs-sim
+  tag: ee0b75bee5bcd426cfc5433b6c69c67fe6319c1b
+  --sha256: 0ss4n302khl13fj5f4l6cxfj5vn558s2wk533ikmxhgigf9qas0q
+  subdir: fs-api
+          fs-sim
+
+source-repository-package
+  type: git
+  location: https://github.com/jasagredo/lsm-tree
+  tag: fa0d862b7a609473538a0007bd0df314cd2fa635
+  subdir:
+
+    blockio

nix/cddlc/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'cddlc'

nix/cddlc/Gemfile.lock

@@ -0,0 +1,19 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cddlc (0.4.2)
+      neatjson (~> 0.10)
+      treetop (~> 1)
+    neatjson (0.10.5)
+    polyglot (0.3.5)
+    treetop (1.6.14)
+      polyglot (~> 0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cddlc
+
+BUNDLED WITH
+   2.6.2

nix/cddlc/gemset.nix

@@ -0,0 +1,47 @@
+{
+  cddlc = {
+    dependencies = [
+      "neatjson"
+      "treetop"
+    ];
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1s3fbgd5yqgji162zsmlwnva1v1r3zc1qiyv6im7karv5f08r8m3";
+      type = "gem";
+    };
+    version = "0.4.2";
+  };
+  neatjson = {
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "0wm1lq8yl6rzysh3wg6fa55w5534k6ppiz0qb7jyvdy582mk5i0s";
+      type = "gem";
+    };
+    version = "0.10.5";
+  };
+  polyglot = {
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1bqnxwyip623d8pr29rg6m8r0hdg08fpr2yb74f46rn1wgsnxmjr";
+      type = "gem";
+    };
+    version = "0.3.5";
+  };
+  treetop = {
+    dependencies = [ "polyglot" ];
+    groups = [ "default" ];
+    platforms = [ ];
+    source = {
+      remotes = [ "https://rubygems.org" ];
+      sha256 = "1m5fqy7vq6y7bgxmw7jmk7y6pla83m16p7lb41lbqgg53j8x2cds";
+      type = "gem";
+    };
+    version = "1.6.14";
+  };
+}

nix/cddlc/package.nix

@@ -0,0 +1,23 @@
+{ lib
+, bundlerApp
+, bundlerUpdateScript
+}:
+
+bundlerApp {
+  pname = "cddlc";
+
+  gemdir = ./.;
+
+  exes = [ "cddlc" ];
+
+  passthru.updateScript = bundlerUpdateScript "cddlc";
+
+  meta = {
+    description = "CDDL conversion utilities";
+    homepage = "https://github.com/cabo/cddlc";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ amesgen ];
+    platforms = lib.platforms.unix;
+    mainProgram = "cddlc";
+  };
+}

nix/haskell.nix

@@ -38,6 +38,19 @@ let
       ({ pkgs, lib, ... }: lib.mkIf pkgs.stdenv.hostPlatform.isWindows {
         # https://github.com/input-output-hk/haskell.nix/issues/2332
         packages.basement.configureFlags = [ "--hsc2hs-option=--cflag=-Wno-int-conversion" ];
+        # We can't cross-compile the ruby gem `cddlc` so we decided to skip this
+        # test on Windows in Hydra.
+        packages.ouroboros-consensus-cardano.components.tests.cardano-test.preCheck = ''
+          export DISABLE_CDDLC=1
+        '';
+      })
+      ({ pkgs, ... }: lib.mkIf (!pkgs.stdenv.hostPlatform.isWindows) {
+        # Tools for CBOR/CDDL tests:
+        packages.ouroboros-consensus-cardano.components.tests.cardano-test = {
+          build-tools =
+            [ pkgs.cddlc pkgs.cuddle ];
+          extraSrcFiles = [ "cddl/**/*" ];
+        };
       })
     ];
     flake.variants = {

nix/shell.nix

@@ -16,6 +16,8 @@ hsPkgs.shellFor {
     pkgs.ghcid
     pkgs.xrefcheck
     pkgs.fourmolu
+    pkgs.cuddle
+    pkgs.cddlc
 
     # release management
     pkgs.scriv

nix/tools.nix

@@ -38,6 +38,18 @@ in
 
   fourmolu = tool "fourmolu" "0.18.0.0" { };
 
+  cuddle = tool "cuddle" "git" {
+    src = final.fetchFromGitHub {
+      owner = "input-output-hk";
+      repo = "cuddle";
+      rev = "cuddle-0.5.0.0";
+      hash = "sha256-06a9N1IAh0kKW/xPu1qiLK22HpXyARnipA1YJxY4jOQ=";
+    };
+  };
+
+  # remove once our nixpkgs contains https://github.com/NixOS/nixpkgs/pull/394873
+  cddlc = final.callPackage ./cddlc/package.nix { };
+
   haskellBuildUtils = prev.haskellBuildUtils.override {
     inherit (final.hsPkgs.args) compiler-nix-name;
     index-state = tool-index-state;

ouroboros-consensus-cardano/cddl/base.cddl

@@ -0,0 +1,53 @@
+telescope7<byron, shelley, allegra, mary, alonzo, babbage, conway>
+  = [pastEra, pastEra, pastEra, pastEra, pastEra, pastEra, currentEra<conway>] /
+    [pastEra, pastEra, pastEra, pastEra, pastEra, currentEra<babbage>] /
+    [pastEra, pastEra, pastEra, pastEra, currentEra<alonzo>] /
+    [pastEra, pastEra, pastEra, currentEra<mary>] /
+    [pastEra, pastEra, currentEra<allegra>] /
+    [pastEra, currentEra<shelley>] /
+    [currentEra<byron>]
+
+ns7<byron, shelley, allegra, mary, alonzo, babbage, conway>
+  = [6, conway] /
+    [5, babbage] /
+    [4, alonzo] /
+    [3, mary] /
+    [2, allegra] /
+    [1, shelley] /
+    [0, byron]
+
+;; Blockchain types
+pastEra = [bound, bound]
+currentEra<st> = [bound, st]
+bound = [relativeTime, slotno, epochno]
+eraIdx = word8
+individualPoolStake = [stake, hash]
+nonce = [0] / [1, hash]
+point = [] / [ slotno, hash ]
+poolDistr = map<keyhash, individualPoolStake>
+slotno = word64
+stake = rational
+
+withOrigin<v> = [] / [v]
+withOriginTH<v> = [0] / [1, v]
+
+;; Collections
+either<x, y> = [0, x] / [1, y]
+map<x, y> = { * x => y }
+maybe<x> = [] / [x]
+seq<x> = [*23 x] / [24* x] ; encoded with indefinite-length encoding
+set<x> = #6.258([* x])
+
+;; Types from other packages
+blockno = word64
+epochno = word64
+coin = word64
+rational = [int, int]
+keyhash = bstr .size 28
+hash = bstr .size 32
+relativeTime = int
+
+;; Base word types
+word8  = uint .size 1
+word32 = uint .size 4
+word64 = uint .size 8

ouroboros-consensus-cardano/cddl/disk/immutable/chunkFile.cddl

@@ -0,0 +1,3 @@
+chunkFile = bytes .cborseq block.cardanoBlock
+
+;# import block as block

ouroboros-consensus-cardano/cddl/disk/immutable/primary.ksy

@@ -0,0 +1,12 @@
+meta:
+  id: primary_file
+  endian: be
+
+seq:
+  - id: current_version
+    type: u1
+    valid: 0x01
+
+  - id: secondary_offsets
+    type: u4
+    repeat: eos

ouroboros-consensus-cardano/cddl/disk/immutable/secondary.ksy

@@ -0,0 +1,29 @@
+meta:
+  id: secondary_file
+  endian: be
+
+types:
+  entry:
+    seq:
+      - id: block_offset
+        type: u8
+
+      - id: header_offset
+        type: u2
+
+      - id: header_size
+        type: u2
+
+      - id: crc
+        type: u4
+
+      - id: header_hash
+        size: 32
+
+      - id: block_or_ebb
+        type: u8
+
+seq:
+  - id: secondary_offsets
+    type: entry
+    repeat: eos

ouroboros-consensus-cardano/cddl/disk/ledger/headerstate.cddl

@@ -0,0 +1,28 @@
+headerState =
+  [withOrigin<headerStateTip>, headerStateChainDep]
+
+headerStateTip =
+  ns7<byronAnnTip, annTip, annTip, annTip, annTip, annTip, annTip>
+
+byronAnnTip = [slotno,
+               hash,
+               blockno,
+               bool]
+
+annTip = [slotno,
+          hash,
+          blockno]
+
+headerStateChainDep =
+  telescope7<versionedPbftState,
+             versionedTPraosState,
+             versionedTPraosState,
+             versionedTPraosState,
+             versionedTPraosState,
+             versionedPraosState,
+             versionedPraosState>
+
+;# import withOrigin, ns7, telescope7, slotno, hash, blockno from base
+;# import versionedPbftState from pbft
+;# import versionedPraosState from praos
+;# import versionedTPraosState from tpraos