Merge remote-tracking branch 'upstream/edge'

Author: dtfiedler

Makefile

@@ -8,7 +8,7 @@ WAMR_DIR = _build/wamr
 
 GENESIS_WASM_BRANCH = tillathehun0/cu-experimental
 GENESIS_WASM_REPO = https://github.com/permaweb/ao.git
-GENESIS_WASM_SERVER_DIR = _build/genesis-wasm-server
+GENESIS_WASM_SERVER_DIR = _build/genesis_wasm/genesis-wasm-server
 
 ifdef HB_DEBUG
 	WAMR_FLAGS = -DWAMR_ENABLE_LOG=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 -DCMAKE_BUILD_TYPE=Debug
@@ -98,7 +98,7 @@ setup-genesis-wasm: $(GENESIS_WASM_SERVER_DIR)
 		echo "Error: Node.js is not installed. Please install Node.js before continuing."; \
 		echo "For Ubuntu/Debian, you can install it with:"; \
 		echo "  curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - && \\"; \
-		echo "  apt-get install -y nodejs && \\"; \
+		echo "  apt-get install -y nodejs=22.16.0-1nodesource1 --allow-downgrades && \\"; \
 		echo "  node -v && npm -v"; \
 		exit 1; \
 	fi

rebar.config

@@ -24,6 +24,11 @@
         {erl_opts, [{d, 'ENABLE_GENESIS_WASM', true}]},
         {pre_hooks, [
             {compile, "make -C \"${REBAR_ROOT_DIR}\" setup-genesis-wasm"}
+        ]},
+        {relx, [
+            {overlay, [
+                {copy, "_build/genesis_wasm/genesis-wasm-server", "genesis-wasm-server"}
+            ]}
         ]}
     ]},
     {rocksdb, [

src/ar_bundles.erl

@@ -379,7 +379,7 @@ encode_tags([]) ->
 encode_tags(Tags) ->
     EncodedBlocks = lists:flatmap(
         fun({Name, Value}) ->
-            Res = [encode_avro_string(Name), encode_avro_string(Value)],
+            Res = [encode_avro_name(Name), encode_avro_value(Value)],
             case lists:member(error, Res) of
                 true ->
                     throw({cannot_encode_empty_string, Name, Value});
@@ -394,14 +394,22 @@ encode_tags(Tags) ->
     <<ZigZagCount/binary, (list_to_binary(EncodedBlocks))/binary, 0>>.
 
 %% @doc Encode a string for Avro using ZigZag and VInt encoding.
-encode_avro_string(<<>>) ->
-    % Zero length strings are treated as a special case, due to the Avro encoder.
+encode_avro_name(<<>>) ->
+    % Zero length names are treated as a special case, due to the Avro encoder.
     << 0 >>;
-encode_avro_string(String) ->
-    StringBytes = unicode:characters_to_binary(String, utf8),
+encode_avro_name(String) ->
+    StringBytes = utf8_encoded(String),
     Length = byte_size(StringBytes),
     <<(encode_zigzag(Length))/binary, StringBytes/binary>>.
 
+encode_avro_value(<<>>) ->
+    % Zero length values are treated as a special case, due to the Avro encoder.
+    << 0 >>;
+encode_avro_value(Value) when is_binary(Value) ->
+    % Tag values can be raw binaries
+    Length = byte_size(Value),
+    <<(encode_zigzag(Length))/binary, Value/binary>>.
+
 %% @doc Encode an integer using ZigZag encoding.
 encode_zigzag(Int) when Int >= 0 ->
     encode_vint(Int bsl 1);
@@ -635,9 +643,45 @@ ar_bundles_test_() ->
         {timeout, 30, fun test_basic_member_id/0},
         {timeout, 30, fun test_deep_member/0},
         {timeout, 30, fun test_extremely_large_bundle/0},
-        {timeout, 30, fun test_serialize_deserialize_deep_signed_bundle/0}
+        {timeout, 30, fun test_serialize_deserialize_deep_signed_bundle/0},
+        {timeout, 30, fun test_encode_tags/0}
     ].
 
+test_encode_tags() ->
+    BinValue = <<1, 2, 3, 255, 254>>,
+    TestCases = [
+        {simple_string_tags, [{<<"tag1">>, <<"value1">>}]},
+        {binary_value_tag, [{<<"binary-tag">>, BinValue}]},
+        {mixed_tags,
+            [
+                {<<"string-tag">>, <<"string-value">>},
+                {<<"binary-tag">>, BinValue}
+            ]
+        },
+        {empty_value_tag, [{<<"empty-value-tag">>, <<>>}]},
+        {unicode_tag, [{<<"unicode-tag">>, <<"你好世界">>}]}
+    ],
+    lists:foreach(
+        fun({Label, InputTags}) ->
+            Encoded = encode_tags(InputTags),
+            Wrapped =
+                <<
+                    (length(InputTags)):64/little,
+                    (byte_size(Encoded)):64/little,
+                    Encoded/binary
+                >>,
+            {DecodedTags, <<>>} = decode_tags(Wrapped),
+            ?assertEqual(InputTags, DecodedTags, Label)
+        end,
+        TestCases
+    ),
+    % Test case: Empty tags list
+    EmptyTags = [],
+    EncodedEmpty = encode_tags(EmptyTags),
+    ?assertEqual(<<>>, EncodedEmpty),
+    WrappedEmpty = <<0:64/little, 0:64/little>>,
+    {[], <<>>} = decode_tags(WrappedEmpty).
+
 run_test() ->
     test_with_zero_length_tag().
 
@@ -648,12 +692,9 @@ test_no_tags() ->
     Anchor = crypto:strong_rand_bytes(32),
     DataItem = new_item(Target, Anchor, [], <<"data">>),
     SignedDataItem = sign_item(DataItem, {Priv, Pub}),
-
     ?assertEqual(true, verify_item(SignedDataItem)),
     assert_data_item(KeyType, Owner, Target, Anchor, [], <<"data">>, SignedDataItem),
-
     SignedDataItem2 = deserialize(serialize(SignedDataItem)),
-
     ?assertEqual(SignedDataItem, SignedDataItem2),
     ?assertEqual(true, verify_item(SignedDataItem2)),
     assert_data_item(KeyType, Owner, Target, Anchor, [], <<"data">>, SignedDataItem2).
@@ -666,12 +707,9 @@ test_with_tags() ->
     Tags = [{<<"tag1">>, <<"value1">>}, {<<"tag2">>, <<"value2">>}],
     DataItem = new_item(Target, Anchor, Tags, <<"taggeddata">>),
     SignedDataItem = sign_item(DataItem, {Priv, Pub}),
-
     ?assertEqual(true, verify_item(SignedDataItem)),
     assert_data_item(KeyType, Owner, Target, Anchor, Tags, <<"taggeddata">>, SignedDataItem),
-
     SignedDataItem2 = deserialize(serialize(SignedDataItem)),
-
     ?assertEqual(SignedDataItem, SignedDataItem2),
     ?assertEqual(true, verify_item(SignedDataItem2)),
     assert_data_item(KeyType, Owner, Target, Anchor, Tags, <<"taggeddata">>, SignedDataItem2).

src/dev_genesis_wasm.erl

@@ -59,6 +59,11 @@ compute(Msg, Msg2, Opts) ->
 ensure_started(Opts) ->
     % Check if the `genesis-wasm@1.0' device is already running. The presence
     % of the registered name implies its availability.
+    {ok, Cwd} = file:get_cwd(),
+    ?event({ensure_started, cwd, Cwd}),
+    % Determine path based on whether we're in a release or development
+    GenesisWasmServerDir = filename:join([Cwd, "genesis-wasm-server"]),
+    ?event({ensure_started, genesis_wasm_server_dir, GenesisWasmServerDir}),
     ?event({ensure_started, genesis_wasm, self()}),
     IsRunning = is_genesis_wasm_server_running(Opts),
     IsCompiled = hb_features:genesis_wasm(),
@@ -77,22 +82,22 @@ ensure_started(Opts) ->
                         NodeURL =
                             "http://localhost:" ++
                             integer_to_list(hb_opts:get(port, no_port, Opts)),
-                        DBDir =
-                            filename:absname(
-                                hb_util:list(
-                                    hb_opts:get(
-                                        genesis_wasm_db_dir,
-                                        "cache-mainnet/genesis-wasm",
-                                        Opts
-                                    )
+                        RelativeDBDir =
+                            hb_util:list(
+                                hb_opts:get(
+                                    genesis_wasm_db_dir,
+                                    "cache-mainnet/genesis-wasm",
+                                    Opts
                                 )
                             ),
+                        DBDir =
+                            filename:absname(RelativeDBDir),
 						CheckpointDir =
                             filename:absname(
                                 hb_util:list(
                                     hb_opts:get(
                                         genesis_wasm_checkpoints_dir,
-                                        "cache-mainnet/genesis-wasm/checkpoints",
+                                        RelativeDBDir ++ "/checkpoints",
                                         Opts
                                     )
                                 )
@@ -103,16 +108,16 @@ ensure_started(Opts) ->
                         Port =
                             open_port(
                                 {spawn_executable,
-                                    "/opt/_build/genesis-wasm-server/launch-monitored.sh"
+                                    filename:join([GenesisWasmServerDir, "launch-monitored.sh"])
                                 },
                                 [
                                     binary, use_stdio, stderr_to_stdout,
                                     {args, [
                                         "npm",
                                         "--prefix",
-                                        "/opt/_build/genesis-wasm-server",
+                                        GenesisWasmServerDir,
                                         "run",
-                                        "dev"
+                                        "start"
                                     ]},
                                     {env,
                                         [
@@ -128,7 +133,7 @@ ensure_started(Opts) ->
                                                 )
                                             },
                                             {"DB_URL", DatabaseUrl},
-                                            {"NODE_CONFIG_ENV", "development"},
+                                            {"NODE_CONFIG_ENV", "production"},
                                             {"DEFAULT_LOG_LEVEL",
                                                 hb_util:list(
                                                     hb_opts:get(

src/dev_green_zone.erl

@@ -388,17 +388,14 @@ finalize_become(KeyResp, NodeLocation, NodeID, GreenZoneAES, Opts) ->
             priv_wallet => GreenZoneWallet
         }
     },
+    NewOpts = Opts#{
+        identities => UpdatedIdentities
+    },
     ok = 
         hb_http_server:set_opts(
-            Opts#{
-                identities => UpdatedIdentities
-            }
+            NewOpts
         ),
-    % Print the updated wallet address
-    Wallet = hb_opts:get(priv_wallet, undefined, Opts),
-    ?event(green_zone,
-        {become, wallet, hb_util:human_id(ar_wallet:to_address(Wallet))}
-    ),
+    try_mount_encrypted_volume(GreenZoneWallet, NewOpts),
     ?event(green_zone, {become, update_wallet, complete}),
     {ok, #{
         <<"body">> => #{
@@ -487,8 +484,6 @@ join_peer(PeerLocation, PeerID, _M1, _M2, InitOpts) ->
                                 priv_green_zone_aes => AESKey
                             },
                             hb_http_server:set_opts(NewOpts),
-                            ?event(successfully_joined_greenzone),
-                            try_mount_encrypted_volume(AESKey, NewOpts),
                             {ok, #{ 
                                 <<"body">> => 
                                     <<"Node joined green zone successfully.">>, 
@@ -614,13 +609,20 @@ validate_peer_opts(Req, Opts) ->
     RequiredConfig =
         hb_ao:normalize_keys(
             hb_opts:get(green_zone_required_opts, #{}, Opts)),
-    ?event(green_zone, {validate_peer_opts, required_config, RequiredConfig}),
+    ConvertedRequiredConfig = 
+        hb_message:uncommitted(
+            hb_cache:ensure_all_loaded(
+                hb_message:commit(RequiredConfig, Opts),
+                Opts
+            )
+        ),
+    ?event(green_zone, {validate_peer_opts, required_config, ConvertedRequiredConfig}),
     PeerOpts =
         hb_ao:normalize_keys(
             hb_ao:get(<<"node-message">>, Req, undefined, Opts)),
     % Validate each item in node_history has required options
     Result = try
-        case hb_opts:ensure_node_history(PeerOpts, RequiredConfig) of
+        case hb_opts:ensure_node_history(PeerOpts, ConvertedRequiredConfig) of
             {ok, _} -> 
                 ?event(green_zone, {validate_peer_opts, history_items_check, valid}),
                 true;
@@ -730,23 +732,23 @@ decrypt_zone_key(EncZoneKey, Opts) ->
 %% The encryption key used for the volume is the same AES key used for green zone
 %% communication, ensuring that only nodes in the green zone can access the data.
 %%
-%% @param AESKey The AES key obtained from joining the green zone.
+%% @param Key The password for the encrypted volume.
 %% @param Opts A map of configuration options.
 %% @returns ok (implicit) in all cases, with detailed event logs of the results.
-try_mount_encrypted_volume(AESKey, Opts) ->
-    ?event(green_zone, {try_mount_encrypted_volume, start}),
+try_mount_encrypted_volume(Key, Opts) ->
+    ?event(debug_volume, {try_mount_encrypted_volume, start}),
     % Set up options for volume mounting with default paths
     VolumeOpts = Opts#{
-        priv_volume_key => AESKey,
+        priv_volume_key => Key,
         volume_skip_decryption => <<"true">>
     },
     % Call the dev_volume:mount function to handle the complete process
     case dev_volume:mount(undefined, undefined, VolumeOpts) of
         {ok, Result} ->
-            ?event(green_zone, {volume_mount, success, Result}),
+            ?event(debug_volume, {volume_mount, success, Result}),
             ok;
         {error, Error} ->
-            ?event(green_zone, {volume_mount, error, Error}),
+            ?event(debug_volume, {volume_mount, error, Error}),
             ok % Still return ok as this is an optional operation
     end.
 
@@ -780,4 +782,4 @@ rsa_wallet_integration_test() ->
     % Verify roundtrip
     ?assertEqual(PlainText, Decrypted),
     % Verify wallet structure
-    ?assertEqual(KeyType, {rsa, 65537}).
+    ?assertEqual(KeyType, {rsa, 65537}).

src/dev_lua_test_ledgers.erl

@@ -591,6 +591,46 @@ subledger_registration_test_disabled() ->
     % Alice can send tokens to Bob on SubLedger2.
     verify_net(RootLedger, [SubLedger1, SubLedger2], Opts).
 
+single_subledger_to_subledger_test_() -> {timeout, 30, fun single_subledger_to_subledger/0}.
+single_subledger_to_subledger() ->
+    Opts = test_opts(),
+    Alice = ar_wallet:new(),
+    Bob = ar_wallet:new(),
+    RootLedger =
+        ledger(
+            <<"scripts/hyper-token.lua">>,
+            #{ <<"balance">> => #{ Alice => 100 } },
+            Opts
+        ),
+    SubLedger1 = subledger(RootLedger, Opts),
+    SL1ID = hb_message:id(SubLedger1, signed, Opts),
+    ?event(debug, {sl1ID, SL1ID}),
+    SubLedger2 = subledger(RootLedger, Opts),
+    SL2ID = hb_message:id(SubLedger2, signed, Opts),
+    ?event(debug, {sl2ID, SL2ID}),
+    Names = #{
+        Alice => alice,
+        Bob => bob,
+        RootLedger => root,
+        SubLedger1 => subledger1,
+        SubLedger2 => subledger2
+    },
+    ?event(debug, {root_ledger, RootLedger}),
+    ?event(debug, {sl1, SubLedger1}),
+    ?event(debug, {sl2, SubLedger2}),
+    ?assertEqual(100, balance(RootLedger, Alice, Opts)),
+    % 2. Alice sends 90 tokens to herself on SubLedger1.
+    ?event(debug, {transfer_1}),
+    transfer(RootLedger, Alice, Alice, 90, SubLedger1, Opts),
+    ?assertEqual(10, balance(RootLedger, Alice, Opts)),
+    ?assertEqual(90, balance(SubLedger1, Alice, Opts)),
+    ?event(debug, {transfer_2}),
+    PushRes = transfer(SubLedger1, Alice, Alice, 80, SubLedger2, Opts),
+    ?event(debug, {push_res, PushRes}),
+    ?event(debug, {map, map([RootLedger, SubLedger1, SubLedger2], Opts)}),
+    ?assertEqual(80, balance(SubLedger2, Alice, Opts)),
+    ?assertEqual(10, balance(SubLedger1, Alice, Opts)).
+
 %% @doc Verify that registered sub-ledgers are able to send tokens to each other
 %% without the need for messages on the root ledger.
 subledger_to_subledger_test_() -> {timeout, 30, fun subledger_to_subledger/0}.
@@ -615,12 +655,11 @@ subledger_to_subledger() ->
     },
     % 1. Alice has tokens on the root ledger.
     ?assertEqual(100, balance(RootLedger, Alice, Opts)),
-    % 2. Alice registers with SubLedger1.
-    register(SubLedger1, SubLedger2, Opts),
-    % 3. Alice sends 90 tokens to herself on SubLedger1.
+    % 2. Alice sends 90 tokens to herself on SubLedger1.
     transfer(RootLedger, Alice, Alice, 90, SubLedger1, Opts),
-    % 4. Alice sends 10 tokens to Bob on SubLedger2.
+    % 3. Alice sends 10 tokens to Bob on SubLedger2.
     transfer(SubLedger1, Alice, Bob, 10, SubLedger2, Opts),
+    ?event(debug, {map, map([RootLedger, SubLedger1, SubLedger2], Names, Opts)}),
     ?assertEqual(10, balance(RootLedger, Alice, Opts)),
     ?assertEqual(80, balance(SubLedger1, Alice, Opts)),
     ?assertEqual(10, balance(SubLedger2, Bob, Opts)),

src/dev_p4.erl

@@ -238,7 +238,7 @@ response(State, RawResponse, NodeMsg) ->
                                     end,
                                 <<"request">> => Request
                             },
-                            hb_opts:get(priv_wallet, no_viable_wallet, NodeMsg)
+                            NodeMsg
                         ),
                     ?event(payment,
                         {post_charge,