Add hard_break_escape test suite.

potatosalad · potatosalad · commit cbaff5fd2a0f · 2025-10-08T16:31:24.000-05:00
diff --git a/apps/markdown/include/markdown_debug_types.hrl b/apps/markdown/include/markdown_debug_types.hrl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<5dfe1184677fa8b2df412792a1cdff14>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
 %%% Copyright (c) WhatsApp LLC
diff --git a/apps/markdown/include/markdown_event.hrl b/apps/markdown/include/markdown_event.hrl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<443ad3aacfcbd218931667178549b4bf>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
 %%% Copyright (c) WhatsApp LLC
diff --git a/apps/markdown/include/markdown_resolve.hrl b/apps/markdown/include/markdown_resolve.hrl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<f3c34f460fc33134b76f71b572dc5dd5>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
 %%% Copyright (c) WhatsApp LLC
diff --git a/apps/markdown/include/markdown_state.hrl b/apps/markdown/include/markdown_state.hrl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<d47b91f960b582a199d7c8e9bd377e58>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
 %%% Copyright (c) WhatsApp LLC
diff --git a/apps/markdown/src/construct/markdown_construct_gfm_table.erl b/apps/markdown/src/construct/markdown_construct_gfm_table.erl
@@ -1151,9 +1151,9 @@ resolve_loop(Tokenizer1 = #markdown_tokenizer{events = Events}, State1 = #resolv
 resolve_loop(Tokenizer1 = #markdown_tokenizer{}, State1 = #resolve{}) ->
     {Tokenizer1, State1}.
 
--spec cell_to_string(cell()) -> dynamic().
-cell_to_string(#cell{exit = Exit, enter = Enter, data_enter = DataEnter, data_exit = DataExit}) ->
-    markdown_debug:rust_debug_string({Exit, Enter, DataEnter, DataExit}).
+% -spec cell_to_string(cell()) -> dynamic().
+% cell_to_string(#cell{exit = Exit, enter = Enter, data_enter = DataEnter, data_exit = DataExit}) ->
+%     markdown_debug:rust_debug_string({Exit, Enter, DataEnter, DataExit}).
 
 %% @private
 -spec resolve__enter_data(Tokenizer, State, Event) -> {Tokenizer, State} when
diff --git a/apps/markdown/src/construct/markdown_construct_text.erl b/apps/markdown/src/construct/markdown_construct_text.erl
@@ -340,9 +340,9 @@ resolve(
         }
     }
 ) ->
-    io:format("\n\n[~w] BEFORE text:resolve\n\n~ts\n\n", [
-        markdown:counter_get(), markdown_debug:rust_debug_string(Tokenizer1)
-    ]),
+    % io:format("\n\n[~w] BEFORE text:resolve\n\n~ts\n\n", [
+    %     markdown:counter_get(), markdown_debug:rust_debug_string(Tokenizer1)
+    % ]),
     Tokenizer2 = markdown_construct_partial_whitespace:resolve_whitespace(Tokenizer1, HardBreakTrailing, true),
     Tokenizer3 =
         case GfmAutolinkLiteral of
@@ -354,7 +354,7 @@ resolve(
     #markdown_tokenizer{events = Events3, map = EditMap3} = Tokenizer3,
     {EditMap4, Events4} = markdown_edit_map:consume(EditMap3, Events3),
     Tokenizer4 = Tokenizer3#markdown_tokenizer{events = Events4, map = EditMap4},
-    io:format("\n\n[~w] AFTER text:resolve\n\n~ts\n\n", [
-        markdown:counter_get(), markdown_debug:rust_debug_string(Tokenizer4)
-    ]),
+    % io:format("\n\n[~w] AFTER text:resolve\n\n~ts\n\n", [
+    %     markdown:counter_get(), markdown_debug:rust_debug_string(Tokenizer4)
+    % ]),
     {Tokenizer4, {ok, none}}.
diff --git a/apps/markdown/src/markdown.erl b/apps/markdown/src/markdown.erl
@@ -91,7 +91,7 @@ counter_reset() ->
 
 -spec doc() -> binary().
 doc() ->
-    <<"| a |\n| - |\n| b |"/utf8>>.
+    <<"a\\\nb."/utf8>>.
     % <<"[foo [bar](/uri)](/uri)\n"/utf8>>.
     % <<"*foo __bar *baz bim__ bam*"/utf8>>.
     % <<"[bar](/foo)">>.
@@ -400,6 +400,7 @@ println!("{:?}", tree);
 to_mdast(MarkdownInput, ParseOptions = #markdown_parse_options{}) when is_binary(MarkdownInput) ->
     case markdown_parser:parse(MarkdownInput, ParseOptions) of
         {ok, {Events, ParseState}} ->
+            io:format("~ts\n", [markdown_debug:rust_debug_string(Events)]),
             Bytes = markdown_parse_state:bytes(ParseState),
             markdown_mdast:compile(Events, Bytes);
         Error = {error, _Message} ->
diff --git a/apps/markdown/src/markdown_debug_types.erl b/apps/markdown/src/markdown_debug_types.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<4917bacc2f6237f415fd2c0bcd0068e9>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown/src/markdown_event.erl b/apps/markdown/src/markdown_event.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<5ac81054a0d69c056c9bd24a051e198a>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown/src/markdown_resolve.erl b/apps/markdown/src/markdown_resolve.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<cc3d043dff4a4ba4cdff18026da3b361>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown/src/markdown_state.erl b/apps/markdown/src/markdown_state.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<cc46d9303501b9cc16c8b3cba83addda>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown/src/mdast/markdown_mdast.erl b/apps/markdown/src/mdast/markdown_mdast.erl
@@ -1552,7 +1552,7 @@ on_exit_line_ending(CompileContext1 = #markdown_mdast_compile_context{events = E
         #markdown_mdast_compile_context{heading_setext_text_after = true} ->
             %% Ignore.
             {ok, CompileContext1};
-        #markdown_mdast_compile_context{heading_setext_text_after = hard_break_after} ->
+        #markdown_mdast_compile_context{hard_break_after = true} ->
             %% Line ending position after hard break is part of it.
             End = markdown_point:to_unist((markdown_vec:get(Events, Index))#markdown_event.point),
             NodeMutFunc = fun on_exit_line_ending__hard_break_after_node_mut_func/2,
@@ -1598,8 +1598,10 @@ on_exit_line_ending__hard_break_after_children_mut_func(Children1, {some, End})
     %% BEGIN: assertions
     ?assertMatch({some, _}, markdown_vec:last_option(Children1), "expected tail (break)"),
     %% END: assertions
-    {some, Tail1} = markdown_vec:last_option(Children1),
-    Tail2 = markdown_mdast_node:position_set(Tail1, {some, End}),
+    Tail1 = markdown_vec:last(Children1),
+    {some, Position1} = markdown_mdast_node:position(Tail1),
+    Position2 = Position1#markdown_unist_position{'end' = End},
+    Tail2 = markdown_mdast_node:position_set(Tail1, {some, Position2}),
     Children2 = markdown_vec:set_last(Children1, Tail2),
     {Children2, none}.
 
diff --git a/apps/markdown/src/util/markdown_util_unicode.erl b/apps/markdown/src/util/markdown_util_unicode.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<45c1fc08db97fdb7da304feb53619b1b>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_commonmark_SUITE.erl b/apps/markdown_test/test/markdown_commonmark_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<e5574d1f03dc19f2cfb2e7bf8f0058e5>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_definition_SUITE.erl b/apps/markdown_test/test/markdown_definition_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<d2b10ff3b1a5193e6c86a46d72edea4a>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_frontmatter_SUITE.erl b/apps/markdown_test/test/markdown_frontmatter_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<550c37c234a1ecdc35d80de4d044f1e2>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_gfm_autolink_literal_SUITE.erl b/apps/markdown_test/test/markdown_gfm_autolink_literal_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<ba745a0705186fc64c95c5da552272b3>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_gfm_footnote_SUITE.erl b/apps/markdown_test/test/markdown_gfm_footnote_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<cc323d60ee2a7a78bb15fb105dc7f43d>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_gfm_strikethrough_SUITE.erl b/apps/markdown_test/test/markdown_gfm_strikethrough_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<20ed987283cfd5171fb09715418a5729>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_gfm_table_SUITE.erl b/apps/markdown_test/test/markdown_gfm_table_SUITE.erl
@@ -1,5 +1,5 @@
 %%% % DO NOT EDIT: this file was generated by 'just codegen'
-%%% % @generated SignedSource<<51dfd5fb3217c53dc2fde97af1f2d348>>
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
 %%% % @format
 %%%-----------------------------------------------------------------------------
 %%% Copyright (c) Meta Platforms, Inc. and affiliates.
diff --git a/apps/markdown_test/test/markdown_hard_break_escape_SUITE.erl b/apps/markdown_test/test/markdown_hard_break_escape_SUITE.erl
@@ -0,0 +1,197 @@
+%%% % DO NOT EDIT: this file was generated by 'just codegen'
+%%% % @generated <<SignedSource::*O*zOeWoEQle#+L!plEphiEmie@IsG>>
+%%% % @format
+%%%-----------------------------------------------------------------------------
+%%% Copyright (c) Meta Platforms, Inc. and affiliates.
+%%% Copyright (c) WhatsApp LLC
+%%%
+%%% This source code is licensed under the MIT license found in the
+%%% LICENSE.md file in the root directory of this source tree.
+%%%-----------------------------------------------------------------------------
+-module(markdown_hard_break_escape_SUITE).
+-moduledoc """
+
+""".
+-moduledoc #{author => ["Andrew Bennett <potatosaladx@meta.com>"]}.
+-moduledoc #{created => "", modified => ""}.
+-moduledoc #{copyright => "Meta Platforms, Inc. and affiliates."}.
+-compile(warn_missing_spec_all).
+-oncall("whatsapp_clr").
+
+-include_lib("markdown/include/markdown_mdast.hrl").
+-include_lib("markdown/include/markdown_util.hrl").
+-include_lib("stdlib/include/assert.hrl").
+
+-behaviour(ct_suite).
+
+%% ct_suite callbacks
+-export([
+    all/0,
+    groups/0,
+    init_per_suite/1,
+    end_per_suite/1,
+    init_per_group/2,
+    end_per_group/2
+]).
+
+%% Test Cases
+-export([
+    test_hard_break_escape_case_1/1,
+    test_hard_break_escape_case_2/1,
+    test_hard_break_escape_case_3/1,
+    test_hard_break_escape_case_4/1,
+    test_hard_break_escape_case_5/1,
+    test_hard_break_escape_case_6/1,
+    test_hard_break_escape_case_7/1,
+    test_hard_break_escape_case_8/1
+]).
+
+%%%=============================================================================
+%%% ct_suite callbacks
+%%%=============================================================================
+
+-spec all() -> markdown_test:all().
+all() ->
+    [
+        {group, static}
+    ].
+
+-spec groups() -> markdown_test:groups().
+groups() ->
+    [
+        {static, [parallel], [
+            test_hard_break_escape_case_1,
+            test_hard_break_escape_case_2,
+            test_hard_break_escape_case_3,
+            test_hard_break_escape_case_4,
+            test_hard_break_escape_case_5,
+            test_hard_break_escape_case_6,
+            test_hard_break_escape_case_7,
+            test_hard_break_escape_case_8
+        ]}
+    ].
+
+-spec init_per_suite(Config :: ct_suite:ct_config()) -> markdown_test:init_per_suite().
+init_per_suite(Config) ->
+    Config.
+
+-spec end_per_suite(Config :: ct_suite:ct_config()) -> markdown_test:end_per_suite().
+end_per_suite(_Config) ->
+    ok.
+
+-spec init_per_group(GroupName :: ct_suite:ct_groupname(), Config :: ct_suite:ct_config()) ->
+    markdown_test:init_per_group().
+init_per_group(_Group, Config) ->
+    Config.
+
+-spec end_per_group(GroupName :: ct_suite:ct_groupname(), Config :: ct_suite:ct_config()) ->
+    markdown_test:end_per_group().
+end_per_group(_Group, _Config) ->
+    ok.
+
+%%%=============================================================================
+%%% Test Cases
+%%%=============================================================================
+
+-spec test_hard_break_escape_case_1(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_1(_Config) ->
+    ?assertMatch(
+        {ok, <<"<p>foo<br />\nbaz</p>"/utf8>>},
+        markdown:to_html(<<"foo\\\nbaz"/utf8>>),
+        "should support a backslash to form a hard break"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_2(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_2(_Config) ->
+    ?assertMatch(
+        {ok, <<"<p>foo<br />\nbar</p>"/utf8>>},
+        markdown:to_html(<<"foo\\\n     bar"/utf8>>),
+        "should support leading spaces after an escape hard break"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_3(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_3(_Config) ->
+    ?assertMatch(
+        {ok, <<"<p><em>foo<br />\nbar</em></p>"/utf8>>},
+        markdown:to_html(<<"*foo\\\nbar*"/utf8>>),
+        "should support escape hard breaks in emphasis"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_4(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_4(_Config) ->
+    ?assertMatch(
+        {ok, <<"<p><code>code\\ text</code></p>"/utf8>>},
+        markdown:to_html(<<"``code\\\ntext``"/utf8>>),
+        "should not support escape hard breaks in code"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_5(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_5(_Config) ->
+    ?assertMatch(
+        {ok, <<"<p>foo\\</p>"/utf8>>},
+        markdown:to_html(<<"foo\\"/utf8>>),
+        "should not support escape hard breaks at the end of a paragraph"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_6(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_6(_Config) ->
+    ?assertMatch(
+        {ok, <<"<h3>foo\\</h3>"/utf8>>},
+        markdown:to_html(<<"### foo\\"/utf8>>),
+        "should not support escape hard breaks at the end of a heading"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_7(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_7(_Config) ->
+    ?assertMatch(
+        {ok, <<"<p>a\\\nb</p>"/utf8>>},
+        markdown:to_html_with_options(
+            <<"a\\\nb"/utf8>>, markdown_options:default(#{parse => #{constructs => #{hard_break_escape => false}}})
+        ),
+        "should support turning off hard break (escape)"
+    ),
+    ok.
+
+-spec test_hard_break_escape_case_8(Config) -> markdown_test:testcase() when
+    Config :: ct_suite:ct_config().
+test_hard_break_escape_case_8(_Config) ->
+    ?assertEqual(
+        {ok,
+            markdown_mdast_node:root(#markdown_mdast_root{
+                children = ?'vec!'([
+                    markdown_mdast_node:paragraph(#markdown_mdast_paragraph{
+                        children = ?'vec!'([
+                            markdown_mdast_node:text(#markdown_mdast_text{
+                                value = <<"a">>,
+                                position = {some, markdown_unist_position:new(1, 1, 0, 1, 2, 1)}
+                            }),
+                            markdown_mdast_node:break(#markdown_mdast_break{
+                                position = {some, markdown_unist_position:new(1, 2, 1, 2, 1, 3)}
+                            }),
+                            markdown_mdast_node:text(#markdown_mdast_text{
+                                value = <<"b.">>,
+                                position = {some, markdown_unist_position:new(2, 1, 3, 2, 3, 5)}
+                            })
+                        ]),
+                        position = {some, markdown_unist_position:new(1, 1, 0, 2, 3, 5)}
+                    })
+                ]),
+                position = {some, markdown_unist_position:new(1, 1, 0, 2, 3, 5)}
+            })},
+        markdown:to_mdast(<<"a\\\nb."/utf8>>, markdown_parse_options:default()),
+        "should support hard break (escape) as `Break`s in mdast"
+    ),
+    ok.
diff --git a/codegen/src/erlang_markdown_codegen/context/test.py b/codegen/src/erlang_markdown_codegen/context/test.py
@@ -212,6 +212,8 @@ def __post_init__(self) -> None:
                         f"For TestCaseToMdast, parse_options must be a TestSharedParseOptions or TestSuiteParseOptions"
                     )
                 self.parse_options = shared_parse_options
+            elif self.schema.parse_options == "ParseOptions::default()":
+                self.parse_options = "markdown_parse_options:default()"
             elif self.schema.parse_options == "ParseOptions::gfm()":
                 self.parse_options = "markdown_parse_options:gfm()"
             else:
diff --git a/codegen/test/hard_break_escape.yaml b/codegen/test/hard_break_escape.yaml
