Add a test for extension compatibility

This adds a test to ensure that the interface for preprocessors and
renderers does not change unexpectedly, particularly in a semver
compatible release.

Closes #1574

Author: ehuss

tests/testsuite/book_test.rs

@@ -533,3 +533,18 @@ pub fn glob_one<P: AsRef<Path>>(path: P, pattern: &str) -> PathBuf {
     }
     first
 }
+
+/// Lists all files at the given directory.
+///
+/// Recursively walks the tree. Paths are relative to the directory.
+pub fn list_all_files(dir: &Path) -> Vec<PathBuf> {
+    walkdir::WalkDir::new(dir)
+        .sort_by_file_name()
+        .into_iter()
+        .map(|entry| {
+            let entry = entry.unwrap();
+            let path = entry.path();
+            path.strip_prefix(dir).unwrap().to_path_buf()
+        })
+        .collect()
+}

tests/testsuite/preprocessor.rs

@@ -1,10 +1,12 @@
 //! Tests for custom preprocessors.
 
+use crate::book_test::list_all_files;
 use crate::prelude::*;
 use anyhow::Result;
-use mdbook_core::book::Book;
+use mdbook_core::book::{Book, BookItem, Chapter};
 use mdbook_driver::builtin_preprocessors::CmdPreprocessor;
 use mdbook_preprocessor::{Preprocessor, PreprocessorContext};
+use snapbox::IntoData;
 use std::sync::{Arc, Mutex};
 
 struct Spy(Arc<Mutex<Inner>>);
@@ -199,3 +201,278 @@ fn with_preprocessor_same_name() {
     assert_eq!(inner.run_count, 1);
     assert_eq!(inner.rendered_with, ["html"]);
 }
+
+// Checks that the interface stays backwards compatible. The interface here
+// should not be changed to fix a compatibility issue unless there is a
+// major-semver version update to mdbook.
+//
+// Note that this tests both preprocessors and renderers. It's in this module
+// for lack of a better location.
+#[test]
+fn extension_compatibility() {
+    // This is here to force you to look at this test if you alter any of
+    // these types such as adding new fields/variants. This test should be
+    // updated accordingly. For example, new `BookItem` variants should be
+    // added to the extension_compatibility book, or new fields should be
+    // added to the expected input/output. This is also a check that these
+    // should only be changed in a semver-breaking release
+    let chapter = Chapter {
+        name: "example".to_string(),
+        content: "content".to_string(),
+        number: None,
+        sub_items: Vec::new(),
+        path: None,
+        source_path: None,
+        parent_names: Vec::new(),
+    };
+    let item = BookItem::Chapter(chapter);
+    match &item {
+        BookItem::Chapter(_) => {}
+        BookItem::Separator => {}
+        BookItem::PartTitle(_) => {}
+    }
+    let items = vec![item];
+    let _book = Book { items };
+
+    let mut test = BookTest::from_dir("preprocessor/extension_compatibility");
+    // Run it once with the preprocessor disabled so that we can verify
+    // that the built book is identical with the preprocessor enabled.
+    test.run("build", |cmd| {
+        cmd.expect_stdout(str![[""]]).expect_stderr(str![[r#"
+[TIMESTAMP] [INFO] (mdbook_driver::mdbook): Book building has started
+[TIMESTAMP] [WARN] (mdbook_driver): The command `./my-preprocessor` for preprocessor `my-preprocessor` was not found, but is marked as optional.
+[TIMESTAMP] [INFO] (mdbook_driver::mdbook): Running the html backend
+[TIMESTAMP] [INFO] (mdbook_html::html_handlebars::hbs_renderer): HTML book written to `[ROOT]/book/html`
+[TIMESTAMP] [WARN] (mdbook_driver): The command `./my-preprocessor` for preprocessor `my-preprocessor` was not found, but is marked as optional.
+[TIMESTAMP] [INFO] (mdbook_driver::mdbook): Running the my-renderer backend
+[TIMESTAMP] [INFO] (mdbook_driver::builtin_renderers): Invoking the "my-renderer" renderer
+[TIMESTAMP] [WARN] (mdbook_driver): The command `./my-renderer` for backend `my-renderer` was not found, but is marked as optional.
+
+"#]]);
+    });
+    let orig_dir = test.dir.join("book.orig");
+    let pre_dir = test.dir.join("book");
+    std::fs::rename(&pre_dir, &orig_dir).unwrap();
+
+    // **CAUTION** DO NOT modify this value unless this is a major-semver change.
+    let book_output = serde_json::json!({
+        "items": [
+          {
+            "Chapter": {
+              "content": "# Prefix chapter\n",
+              "name": "Prefix chapter",
+              "number": null,
+              "parent_names": [],
+              "path": "prefix.md",
+              "source_path": "prefix.md",
+              "sub_items": []
+            }
+          },
+          {
+            "Chapter": {
+              "content": "# Chapter 1\n",
+              "name": "Chapter 1",
+              "number": [
+                1
+              ],
+              "parent_names": [],
+              "path": "chapter_1.md",
+              "source_path": "chapter_1.md",
+              "sub_items": []
+            }
+          },
+          {
+            "Chapter": {
+              "content": "",
+              "name": "Draft chapter",
+              "number": [
+                2
+              ],
+              "parent_names": [],
+              "path": null,
+              "source_path": null,
+              "sub_items": []
+            }
+          },
+          {
+            "PartTitle": "Part title"
+          },
+          {
+            "Chapter": {
+              "content": "# Part chapter\n",
+              "name": "Part chapter",
+              "number": [
+                3
+              ],
+              "parent_names": [],
+              "path": "part/chapter.md",
+              "source_path": "part/chapter.md",
+              "sub_items": [
+                {
+                  "Chapter": {
+                    "content": "# Part sub chapter\n",
+                    "name": "Part sub chapter",
+                    "number": [
+                      3,
+                      1
+                    ],
+                    "parent_names": [
+                      "Part chapter"
+                    ],
+                    "path": "part/sub-chapter.md",
+                    "source_path": "part/sub-chapter.md",
+                    "sub_items": []
+                  }
+                }
+              ]
+            }
+          },
+          "Separator",
+          {
+            "Chapter": {
+              "content": "# Suffix chapter\n",
+              "name": "Suffix chapter",
+              "number": null,
+              "parent_names": [],
+              "path": "suffix.md",
+              "source_path": "suffix.md",
+              "sub_items": []
+            }
+          }
+        ]
+    });
+    let output_str = serde_json::to_string(&book_output).unwrap();
+    // **CAUTION** The only updates allowed here in a semver-compatible
+    // release is to add new fields.
+    let expected_config = serde_json::json!({
+        "book": {
+          "authors": [],
+          "description": null,
+          "language": "en",
+          "text-direction": null,
+          "title": "extension_compatibility"
+        },
+        "output": {
+          "html": {},
+          "my-renderer": {
+            "command": "./my-renderer",
+            "custom-config": "renderer settings",
+            "custom-table": {
+              "extra": "xyz"
+            },
+            "optional": true
+          }
+        },
+        "preprocessor": {
+          "my-preprocessor": {
+            "command": "./my-preprocessor",
+            "custom-config": true,
+            "custom-table": {
+              "extra": "abc"
+            },
+            "optional": true
+          }
+        }
+    });
+
+    // **CAUTION** The only updates allowed here in a semver-compatible
+    // release is to add new fields. The output should not change.
+    let expected_preprocessor_input = serde_json::json!([
+        {
+            "config": expected_config,
+            "mdbook_version": "[VERSION]",
+            "renderer": "html",
+            "root": "[ROOT]"
+        },
+        book_output
+    ]);
+    let expected_renderer_input = serde_json::json!(
+        {
+            "version": "[VERSION]",
+            "root": "[ROOT]",
+            "book": book_output,
+            "config": expected_config,
+            "destination": "[ROOT]/book/my-renderer",
+        }
+    );
+
+    // This preprocessor writes its input to some files, and writes the
+    // hard-coded output specified above.
+    test.rust_program(
+        "my-preprocessor",
+        &r###"
+            use std::fs::OpenOptions;
+            use std::io::{Read, Write};
+            fn main() {
+                let mut args = std::env::args().skip(1);
+                if args.next().as_deref() == Some("supports") {
+                    let mut file = OpenOptions::new()
+                        .create(true)
+                        .append(true)
+                        .open("support-check")
+                        .unwrap();
+                    let renderer = args.next().unwrap();
+                    writeln!(file, "{renderer}").unwrap();
+                    if renderer != "html" {
+                        std::process::exit(1);
+                    }
+                    return;
+                }
+                let mut s = String::new();
+                std::io::stdin().read_to_string(&mut s).unwrap();
+                std::fs::write("preprocessor-input", &s).unwrap();
+                let output = r##"OUTPUT_REPLACE"##;
+                println!("{output}");
+            }
+            "###
+        .replace("OUTPUT_REPLACE", &output_str),
+    )
+    // This renderer writes its input to a file.
+    .rust_program(
+        "my-renderer",
+        &r#"
+            fn main() {
+                use std::io::Read;
+                let mut s = String::new();
+                std::io::stdin().read_to_string(&mut s).unwrap();
+                std::fs::write("renderer-input", &s).unwrap();
+            }
+        "#,
+    )
+    .run("build", |cmd| {
+        cmd.expect_stdout(str![[""]]).expect_stderr(str![[r#"
+[TIMESTAMP] [INFO] (mdbook_driver::mdbook): Book building has started
+[TIMESTAMP] [INFO] (mdbook_driver::mdbook): Running the html backend
+[TIMESTAMP] [INFO] (mdbook_html::html_handlebars::hbs_renderer): HTML book written to `[ROOT]/book/html`
+[TIMESTAMP] [INFO] (mdbook_driver::mdbook): Running the my-renderer backend
+[TIMESTAMP] [INFO] (mdbook_driver::builtin_renderers): Invoking the "my-renderer" renderer
+
+"#]]);
+    })
+    .check_file("support-check", "html\nmy-renderer\n")
+    .check_file(
+        "preprocessor-input",
+        serde_json::to_string(&expected_preprocessor_input)
+            .unwrap()
+            .is_json(),
+    )
+    .check_file(
+        "book/my-renderer/renderer-input",
+        serde_json::to_string(&expected_renderer_input)
+            .unwrap()
+            .is_json(),
+    );
+    // Verify both directories have the exact same output.
+    test.rm_r("book/my-renderer/renderer-input");
+    let orig_files = list_all_files(&orig_dir);
+    let pre_files = list_all_files(&pre_dir);
+    assert_eq!(orig_files, pre_files);
+    for file in &orig_files {
+        let orig_path = orig_dir.join(file);
+        if orig_path.is_file() {
+            let orig = std::fs::read(&orig_path).unwrap();
+            let pre = std::fs::read(&pre_dir.join(file)).unwrap();
+            test.assert.eq(pre, orig);
+        }
+    }
+}

tests/testsuite/preprocessor/extension_compatibility/book.toml

@@ -0,0 +1,18 @@
+[book]
+title = "extension_compatibility"
+
+[preprocessor.my-preprocessor]
+command = "./my-preprocessor"
+custom-config = true
+optional = true
+[preprocessor.my-preprocessor.custom-table]
+extra = "abc"
+
+[output.html]
+
+[output.my-renderer]
+command = "./my-renderer"
+custom-config = "renderer settings"
+optional = true
+[output.my-renderer.custom-table]
+extra = "xyz"

tests/testsuite/preprocessor/extension_compatibility/src/SUMMARY.md

@@ -0,0 +1,15 @@
+# Summary
+
+[Prefix chapter](./prefix.md)
+
+- [Chapter 1](./chapter_1.md)
+- [Draft chapter]()
+
+# Part title
+
+- [Part chapter](./part/chapter.md)
+    - [Part sub chapter](./part/sub-chapter.md)
+
+---
+
+[Suffix chapter](./suffix.md)

tests/testsuite/preprocessor/extension_compatibility/src/chapter_1.md

@@ -0,0 +1 @@
+# Chapter 1

tests/testsuite/preprocessor/extension_compatibility/src/part/chapter.md

@@ -0,0 +1 @@
+# Part chapter

tests/testsuite/preprocessor/extension_compatibility/src/part/sub-chapter.md

@@ -0,0 +1 @@
+# Part sub chapter

tests/testsuite/preprocessor/extension_compatibility/src/prefix.md

@@ -0,0 +1 @@
+# Prefix chapter

tests/testsuite/preprocessor/extension_compatibility/src/suffix.md

@@ -0,0 +1 @@
+# Suffix chapter