From 37fdec8fc264b9886e91cb0ae1947d2da7901488 Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Tue, 5 Dec 2023 20:54:57 +0100 Subject: [PATCH 01/12] Implement localized quotation marks --- src/csl/mod.rs | 5 +++++ src/types/strings.rs | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/csl/mod.rs b/src/csl/mod.rs index a366f75f..db56a97b 100644 --- a/src/csl/mod.rs +++ b/src/csl/mod.rs @@ -2390,6 +2390,11 @@ impl<'a, T: EntryLike> Context<'a, T> { self.writing.buf.push_verbatim(&chunk.value); self.writing.pull_punctuation = false; } + ChunkKind::Quote => { + self.push_quotes(); + self.push_str(&chunk.value); + self.pop_quotes(); + } ChunkKind::Math => { self.writing.buf.prevent_trimming(); self.writing.save_to_block(); diff --git a/src/types/strings.rs b/src/types/strings.rs index c374fe9b..a3fd7e75 100644 --- a/src/types/strings.rs +++ b/src/types/strings.rs @@ -357,7 +357,7 @@ impl ChunkedString { let config = c.case(); for chunk in &self.0 { match chunk.kind { - ChunkKind::Normal => c.reconfigure(config), + ChunkKind::Normal | ChunkKind::Quote => c.reconfigure(config), ChunkKind::Verbatim | ChunkKind::Math => c.reconfigure(Case::NoTransform), }; @@ -416,6 +416,13 @@ impl FromStr for ChunkedString { '$' => { kind = ChunkKind::Math; } + '"' if kind == ChunkKind::Quote => { + kind = + if depth > 0 { ChunkKind::Verbatim } else { ChunkKind::Normal }; + } + '"' => { + kind = ChunkKind::Quote; + } _ => chunks.push_char(c, kind), } } @@ -573,6 +580,11 @@ impl StringChunk { write_escaped(self, buf)?; buf.write_char('$')?; } + ChunkKind::Quote => { + buf.write_char('"')?; + write_escaped(self, buf)?; + buf.write_char('"')?; + } } Ok(()) @@ -590,6 +602,8 @@ pub enum ChunkKind { /// The contained markup is expected to be evaluated using /// [Typst](https://typst.app/). Math, + /// Quotation marks to be formatted according to the locale. + Quote, } /// The kind of a string chunk for use with the case folder. @@ -609,7 +623,7 @@ impl TryFrom for FoldableKind { match value { ChunkKind::Normal => Ok(Self::Normal), ChunkKind::Verbatim => Ok(Self::Verbatim), - ChunkKind::Math => Err(()), + ChunkKind::Math | ChunkKind::Quote => Err(()), } } } From 2e0bfcbef9b4097fbc2878f1f1be527de9ce9d09 Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Tue, 5 Dec 2023 21:08:23 +0100 Subject: [PATCH 02/12] Document changes --- docs/file-format.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/file-format.md b/docs/file-format.md index 2ec887a4..4fbb9afc 100644 --- a/docs/file-format.md +++ b/docs/file-format.md @@ -421,6 +421,20 @@ publisher: Title and sentence case folding will always be deactivated if your item has set the `language` key to something other than English. +Double quotation marks in a formattable string will be we transformed to +fitting opening and closing quotation marks according to the chosen locale. +Quotes will always be appropriately nested: + +```yaml +title: Reflections on "Harlem" and Other Poems +``` + +If you use quotation marks as YAML delimiters, simply escape the quotation marks in the title: + +```yaml +title: "Reflections on \"Harlem\" and Other Poems" +``` + You can also include mathematical markup evaluated by [Typst](https://typst.app) by wrapping it in dollars. From 7d6aa4e4051c504ed5a96ebbeab28a4966edddcd Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Tue, 9 Apr 2024 15:21:12 +0200 Subject: [PATCH 03/12] Start smart quote --- src/csl/mod.rs | 6 +----- src/types/strings.rs | 23 ++++++----------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/csl/mod.rs b/src/csl/mod.rs index e815fdb8..4753a6b9 100644 --- a/src/csl/mod.rs +++ b/src/csl/mod.rs @@ -36,6 +36,7 @@ use self::taxonomy::{EntryLike, NumberVariableResult}; pub mod archive; mod citation_label; mod elem; +mod quote; mod rendering; mod sort; mod taxonomy; @@ -2372,11 +2373,6 @@ impl<'a, T: EntryLike> Context<'a, T> { self.writing.buf.push_verbatim(&chunk.value); self.writing.pull_punctuation = false; } - ChunkKind::Quote => { - self.push_quotes(); - self.push_str(&chunk.value); - self.pop_quotes(); - } ChunkKind::Math => { self.writing.buf.prevent_trimming(); self.writing.save_to_block(); diff --git a/src/types/strings.rs b/src/types/strings.rs index a3fd7e75..2b9c3bb7 100644 --- a/src/types/strings.rs +++ b/src/types/strings.rs @@ -8,7 +8,10 @@ use serde::{de::Visitor, ser::SerializeMap, Deserialize, Serialize}; use thiserror::Error; use unscanny::Scanner; -use crate::lang::{Case, CaseFolder, SentenceCase, TitleCase}; +use crate::{ + csl::Context, + lang::{Case, CaseFolder, SentenceCase, TitleCase}, +}; /// A string for presentation. /// @@ -357,7 +360,7 @@ impl ChunkedString { let config = c.case(); for chunk in &self.0 { match chunk.kind { - ChunkKind::Normal | ChunkKind::Quote => c.reconfigure(config), + ChunkKind::Normal => c.reconfigure(config), ChunkKind::Verbatim | ChunkKind::Math => c.reconfigure(Case::NoTransform), }; @@ -416,13 +419,6 @@ impl FromStr for ChunkedString { '$' => { kind = ChunkKind::Math; } - '"' if kind == ChunkKind::Quote => { - kind = - if depth > 0 { ChunkKind::Verbatim } else { ChunkKind::Normal }; - } - '"' => { - kind = ChunkKind::Quote; - } _ => chunks.push_char(c, kind), } } @@ -580,11 +576,6 @@ impl StringChunk { write_escaped(self, buf)?; buf.write_char('$')?; } - ChunkKind::Quote => { - buf.write_char('"')?; - write_escaped(self, buf)?; - buf.write_char('"')?; - } } Ok(()) @@ -602,8 +593,6 @@ pub enum ChunkKind { /// The contained markup is expected to be evaluated using /// [Typst](https://typst.app/). Math, - /// Quotation marks to be formatted according to the locale. - Quote, } /// The kind of a string chunk for use with the case folder. @@ -623,7 +612,7 @@ impl TryFrom for FoldableKind { match value { ChunkKind::Normal => Ok(Self::Normal), ChunkKind::Verbatim => Ok(Self::Verbatim), - ChunkKind::Math | ChunkKind::Quote => Err(()), + ChunkKind::Math => Err(()), } } } From aa474f70aff19fb6c653080f6ef41a5fc2e2d6a9 Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Tue, 22 Apr 2025 10:46:11 +0200 Subject: [PATCH 04/12] Use SmartQuoter --- src/csl/mod.rs | 16 ++- src/csl/quote.rs | 171 +++++++++++++++++++++++++++ src/types/strings.rs | 5 +- tests/local/flipflop_OrphanQuote.txt | 51 ++++++++ 4 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 src/csl/quote.rs create mode 100644 tests/local/flipflop_OrphanQuote.txt diff --git a/src/csl/mod.rs b/src/csl/mod.rs index d668a02e..6feb67d2 100644 --- a/src/csl/mod.rs +++ b/src/csl/mod.rs @@ -19,6 +19,7 @@ use citationberg::{ }; use citationberg::{DateForm, LongShortForm, OrdinalLookup, TextCase}; use indexmap::IndexSet; +use quote::{SmartQuoter, SmartQuotes}; use crate::csl::elem::{simplify_children, NonEmptyStack}; use crate::csl::rendering::names::NameDisambiguationProperties; @@ -2398,7 +2399,20 @@ impl<'a, T: EntryLike> Context<'a, T> { /// Add a string to the buffer. fn push_str(&mut self, s: &str) { - let s = self.do_pull_punctuation(s); + let mut modified = String::with_capacity(s.len()); + let mut quoter = SmartQuoter::default(); + let quotes = SmartQuotes::get(self); + let mut before = None; + for c in s.chars() { + match c { + '"' => modified.push_str(quoter.quote(before, "es, true)), + '\'' => modified.push_str(quoter.quote(before, "es, true)), + c => modified.push(c), + } + before = Some(c); + } + + let s = self.do_pull_punctuation(&modified); self.writing.reconfigure(); diff --git a/src/csl/quote.rs b/src/csl/quote.rs new file mode 100644 index 00000000..78c2e060 --- /dev/null +++ b/src/csl/quote.rs @@ -0,0 +1,171 @@ +use citationberg::{taxonomy::OtherTerm, TermForm}; + +use super::{taxonomy::EntryLike, Context}; + +/// A smart quote substitutor with zero lookahead. +/// TODO: Move to its own crate. +#[derive(Debug, Clone)] +pub struct SmartQuoter { + /// The amount of quotes that have been opened. + depth: u8, + /// Each bit indicates whether the quote at this nesting depth is a double. + /// Maximum supported depth is thus 32. + kinds: u32, +} + +impl SmartQuoter { + /// Start quoting. + pub fn new() -> Self { + Self { depth: 0, kinds: 0 } + } + + /// Determine which smart quote to substitute given this quoter's nesting + /// state and the character immediately preceding the quote. + pub fn quote<'a>( + &mut self, + before: Option, + quotes: &SmartQuotes<'a>, + double: bool, + ) -> &'a str { + let opened = self.top(); + let before = before.unwrap_or(' '); + + // If we are after a number and haven't most recently opened a quote of + // this kind, produce a prime. Otherwise, we prefer a closing quote. + if before.is_numeric() && opened != Some(double) { + return if double { "″" } else { "′" }; + } + + // If we have a single smart quote, didn't recently open a single + // quotation, and are after an alphabetic char or an object (e.g. a + // math equation), interpret this as an apostrophe. + if !double + && opened != Some(false) + && (before.is_alphabetic() || before == '\u{FFFC}') + { + return "’"; + } + + // If the most recently opened quotation is of this kind and the + // previous char does not indicate a nested quotation, close it. + if opened == Some(double) + && !before.is_whitespace() + && !is_newline(before) + && !is_opening_bracket(before) + { + self.pop(); + return quotes.close(double); + } + + // Otherwise, open a new the quotation. + self.push(double); + quotes.open(double) + } + + /// The top of our quotation stack. Returns `Some(double)` for the most + /// recently opened quote or `None` if we didn't open one. + fn top(&self) -> Option { + self.depth.checked_sub(1).map(|i| (self.kinds >> i) & 1 == 1) + } + + /// Push onto the quotation stack. + fn push(&mut self, double: bool) { + if self.depth < 32 { + self.kinds |= (double as u32) << self.depth; + self.depth += 1; + } + } + + /// Pop from the quotation stack. + fn pop(&mut self) { + self.depth -= 1; + self.kinds &= (1 << self.depth) - 1; + } +} + +impl Default for SmartQuoter { + fn default() -> Self { + Self::new() + } +} + +/// Whether the character is an opening bracket, parenthesis, or brace. +fn is_opening_bracket(c: char) -> bool { + matches!(c, '(' | '{' | '[') +} + +/// Whether a character is interpreted as a newline by Typst. +#[inline] +pub fn is_newline(character: char) -> bool { + matches!( + character, + // Line Feed, Vertical Tab, Form Feed, Carriage Return. + '\n' | '\x0B' | '\x0C' | '\r' | + // Next Line, Line Separator, Paragraph Separator. + '\u{0085}' | '\u{2028}' | '\u{2029}' + ) +} + +/// Decides which quotes to substitute smart quotes with. +pub struct SmartQuotes<'s> { + /// The opening single quote. + pub single_open: &'s str, + /// The closing single quote. + pub single_close: &'s str, + /// The opening double quote. + pub double_open: &'s str, + /// The closing double quote. + pub double_close: &'s str, +} + +impl<'s> SmartQuotes<'s> { + /// Create a new `Quotes` struct with the given quotes, optionally falling + /// back to the defaults for a language and region. + /// + /// The language should be specified as an all-lowercase ISO 639-1 code, the + /// region as an all-uppercase ISO 3166-alpha2 code. + /// + /// Currently, the supported languages are: English, Czech, Danish, German, + /// Swiss / Liechtensteinian German, Estonian, Icelandic, Italian, Latin, + /// Lithuanian, Latvian, Slovak, Slovenian, Spanish, Bosnian, Finnish, + /// Swedish, French, Swiss French, Hungarian, Polish, Romanian, Japanese, + /// Traditional Chinese, Russian, Norwegian, Hebrew and Croatian. + /// + /// For unknown languages, the English quotes are used as fallback. + pub fn get(ctx: &'s Context<'s, T>) -> Self { + let default = ("'", "\""); + + Self { + single_open: ctx + .term(OtherTerm::OpenInnerQuote.into(), TermForm::default(), false) + .unwrap_or(default.0), + single_close: ctx + .term(OtherTerm::CloseInnerQuote.into(), TermForm::default(), false) + .unwrap_or(default.0), + double_open: ctx + .term(OtherTerm::OpenQuote.into(), TermForm::default(), false) + .unwrap_or(default.1), + double_close: ctx + .term(OtherTerm::CloseQuote.into(), TermForm::default(), false) + .unwrap_or(default.1), + } + } + + /// The opening quote. + pub fn open(&self, double: bool) -> &'s str { + if double { + self.double_open + } else { + self.single_open + } + } + + /// The closing quote. + pub fn close(&self, double: bool) -> &'s str { + if double { + self.double_close + } else { + self.single_close + } + } +} diff --git a/src/types/strings.rs b/src/types/strings.rs index 2b9c3bb7..c374fe9b 100644 --- a/src/types/strings.rs +++ b/src/types/strings.rs @@ -8,10 +8,7 @@ use serde::{de::Visitor, ser::SerializeMap, Deserialize, Serialize}; use thiserror::Error; use unscanny::Scanner; -use crate::{ - csl::Context, - lang::{Case, CaseFolder, SentenceCase, TitleCase}, -}; +use crate::lang::{Case, CaseFolder, SentenceCase, TitleCase}; /// A string for presentation. /// diff --git a/tests/local/flipflop_OrphanQuote.txt b/tests/local/flipflop_OrphanQuote.txt new file mode 100644 index 00000000..02835066 --- /dev/null +++ b/tests/local/flipflop_OrphanQuote.txt @@ -0,0 +1,51 @@ +>>==== MODE ====>> +citation +<<==== MODE ====<< + +>>==== RESULT ====>> +Nation of “Positive Obligations “ of State under the European Convention on Human Rights (1) +<<==== RESULT ====<< + +>>==== CITATION-ITEMS ====>> +[ + [ + { + "id": "ITEM-1" + } + ] +] +<<==== CITATION-ITEMS ====<< + +>>==== CSL ====>> + +<<==== CSL ====<< + +>>==== INPUT ====>> +[ + { + "id": "ITEM-1", + "title": "Nation of \"Positive Obligations \" of State under the European Convention on Human Rights (1)", + "type": "book" + } +] +<<==== INPUT ====<< + + + +>>===== VERSION =====>> +1.0 +<<===== VERSION =====<< \ No newline at end of file From f5db8ee95e52aa361fb08d9a70fff2d1f2c4f27b Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Tue, 22 Apr 2025 10:48:43 +0200 Subject: [PATCH 05/12] Remove citeproc test --- tests/citeproc-pass.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/citeproc-pass.txt b/tests/citeproc-pass.txt index f325435c..6c0e60bf 100644 --- a/tests/citeproc-pass.txt +++ b/tests/citeproc-pass.txt @@ -149,7 +149,6 @@ disambiguate_YearSuffixMacroSameYearImplicit disambiguate_YearSuffixWithEtAlSubsequent display_DisplayBlock etal_CitationAndBibliographyDecorationsInBibliography -flipflop_OrphanQuote form_TitleShort form_TitleShortNoLong form_TitleTestNoLongFalse From 6f128f0124e11ed0aa4d19fb54c2d0f433e5cdbd Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Wed, 23 Apr 2025 21:50:47 +0200 Subject: [PATCH 06/12] Fix handling of quotation marks within quotes; add local test; update passing tests --- src/csl/mod.rs | 5 +- tests/citeproc-pass.txt | 6 ++ tests/local/smartQuotationMarks.txt | 143 ++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 tests/local/smartQuotationMarks.txt diff --git a/src/csl/mod.rs b/src/csl/mod.rs index 6feb67d2..6d4bf167 100644 --- a/src/csl/mod.rs +++ b/src/csl/mod.rs @@ -2403,10 +2403,11 @@ impl<'a, T: EntryLike> Context<'a, T> { let mut quoter = SmartQuoter::default(); let quotes = SmartQuotes::get(self); let mut before = None; + let inner = self.writing.inner_quotes; for c in s.chars() { match c { - '"' => modified.push_str(quoter.quote(before, "es, true)), - '\'' => modified.push_str(quoter.quote(before, "es, true)), + '"' => modified.push_str(quoter.quote(before, "es, !inner)), + '\'' => modified.push_str(quoter.quote(before, "es, inner)), c => modified.push(c), } before = Some(c); diff --git a/tests/citeproc-pass.txt b/tests/citeproc-pass.txt index 6c0e60bf..72d18b5b 100644 --- a/tests/citeproc-pass.txt +++ b/tests/citeproc-pass.txt @@ -14,6 +14,7 @@ bugreports_BadCitationUpdate bugreports_ChineseCharactersFamilyOnlyPluralLabel bugreports_ContextualPluralWithMainItemFields bugreports_DisambiguationAddNamesBibliography +bugreports_DuplicateTerminalPunctuationInBibliography bugreports_EmptyIfMatchNoneFail bugreports_MatchedAuthorAndDate bugreports_NoEventInNestedMacroWithOldProcessor @@ -149,6 +150,7 @@ disambiguate_YearSuffixMacroSameYearImplicit disambiguate_YearSuffixWithEtAlSubsequent display_DisplayBlock etal_CitationAndBibliographyDecorationsInBibliography +flipflop_QuotesNodeLevelMarkup form_TitleShort form_TitleShortNoLong form_TitleTestNoLongFalse @@ -200,6 +202,7 @@ locator_WorkaroundTestForSubVerbo name_AfterInvertedName name_AllCapsInitialsUntouched name_AndTextDelimiterPrecedesLastAlways +name_ApostropheInGivenName name_ArabicShortForms name_ArticularNameAsSortOrder name_ArticularPlain @@ -238,6 +241,7 @@ name_NoNameNode name_NonDroppingParticleDefault name_OnlyFamilyname name_OverridingHierarchicalDelimiter +name_ParsedCommaDelimitedDroppingParticleSortOrderingWithoutAffixes name_ParticleFormatting name_PeriodAfterInitials name_QuashOrdinaryVariableRenderedViaSubstitute @@ -408,6 +412,7 @@ sort_NameParticleInNameSortFalse sort_NameParticleInNameSortTrue sort_NamesUseLast sort_StatusFieldDescending +sort_SubstituteTitle sort_TestInheritance sortseparator_SortSeparatorEmpty substitute_RepeatedNamesOk @@ -417,6 +422,7 @@ substitute_SubstituteOnlyOnceVariable substitute_SuppressOrdinaryVariable textcase_AfterQuote textcase_CapitalsUntouched +textcase_NoSpaceBeforeApostrophe textcase_StopWordBeforeHyphen textcase_TitleCaseNonEnglish textcase_TitleCaseNonEnglish2 diff --git a/tests/local/smartQuotationMarks.txt b/tests/local/smartQuotationMarks.txt new file mode 100644 index 00000000..7502aee0 --- /dev/null +++ b/tests/local/smartQuotationMarks.txt @@ -0,0 +1,143 @@ +>>===== MODE =====>> +citation +<<===== MODE =====<< + + +>>===== DESCRIPTION =====>> +Quotation marks are correctly prettified. + +https://github.com/typst/hayagriva/issues/70 +<<===== DESCRIPTION =====<< + + +>>===== RESULT =====>> +B. Meyer, “Applying ‘Design by contract’,” Computer, vol. 25, no. 10, pp. 40–51, 1992, doi: 10.1109/2.161279. +<<===== RESULT =====<< + + +>>===== INPUT =====>> +[ + { + "id": "M1992", + "type": "article-journal", + "container-title": "Computer", + "DOI": "10.1109/2.161279", + "issue": "10", + "page": "40–51", + "title": "Applying \"Design by contract\"", + "volume": "25", + "author": [ + { + "family": "Meyer", + "given": "Bertrand" + } + ], + "issued": { + "date-parts": [ + [ + "1992" + ] + ] + } + } +] +<<===== INPUT =====<< + + +>>===== CSL =====>> + + +<<===== CSL =====<< + From efb5309148ea8f371373465f27a795f382a4cd03 Mon Sep 17 00:00:00 2001 From: Daniel Drodt <132357467+Drodt@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:10:08 +0200 Subject: [PATCH 07/12] Apply suggestions from code review Co-authored-by: PgBiel <9021226+PgBiel@users.noreply.github.com> --- src/csl/quote.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/csl/quote.rs b/src/csl/quote.rs index 78c2e060..a40f260f 100644 --- a/src/csl/quote.rs +++ b/src/csl/quote.rs @@ -1,9 +1,10 @@ -use citationberg::{taxonomy::OtherTerm, TermForm}; +use citationberg::taxonomy::OtherTerm; +use citationberg::TermForm; -use super::{taxonomy::EntryLike, Context}; +use super::taxonomy::EntryLike; +use super::Context; /// A smart quote substitutor with zero lookahead. -/// TODO: Move to its own crate. #[derive(Debug, Clone)] pub struct SmartQuoter { /// The amount of quotes that have been opened. @@ -50,7 +51,6 @@ impl SmartQuoter { // previous char does not indicate a nested quotation, close it. if opened == Some(double) && !before.is_whitespace() - && !is_newline(before) && !is_opening_bracket(before) { self.pop(); @@ -90,6 +90,7 @@ impl Default for SmartQuoter { } /// Whether the character is an opening bracket, parenthesis, or brace. +#[inline] fn is_opening_bracket(c: char) -> bool { matches!(c, '(' | '{' | '[') } @@ -119,19 +120,8 @@ pub struct SmartQuotes<'s> { } impl<'s> SmartQuotes<'s> { - /// Create a new `Quotes` struct with the given quotes, optionally falling - /// back to the defaults for a language and region. - /// - /// The language should be specified as an all-lowercase ISO 639-1 code, the - /// region as an all-uppercase ISO 3166-alpha2 code. - /// - /// Currently, the supported languages are: English, Czech, Danish, German, - /// Swiss / Liechtensteinian German, Estonian, Icelandic, Italian, Latin, - /// Lithuanian, Latvian, Slovak, Slovenian, Spanish, Bosnian, Finnish, - /// Swedish, French, Swiss French, Hungarian, Polish, Romanian, Japanese, - /// Traditional Chinese, Russian, Norwegian, Hebrew and Croatian. - /// - /// For unknown languages, the English quotes are used as fallback. + /// Create a new `Quotes` struct with quotes taken from the current CSL locale's + /// terms, falling back to `"` and `'` when not available. pub fn get(ctx: &'s Context<'s, T>) -> Self { let default = ("'", "\""); From 074c2e748e3c6cf6dba0aebc3188f634246fb749 Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Fri, 25 Apr 2025 08:13:57 +0200 Subject: [PATCH 08/12] Remove unused function --- src/csl/quote.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/csl/quote.rs b/src/csl/quote.rs index a40f260f..3151732b 100644 --- a/src/csl/quote.rs +++ b/src/csl/quote.rs @@ -95,18 +95,6 @@ fn is_opening_bracket(c: char) -> bool { matches!(c, '(' | '{' | '[') } -/// Whether a character is interpreted as a newline by Typst. -#[inline] -pub fn is_newline(character: char) -> bool { - matches!( - character, - // Line Feed, Vertical Tab, Form Feed, Carriage Return. - '\n' | '\x0B' | '\x0C' | '\r' | - // Next Line, Line Separator, Paragraph Separator. - '\u{0085}' | '\u{2028}' | '\u{2029}' - ) -} - /// Decides which quotes to substitute smart quotes with. pub struct SmartQuotes<'s> { /// The opening single quote. From b69d1b517c42cec9aaae9dea1d9664f9ef40c9e2 Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Fri, 25 Apr 2025 09:22:12 +0200 Subject: [PATCH 09/12] Handled escaped quotation marks and add tests --- src/csl/mod.rs | 18 +++---------- src/csl/quote.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/src/csl/mod.rs b/src/csl/mod.rs index 5e12e755..54f2e28a 100644 --- a/src/csl/mod.rs +++ b/src/csl/mod.rs @@ -19,7 +19,7 @@ use citationberg::{ }; use citationberg::{DateForm, LongShortForm, OrdinalLookup, TextCase}; use indexmap::IndexSet; -use quote::{SmartQuoter, SmartQuotes}; +use quote::{apply_quotes, SmartQuotes}; use crate::csl::elem::{simplify_children, NonEmptyStack}; use crate::csl::rendering::names::NameDisambiguationProperties; @@ -2399,21 +2399,9 @@ impl<'a, T: EntryLike> Context<'a, T> { /// Add a string to the buffer. fn push_str(&mut self, s: &str) { - let mut modified = String::with_capacity(s.len()); - let mut quoter = SmartQuoter::default(); - let quotes = SmartQuotes::get(self); - let mut before = None; - let inner = self.writing.inner_quotes; - for c in s.chars() { - match c { - '"' => modified.push_str(quoter.quote(before, "es, !inner)), - '\'' => modified.push_str(quoter.quote(before, "es, inner)), - c => modified.push(c), - } - before = Some(c); - } + let quoted = apply_quotes(s, &SmartQuotes::get(self), self.writing.inner_quotes); - let s = self.do_pull_punctuation(&modified); + let s = self.do_pull_punctuation("ed); self.writing.reconfigure(); diff --git a/src/csl/quote.rs b/src/csl/quote.rs index 3151732b..e38d681e 100644 --- a/src/csl/quote.rs +++ b/src/csl/quote.rs @@ -4,6 +4,31 @@ use citationberg::TermForm; use super::taxonomy::EntryLike; use super::Context; +pub fn apply_quotes(s: &str, quotes: &SmartQuotes, inner: bool) -> String { + let mut res = String::with_capacity(s.len()); + let mut before = None; + let mut quoter = SmartQuoter::new(); + let mut escape = false; + for c in s.chars() { + match c { + '"' | '\'' if escape => { + res.push(c); + escape = false + } + '"' => res.push_str(quoter.quote(before, "es, !inner)), + '\'' => res.push_str(quoter.quote(before, "es, inner)), + '\\' if escape => { + res.push('\\'); + escape = false + } + '\\' => escape = true, + c => res.push(c), + } + before = Some(c); + } + res +} + /// A smart quote substitutor with zero lookahead. #[derive(Debug, Clone)] pub struct SmartQuoter { @@ -147,3 +172,46 @@ impl<'s> SmartQuotes<'s> { } } } + +#[cfg(test)] +mod tests { + use super::*; + + const US_MARKS: SmartQuotes = SmartQuotes { + single_open: "‘", + single_close: "’", + double_open: "“", + double_close: "”", + }; + const DE_MARKS: SmartQuotes = SmartQuotes { + single_open: "‚", + single_close: "‘", + double_open: "„", + double_close: "“", + }; + + #[test] + fn typst_tests() { + let cases = vec![ + ("“The horse eats no cucumber salad” was the first sentence ever uttered on the ‘telephone.’", r#""The horse eats no cucumber salad" was the first sentence ever uttered on the 'telephone.'"#, &US_MARKS), + ("„Das Pferd frisst keinen Gurkensalat“ war der erste jemals am ‚Fernsprecher‘ gesagte Satz.", r#""Das Pferd frisst keinen Gurkensalat" war der erste jemals am 'Fernsprecher' gesagte Satz."#, &DE_MARKS), + ("“”", r#""""#, &US_MARKS), + ("The 5′11″ ‘quick’ brown fox jumps over the “lazy” dog’s ear.", r#"The 5'11" 'quick' brown fox jumps over the "lazy" dog's ear."#, &US_MARKS), + ("He said “I’m a big fella.”", r#"He said "I'm a big fella.""#, &US_MARKS), + (r#"The 5'11" ‘quick' brown fox jumps over the "lazy’ dog's ear."#, r#"The 5\'11\" 'quick\' brown fox jumps over the \"lazy' dog\'s ear."#, &US_MARKS), + ("“Hello”/“World”", r#""Hello"/"World""#, &US_MARKS), + ("‘“Hello”/“World”’", r#"'"Hello"/"World"'"#, &US_MARKS), + ("“”Hello“/”World“”", r#"""Hello"/"World"""#, &US_MARKS), + ("Straight “A”s and “B”s", r#"Straight "A"s and "B"s"#, &US_MARKS), + ("A 2″ nail.", r#"A 2" nail."#, &US_MARKS), + ("‘A 2″ nail.’", r#"'A 2" nail.'"#, &US_MARKS), + ("“A 2” nail.“", r#""A 2" nail.""#, &US_MARKS), + ("“a [“b”] c”", r#""a ["b"] c""#, &US_MARKS), + ("“a b”c“d e”", r#""a b"c"d e""#, &US_MARKS) + ]; + + for (expected, input, quotes) in cases { + assert_eq!(expected, apply_quotes(input, quotes, false)); + } + } +} From 6bca4765c062e1e1f88ba53381de00405a3b18ca Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Fri, 25 Apr 2025 09:22:56 +0200 Subject: [PATCH 10/12] Clippy --- src/csl/quote.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csl/quote.rs b/src/csl/quote.rs index e38d681e..9e714a30 100644 --- a/src/csl/quote.rs +++ b/src/csl/quote.rs @@ -15,8 +15,8 @@ pub fn apply_quotes(s: &str, quotes: &SmartQuotes, inner: bool) -> String { res.push(c); escape = false } - '"' => res.push_str(quoter.quote(before, "es, !inner)), - '\'' => res.push_str(quoter.quote(before, "es, inner)), + '"' => res.push_str(quoter.quote(before, quotes, !inner)), + '\'' => res.push_str(quoter.quote(before, quotes, inner)), '\\' if escape => { res.push('\\'); escape = false From db812db48800b80c3e65b78a89082de552c70569 Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Fri, 14 Nov 2025 20:14:54 +0100 Subject: [PATCH 11/12] Formatting --- src/csl/mod.rs | 2 +- src/csl/quote.rs | 42 +++++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/csl/mod.rs b/src/csl/mod.rs index 08a36d85..a6bf7ec3 100644 --- a/src/csl/mod.rs +++ b/src/csl/mod.rs @@ -21,7 +21,7 @@ use citationberg::{ }; use citationberg::{DateForm, LongShortForm, OrdinalLookup, TextCase}; use indexmap::IndexSet; -use quote::{apply_quotes, SmartQuotes}; +use quote::{SmartQuotes, apply_quotes}; use crate::csl::elem::{NonEmptyStack, simplify_children}; use crate::csl::rendering::RenderCsl; diff --git a/src/csl/quote.rs b/src/csl/quote.rs index 9e714a30..3b4b67ea 100644 --- a/src/csl/quote.rs +++ b/src/csl/quote.rs @@ -1,8 +1,8 @@ -use citationberg::taxonomy::OtherTerm; use citationberg::TermForm; +use citationberg::taxonomy::OtherTerm; -use super::taxonomy::EntryLike; use super::Context; +use super::taxonomy::EntryLike; pub fn apply_quotes(s: &str, quotes: &SmartQuotes, inner: bool) -> String { let mut res = String::with_capacity(s.len()); @@ -156,20 +156,12 @@ impl<'s> SmartQuotes<'s> { /// The opening quote. pub fn open(&self, double: bool) -> &'s str { - if double { - self.double_open - } else { - self.single_open - } + if double { self.double_open } else { self.single_open } } /// The closing quote. pub fn close(&self, double: bool) -> &'s str { - if double { - self.double_close - } else { - self.single_close - } + if double { self.double_close } else { self.single_close } } } @@ -193,12 +185,28 @@ mod tests { #[test] fn typst_tests() { let cases = vec![ - ("“The horse eats no cucumber salad” was the first sentence ever uttered on the ‘telephone.’", r#""The horse eats no cucumber salad" was the first sentence ever uttered on the 'telephone.'"#, &US_MARKS), - ("„Das Pferd frisst keinen Gurkensalat“ war der erste jemals am ‚Fernsprecher‘ gesagte Satz.", r#""Das Pferd frisst keinen Gurkensalat" war der erste jemals am 'Fernsprecher' gesagte Satz."#, &DE_MARKS), + ( + "“The horse eats no cucumber salad” was the first sentence ever uttered on the ‘telephone.’", + r#""The horse eats no cucumber salad" was the first sentence ever uttered on the 'telephone.'"#, + &US_MARKS, + ), + ( + "„Das Pferd frisst keinen Gurkensalat“ war der erste jemals am ‚Fernsprecher‘ gesagte Satz.", + r#""Das Pferd frisst keinen Gurkensalat" war der erste jemals am 'Fernsprecher' gesagte Satz."#, + &DE_MARKS, + ), ("“”", r#""""#, &US_MARKS), - ("The 5′11″ ‘quick’ brown fox jumps over the “lazy” dog’s ear.", r#"The 5'11" 'quick' brown fox jumps over the "lazy" dog's ear."#, &US_MARKS), + ( + "The 5′11″ ‘quick’ brown fox jumps over the “lazy” dog’s ear.", + r#"The 5'11" 'quick' brown fox jumps over the "lazy" dog's ear."#, + &US_MARKS, + ), ("He said “I’m a big fella.”", r#"He said "I'm a big fella.""#, &US_MARKS), - (r#"The 5'11" ‘quick' brown fox jumps over the "lazy’ dog's ear."#, r#"The 5\'11\" 'quick\' brown fox jumps over the \"lazy' dog\'s ear."#, &US_MARKS), + ( + r#"The 5'11" ‘quick' brown fox jumps over the "lazy’ dog's ear."#, + r#"The 5\'11\" 'quick\' brown fox jumps over the \"lazy' dog\'s ear."#, + &US_MARKS, + ), ("“Hello”/“World”", r#""Hello"/"World""#, &US_MARKS), ("‘“Hello”/“World”’", r#"'"Hello"/"World"'"#, &US_MARKS), ("“”Hello“/”World“”", r#"""Hello"/"World"""#, &US_MARKS), @@ -207,7 +215,7 @@ mod tests { ("‘A 2″ nail.’", r#"'A 2" nail.'"#, &US_MARKS), ("“A 2” nail.“", r#""A 2" nail.""#, &US_MARKS), ("“a [“b”] c”", r#""a ["b"] c""#, &US_MARKS), - ("“a b”c“d e”", r#""a b"c"d e""#, &US_MARKS) + ("“a b”c“d e”", r#""a b"c"d e""#, &US_MARKS), ]; for (expected, input, quotes) in cases { From 72a2cc547c92cba934cfa166ca2af7f50bdbc85d Mon Sep 17 00:00:00 2001 From: DerDrodt Date: Fri, 14 Nov 2025 20:25:23 +0100 Subject: [PATCH 12/12] Re-remove mistakenly added passing test --- tests/citeproc-pass.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/citeproc-pass.txt b/tests/citeproc-pass.txt index 5b964ff2..91f57e18 100644 --- a/tests/citeproc-pass.txt +++ b/tests/citeproc-pass.txt @@ -169,7 +169,6 @@ disambiguate_YearSuffixWithEtAlSubsequent display_DisplayBlock etal_CitationAndBibliographyDecorationsInBibliography etal_UseZeroFirst -flipflop_OrphanQuote flipflop_QuotesNodeLevelMarkup form_TitleShort form_TitleShortNoLong