[TextServer] Reorganize ICU data handling.

doc/classes/ProjectSettings.xml

@@ -1702,8 +1702,11 @@
 			The locale to fall back to if a translation isn't available in a given language. If left empty, [code]en[/code] (English) will be used.
 			[b]Note:[/b] Not to be confused with [TextServerFallback].
 		</member>
-		<member name="internationalization/locale/include_text_server_data" type="bool" setter="" getter="" default="false">
-			If [code]true[/code], text server break iteration rule sets, dictionaries and other optional data are included in the exported project.
+		<member name="internationalization/locale/include_text_server_break_iterator_data" type="int" setter="" getter="" default="1">
+			Specifies if text server break iteration rule sets, dictionaries, and other optional data is included in the exported project.
+			- [b]Always[/b] ([code]0[/code]) - full data is always included.
+			- [b]Auto[/b] ([code]1[/code]) - full data is included only if it's required for one of locales used by project.
+			- [b]Never[/b] ([code]2[/code]) - full data is never included.
 			[b]Note:[/b] "ICU / HarfBuzz / Graphite" text server data includes dictionaries for Burmese, Chinese, Japanese, Khmer, Lao and Thai as well as Unicode Standard Annex #29 and Unicode Standard Annex #14 word and line breaking rules. Data is about 4 MB large.
 			[b]Note:[/b] [TextServerFallback] does not use additional data.
 		</member>

doc/classes/TextServer.xml

@@ -1113,6 +1113,7 @@
 		</method>
 		<method name="get_support_data" qualifiers="const">
 			<return type="PackedByteArray" />
+			<param index="0" name="config" type="String" default="&quot;full&quot;" />
 			<description>
 				Returns default TextServer database (e.g. ICU break iterators and dictionaries).
 			</description>
@@ -1233,6 +1234,7 @@
 		<method name="save_support_data" qualifiers="const">
 			<return type="bool" />
 			<param index="0" name="filename" type="String" />
+			<param index="1" name="config" type="String" default="&quot;full&quot;" />
 			<description>
 				Saves optional TextServer database (e.g. ICU break iterators and dictionaries) to the file.
 				[b]Note:[/b] This function is used by during project export, to include TextServer database.

doc/classes/TextServerExtension.xml

@@ -1092,6 +1092,7 @@
 		</method>
 		<method name="_get_support_data" qualifiers="virtual const">
 			<return type="PackedByteArray" />
+			<param index="0" name="config" type="String" />
 			<description>
 				Returns default TextServer database (e.g. ICU break iterators and dictionaries).
 			</description>
@@ -1206,6 +1207,7 @@
 		<method name="_save_support_data" qualifiers="virtual const">
 			<return type="bool" />
 			<param index="0" name="filename" type="String" />
+			<param index="1" name="config" type="String" />
 			<description>
 				Saves optional TextServer database (e.g. ICU break iterators and dictionaries) to the file.
 			</description>

editor/export/editor_export_platform.cpp

@@ -1005,57 +1005,63 @@ Dictionary EditorExportPlatform::get_internal_export_files(const Ref<EditorExpor
 
 	// Text server support data.
 	if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) {
-		bool include_data = (bool)get_project_setting(p_preset, "internationalization/locale/include_text_server_data");
-		if (!include_data) {
+		int include_data_mode = (int)get_project_setting(p_preset, "internationalization/locale/include_text_server_break_iterator_data");
+		bool include_full_data = false;
+		if (include_data_mode == 0) {
+			include_full_data = true;
+		} else if (include_data_mode == 1) {
 			Vector<String> translations = get_project_setting(p_preset, "internationalization/locale/translations");
 			translations.push_back(get_project_setting(p_preset, "internationalization/locale/fallback"));
 			for (const String &t : translations) {
 				if (TS->is_locale_using_support_data(t)) {
-					include_data = true;
+					include_full_data = true;
 					break;
 				}
 			}
 		}
-		if (include_data) {
-			String ts_name = TS->get_support_data_filename();
-			String ts_target = "res://" + ts_name;
-			if (!ts_name.is_empty()) {
-				bool export_ok = false;
-				if (FileAccess::exists(ts_target)) { // Include user supplied data file.
-					const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(ts_target);
+		String ts_name = TS->get_support_data_filename();
+		String ts_name_ext = ts_name.get_extension();
+		String ts_target = "res://" + ts_name;
+		if (!ts_name.is_empty()) {
+			bool export_ok = false;
+			if (FileAccess::exists(ts_target)) { // Include user supplied data file.
+				const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(ts_target);
+				if (!ts_data.is_empty()) {
+					add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using user provided text server data, text display in the exported project might be broken if export template was built with different ICU version!"));
+					files[ts_target] = ts_data;
+					export_ok = true;
+				}
+			} else {
+				String current_version = GODOT_VERSION_FULL_CONFIG;
+				String template_path = EditorPaths::get_singleton()->get_export_templates_dir().path_join(current_version);
+				if (p_debug && p_preset->has("custom_template/debug") && p_preset->get("custom_template/debug") != "") {
+					template_path = p_preset->get("custom_template/debug").operator String().get_base_dir();
+				} else if (!p_debug && p_preset->has("custom_template/release") && p_preset->get("custom_template/release") != "") {
+					template_path = p_preset->get("custom_template/release").operator String().get_base_dir();
+				}
+				String source_file_name = include_full_data ? ts_name.get_basename() : ts_name.get_basename() + "_base";
+				if (!ts_name_ext.is_empty()) {
+					source_file_name += "." + ts_name_ext;
+				}
+				String data_file_name = template_path.path_join(source_file_name);
+				if (FileAccess::exists(data_file_name)) {
+					const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(data_file_name);
 					if (!ts_data.is_empty()) {
-						add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using user provided text server data, text display in the exported project might be broken if export template was built with different ICU version!"));
+						print_line("Using text server data from export templates.");
 						files[ts_target] = ts_data;
 						export_ok = true;
 					}
 				} else {
-					String current_version = GODOT_VERSION_FULL_CONFIG;
-					String template_path = EditorPaths::get_singleton()->get_export_templates_dir().path_join(current_version);
-					if (p_debug && p_preset->has("custom_template/debug") && p_preset->get("custom_template/debug") != "") {
-						template_path = p_preset->get("custom_template/debug").operator String().get_base_dir();
-					} else if (!p_debug && p_preset->has("custom_template/release") && p_preset->get("custom_template/release") != "") {
-						template_path = p_preset->get("custom_template/release").operator String().get_base_dir();
-					}
-					String data_file_name = template_path.path_join(ts_name);
-					if (FileAccess::exists(data_file_name)) {
-						const PackedByteArray &ts_data = FileAccess::get_file_as_bytes(data_file_name);
-						if (!ts_data.is_empty()) {
-							print_line("Using text server data from export templates.");
-							files[ts_target] = ts_data;
-							export_ok = true;
-						}
-					} else {
-						const PackedByteArray &ts_data = TS->get_support_data();
-						if (!ts_data.is_empty()) {
-							add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using editor embedded text server data, text display in the exported project might be broken if export template was built with different ICU version!"));
-							files[ts_target] = ts_data;
-							export_ok = true;
-						}
+					const PackedByteArray &ts_data = TS->get_support_data(include_full_data ? "full" : "base");
+					if (!ts_data.is_empty()) {
+						add_message(EXPORT_MESSAGE_INFO, TTR("Export"), TTR("Using editor embedded text server data, text display in the exported project might be broken if export template was built with different ICU version!"));
+						files[ts_target] = ts_data;
+						export_ok = true;
 					}
 				}
-				if (!export_ok) {
-					add_message(EXPORT_MESSAGE_WARNING, TTR("Export"), TTR("Missing text server data, text display in the exported project might be broken!"));
-				}
+			}
+			if (!export_ok) {
+				add_message(EXPORT_MESSAGE_WARNING, TTR("Export"), TTR("Missing text server data, text display in the exported project might be broken!"));
 			}
 		}
 	}

misc/extension_api_validation/4.5-stable/GH-XXXX.txt

@@ -0,0 +1,8 @@
+GH-XXXXX
+---------
+Validate extension JSON: Error: Field 'classes/TextServer/methods/save_support_data/arguments': size changed value in new API, from 1 to 2.
+Validate extension JSON: Error: Field 'classes/TextServerExtension/methods/_save_support_data/arguments': size changed value in new API, from 1 to 2.
+Validate extension JSON: JSON file: Field was added in a way that breaks compatibility 'classes/TextServer/methods/get_support_data': arguments
+Validate extension JSON: JSON file: Field was added in a way that breaks compatibility 'classes/TextServerExtension/methods/_get_support_data': arguments
+
+Optional "config" argument added. Compatibility methods registered.

modules/text_server_adv/SCsub

@@ -462,6 +462,11 @@ if env["builtin_icu4c"]:
             "#thirdparty/icu4c/icudt_godot.dat",
             env.Run(text_server_adv_builders.make_icu_data),
         )
+        icudata_base = env_icu.CommandNoCache(
+            "#thirdparty/icu4c/icudata_base.gen.h",
+            "#thirdparty/icu4c/icudt_godot_base.dat",
+            env.Run(text_server_adv_builders.make_icu_data_base),
+        )
         env_text_server_adv.Prepend(CPPPATH=["#thirdparty/icu4c/"])
     else:
         thirdparty_sources += ["icu_data/icudata_stub.cpp"]

modules/text_server_adv/gdextension_build/SConstruct

@@ -705,6 +705,12 @@ if env["static_icu_data"]:
     env_icu.Command(
         "../../../thirdparty/icu4c/icudata.gen.h", "../../../thirdparty/icu4c/icudt_godot.dat", methods.make_icu_data
     )
+    env_icu.Depends("../../../thirdparty/icu4c/icudata_base.gen.h", "../../../thirdparty/icu4c/icudt_godot_base.dat")
+    env_icu.Command(
+        "../../../thirdparty/icu4c/icudata_base.gen.h",
+        "../../../thirdparty/icu4c/icudt_godot_base.dat",
+        methods.make_icu_data_base,
+    )
     env.Append(CPPDEFINES=["ICU_STATIC_DATA"])
     env.Append(CPPPATH=["../../../thirdparty/icu4c/"])
 else:

modules/text_server_adv/gdextension_build/methods.py

@@ -49,6 +49,30 @@ def make_icu_data(target, source, env):
         g.write("#endif")
 
 
+def make_icu_data_base(target, source, env):
+    dst = target[0].srcnode().abspath
+    with open(dst, "w", encoding="utf-8", newline="\n") as g:
+        g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
+        g.write("/* (C) 2016 and later: Unicode, Inc. and others. */\n")
+        g.write("/* License & terms of use: https://www.unicode.org/copyright.html */\n")
+        g.write("#ifndef _ICU_DATA_DATA_H\n")
+        g.write("#define _ICU_DATA_DATA_H\n")
+        g.write('#include "unicode/utypes.h"\n')
+        g.write('#include "unicode/udata.h"\n')
+        g.write('#include "unicode/uversion.h"\n')
+
+        with open(source[0].srcnode().abspath, "rb") as f:
+            buf = f.read()
+
+        g.write('extern "C" U_EXPORT const size_t U_ICUDATA_SIZE_DATA = ' + str(len(buf)) + ";\n")
+        g.write('extern "C" U_EXPORT const unsigned char U_ICUDATA_ENTRY_POINT_DATA[] = {\n')
+        for i in range(len(buf)):
+            g.write("\t" + str(buf[i]) + ",\n")
+
+        g.write("};\n")
+        g.write("#endif")
+
+
 def write_macos_plist(target, binary_name, identifier, name):
     import os
 

modules/text_server_adv/text_server_adv.cpp

@@ -63,6 +63,7 @@ using namespace godot;
 
 #ifdef ICU_STATIC_DATA
 #include <icudata.gen.h>
+#include <icudata_base.gen.h>
 #endif
 
 // Thirdparty headers.
@@ -483,10 +484,10 @@ String TextServerAdvanced::_get_support_data_filename() const {
 }
 
 String TextServerAdvanced::_get_support_data_info() const {
-	return String("ICU break iteration data (\"icudt_godot.dat\").");
+	return String("ICU common and break iteration data (\"icudt_godot.dat\").");
 }
 
-bool TextServerAdvanced::_save_support_data(const String &p_filename) const {
+bool TextServerAdvanced::_save_support_data(const String &p_filename, const String &p_config) const {
 	_THREAD_SAFE_METHOD_
 #ifdef ICU_STATIC_DATA
 
@@ -498,8 +499,15 @@ bool TextServerAdvanced::_save_support_data(const String &p_filename) const {
 	}
 
 	PackedByteArray icu_data_static;
-	icu_data_static.resize(U_ICUDATA_SIZE);
-	memcpy(icu_data_static.ptrw(), U_ICUDATA_ENTRY_POINT, U_ICUDATA_SIZE);
+	if (p_config == "full") {
+		icu_data_static.resize(U_ICUDATA_SIZE);
+		memcpy(icu_data_static.ptrw(), U_ICUDATA_ENTRY_POINT, U_ICUDATA_SIZE);
+	} else if (p_config == "base") {
+		icu_data_static.resize(U_ICUDATA_SIZE_BASE);
+		memcpy(icu_data_static.ptrw(), U_ICUDATA_ENTRY_POINT_BASE, U_ICUDATA_SIZE_BASE);
+	} else {
+		ERR_FAIL_V_MSG(false, "Invalid data config.");
+	}
 	f->store_buffer(icu_data_static);
 
 	return true;
@@ -508,13 +516,20 @@ bool TextServerAdvanced::_save_support_data(const String &p_filename) const {
 #endif
 }
 
-PackedByteArray TextServerAdvanced::_get_support_data() const {
+PackedByteArray TextServerAdvanced::_get_support_data(const String &p_config) const {
 	_THREAD_SAFE_METHOD_
 #ifdef ICU_STATIC_DATA
 
 	PackedByteArray icu_data_static;
-	icu_data_static.resize(U_ICUDATA_SIZE);
-	memcpy(icu_data_static.ptrw(), U_ICUDATA_ENTRY_POINT, U_ICUDATA_SIZE);
+	if (p_config == "full") {
+		icu_data_static.resize(U_ICUDATA_SIZE);
+		memcpy(icu_data_static.ptrw(), U_ICUDATA_ENTRY_POINT, U_ICUDATA_SIZE);
+	} else if (p_config == "base") {
+		icu_data_static.resize(U_ICUDATA_SIZE_BASE);
+		memcpy(icu_data_static.ptrw(), U_ICUDATA_ENTRY_POINT_BASE, U_ICUDATA_SIZE_BASE);
+	} else {
+		ERR_FAIL_V_MSG(PackedByteArray(), "Invalid data config.");
+	}
 
 	return icu_data_static;
 #else

modules/text_server_adv/text_server_adv.h

@@ -833,8 +833,8 @@ class TextServerAdvanced : public TextServerExtension {
 
 	MODBIND0RC(String, get_support_data_filename);
 	MODBIND0RC(String, get_support_data_info);
-	MODBIND1RC(bool, save_support_data, const String &);
-	MODBIND0RC(PackedByteArray, get_support_data);
+	MODBIND2RC(bool, save_support_data, const String &, const String &);
+	MODBIND1RC(PackedByteArray, get_support_data, const String &);
 	MODBIND1RC(bool, is_locale_using_support_data, const String &);
 
 	MODBIND1RC(bool, is_locale_right_to_left, const String &);

modules/text_server_adv/text_server_adv_builders.py

@@ -19,3 +19,21 @@ def make_icu_data(target, source, env):
 	{methods.format_buffer(buffer, 1)}
 }};
 """)
+
+
+def make_icu_data_base(target, source, env):
+    buffer = methods.get_buffer(str(source[0]))
+    with methods.generated_wrapper(str(target[0])) as file:
+        file.write(f"""\
+/* (C) 2016 and later: Unicode, Inc. and others. */
+/* License & terms of use: https://www.unicode.org/copyright.html */
+
+#include <unicode/utypes.h>
+#include <unicode/udata.h>
+#include <unicode/uversion.h>
+
+extern "C" U_EXPORT const size_t U_ICUDATA_SIZE_BASE = {len(buffer)};
+extern "C" U_EXPORT const unsigned char U_ICUDATA_ENTRY_POINT_BASE[] = {{
+	{methods.format_buffer(buffer, 1)}
+}};
+""")

modules/text_server_fb/text_server_fb.cpp

@@ -175,11 +175,11 @@ bool TextServerFallback::_load_support_data(const String &p_filename) {
 	return false; // No extra data used.
 }
 
-bool TextServerFallback::_save_support_data(const String &p_filename) const {
+bool TextServerFallback::_save_support_data(const String &p_filename, const String &p_config) const {
 	return false; // No extra data used.
 }
 
-PackedByteArray TextServerFallback::_get_support_data() const {
+PackedByteArray TextServerFallback::_get_support_data(const String &p_config) const {
 	return PackedByteArray(); // No extra data used.
 }
 

modules/text_server_fb/text_server_fb.h

@@ -614,8 +614,8 @@ class TextServerFallback : public TextServerExtension {
 
 	MODBIND0RC(String, get_support_data_filename);
 	MODBIND0RC(String, get_support_data_info);
-	MODBIND1RC(bool, save_support_data, const String &);
-	MODBIND0RC(PackedByteArray, get_support_data);
+	MODBIND2RC(bool, save_support_data, const String &, const String &);
+	MODBIND1RC(PackedByteArray, get_support_data, const String &);
 	MODBIND1RC(bool, is_locale_using_support_data, const String &);
 
 	MODBIND1RC(bool, is_locale_right_to_left, const String &);

servers/text/text_server.compat.inc

@@ -30,6 +30,14 @@
 
 #ifndef DISABLE_DEPRECATED
 
+bool TextServer::_save_support_data_bind_compat_XXXX(const String &p_filename) const {
+	return save_support_data(p_filename, "full");
+}
+
+PackedByteArray TextServer::_get_support_data_bind_compat_XXXX() const {
+	return get_support_data("full");
+}
+
 void TextServer::_font_draw_glyph_bind_compat_104872(const RID &p_font, const RID &p_canvas, int64_t p_size, const Vector2 &p_pos, int64_t p_index, const Color &p_color) const {
 	font_draw_glyph(p_font, p_canvas, p_size, p_pos, p_index, p_color, 0.0);
 }
@@ -51,6 +59,8 @@ PackedInt32Array TextServer::_shaped_text_get_word_breaks_bind_compat_90732(cons
 }
 
 void TextServer::_bind_compatibility_methods() {
+	ClassDB::bind_compatibility_method(D_METHOD("save_support_data", "filename"), &TextServer::_save_support_data_bind_compat_XXXX);
+	ClassDB::bind_compatibility_method(D_METHOD("get_support_data"), &TextServer::_get_support_data_bind_compat_XXXX);
 	ClassDB::bind_compatibility_method(D_METHOD("font_draw_glyph", "font_rid", "canvas", "size", "pos", "index", "color"), &TextServer::_font_draw_glyph_bind_compat_104872, DEFVAL(Color(1, 1, 1)));
 	ClassDB::bind_compatibility_method(D_METHOD("font_draw_glyph_outline", "font_rid", "canvas", "size", "outline_size", "pos", "index", "color"), &TextServer::_font_draw_glyph_outline_bind_compat_104872, DEFVAL(Color(1, 1, 1)));
 	ClassDB::bind_compatibility_method(D_METHOD("shaped_text_draw", "shaped", "canvas", "pos", "clip_l", "clip_r", "color"), &TextServer::_shaped_text_draw_bind_compat_104872, DEFVAL(-1), DEFVAL(-1), DEFVAL(Color(1, 1, 1)));

servers/text/text_server.cpp

@@ -203,8 +203,8 @@ void TextServer::_bind_methods() {
 
 	ClassDB::bind_method(D_METHOD("get_support_data_filename"), &TextServer::get_support_data_filename);
 	ClassDB::bind_method(D_METHOD("get_support_data_info"), &TextServer::get_support_data_info);
-	ClassDB::bind_method(D_METHOD("save_support_data", "filename"), &TextServer::save_support_data);
-	ClassDB::bind_method(D_METHOD("get_support_data"), &TextServer::get_support_data);
+	ClassDB::bind_method(D_METHOD("save_support_data", "filename", "config"), &TextServer::save_support_data, DEFVAL(String("full")));
+	ClassDB::bind_method(D_METHOD("get_support_data", "config"), &TextServer::get_support_data, DEFVAL(String("full")));
 	ClassDB::bind_method(D_METHOD("is_locale_using_support_data", "locale"), &TextServer::is_locale_using_support_data);
 
 	ClassDB::bind_method(D_METHOD("is_locale_right_to_left", "locale"), &TextServer::is_locale_right_to_left);
@@ -2386,7 +2386,7 @@ TextServer::TextServer() {
 	GLOBAL_DEF_RST("gui/theme/default_font_generate_mipmaps", false);
 
 	GLOBAL_DEF(PropertyInfo(Variant::INT, "gui/theme/lcd_subpixel_layout", PROPERTY_HINT_ENUM, "Disabled,Horizontal RGB,Horizontal BGR,Vertical RGB,Vertical BGR"), 1);
-	GLOBAL_DEF_BASIC("internationalization/locale/include_text_server_data", false);
+	GLOBAL_DEF_BASIC(PropertyInfo(Variant::INT, "internationalization/locale/include_text_server_break_iterator_data", PROPERTY_HINT_ENUM, "Always,Auto,Never"), 1);
 	GLOBAL_DEF_BASIC(PropertyInfo(Variant::INT, "internationalization/locale/line_breaking_strictness", PROPERTY_HINT_ENUM, "Auto,Loose,Normal,Strict"), 0);
 
 	_init_diacritics_map();