feat: enhance emoji sequence handling by adding support for zwj sequences and improving data fetching logic

Author: luxass

src/adapter/base.ts

@@ -15,6 +15,7 @@ export default defineMojiAdapter({
   range: "*",
   groups: async ({ version, force }) => {
     if (version === "1.0" || version === "2.0" || version === "3.0") {
+      console.warn(`version ${version} does not have group data`);
       return [];
     }
 

src/adapter/index.ts

@@ -36,7 +36,7 @@ export interface BaseAdapterContext {
 }
 
 export type GroupFn = (ctx: BaseAdapterContext) => Promise<EmojiGroup[]>;
-export type SequenceFn = (ctx: BaseAdapterContext) => Promise<EmojiSequence[]>;
+export type SequenceFn = (ctx: BaseAdapterContext) => Promise<{ zwj: EmojiSequence[]; sequences: EmojiSequence[] }>;
 
 export const ADAPTERS = new Map<string, MojiAdapter>();
 

src/adapter/v16.ts

@@ -10,48 +10,62 @@ export default defineMojiAdapter({
   range: ">=16.0.0 <17.0.0",
   extend: "base",
   sequences: async (ctx) => {
-    const sequences = await fetchCache(`https://unicode.org/Public/emoji/${ctx.version}/emoji-sequences.txt`, {
-      cacheKey: `v${ctx.version}/sequences.json`,
-      parser(data) {
-        const lines = data.split("\n");
+    const [sequences, zwj] = await Promise.all([
+      {
+        cacheKey: `v${ctx.version}/sequences.json`,
+        url: `https://unicode.org/Public/emoji/${ctx.version}/emoji-sequences.txt`,
+      },
+      {
+        cacheKey: `v${ctx.version}/zwj-sequences.json`,
+        url: `https://unicode.org/Public/emoji/${ctx.version}/emoji-zwj-sequences.txt`,
+      },
+    ].map(async ({ cacheKey, url }) => {
+      return await fetchCache(url, {
+        cacheKey,
+        parser(data) {
+          const lines = data.split("\n");
 
-        const sequences: EmojiSequence[] = [];
+          const sequences: EmojiSequence[] = [];
 
-        for (let line of lines) {
+          for (let line of lines) {
           // skip empty line & comments
-          if (line.trim() === "" || line.startsWith("#")) {
-            continue;
-          }
+            if (line.trim() === "" || line.startsWith("#")) {
+              continue;
+            }
 
-          // remove line comment
-          const commentIndex = line.indexOf("#");
-          if (commentIndex !== -1) {
-            line = line.slice(0, commentIndex).trim();
-          }
+            // remove line comment
+            const commentIndex = line.indexOf("#");
+            if (commentIndex !== -1) {
+              line = line.slice(0, commentIndex).trim();
+            }
 
-          const [hex, property, description] = line.split(";").map((col) => col.trim()).slice(0, 4);
+            const [hex, property, description] = line.split(";").map((col) => col.trim()).slice(0, 4);
 
-          if (hex == null || property == null || description == null) {
-            throw new Error(`invalid line: ${line}`);
-          }
+            if (hex == null || property == null || description == null) {
+              throw new Error(`invalid line: ${line}`);
+            }
 
-          const expandedHex = expandHexRange(hex);
+            const expandedHex = expandHexRange(hex);
 
-          for (const hex of expandedHex) {
-            sequences.push({
-              hex: hex.replace(/\s+/g, "-"),
-              property,
-              description,
-              gender: hex === FEMALE_SIGN ? "female" : hex === MALE_SIGN ? "male" : null,
-            });
+            for (const hex of expandedHex) {
+              sequences.push({
+                hex: hex.replace(/\s+/g, "-"),
+                property,
+                description,
+                gender: hex.includes(FEMALE_SIGN) ? "female" : hex.includes(MALE_SIGN) ? "male" : null,
+              });
+            }
           }
-        }
 
-        return sequences;
-      },
-      bypassCache: ctx.force,
-    });
+          return sequences;
+        },
+        bypassCache: ctx.force,
+      });
+    }));
 
-    return sequences;
+    return {
+      sequences: sequences || [],
+      zwj: zwj || [],
+    };
   },
 });

src/cli.ts

@@ -53,12 +53,17 @@ cli.command(
         throw new Error(`no adapter found for version ${version}`);
       }
 
-      const groups = await adapter.sequences!({ version, force });
+      const { sequences, zwj } = await adapter.sequences!({ version, force });
 
       await fs.ensureDir(`./data/v${version}`);
+      await fs.writeFile(
+        `./data/v${version}/zwj-sequences.json`,
+        JSON.stringify(zwj, null, 2),
+        "utf-8",
+      );
       return fs.writeFile(
         `./data/v${version}/sequences.json`,
-        JSON.stringify(groups, null, 2),
+        JSON.stringify(sequences, null, 2),
         "utf-8",
       );
     });