feat: quotation marks localization (@anakojm)

docs/LANGUAGES.md

@@ -20,6 +20,10 @@ The contents of the file should be as follows:
 ```json
 {
   "name": string,
+  "quotationMarks": {
+    "primary": tuple[string, string],
+    "secondary": tuple[string, string]
+  },
   "rightToLeft": boolean,
   "joiningScript": boolean,
   "orderedByFrequency": boolean,
@@ -29,6 +33,13 @@ The contents of the file should be as follows:
 ```
 
 It is recommended that you familiarize yourselves with JSON before adding a language. For the `name` field, put the name of your language.
+`quotationMarks` indicates the quotation marks used for that language, at depth 1 (`primary`) and 2 (`secondary`) respectively, and for the left quotation mark and the right one respectively. For instance, American English uses `“”` at depth 1 and `‘’` at depth 2. In that case it would look like:
+```json
+ "quotationMarks": {
+    "primary": ["“", "”"],
+    "secondary": ["‘", "’"]
+  },
+```
 `rightToLeft` indicates how the language is written. If it is written right to left then put `true`, otherwise put `false`.
 `joiningScript` indicates whether the language requires joining letters to render correctly. Set it to `true` if characters must join with surrounding characters or if their shapes change based on position in a word (initial, medial, final, or isolated), or if they use connecting marks (matras/vowel signs) that reshape the base characters. Otherwise, set it to `false.`
 For `bcp47` put your languages [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag).

frontend/__tests__/test/british-english.spec.ts

@@ -40,23 +40,23 @@ describe("british-english", () => {
     });
 
     it("should convert double quotes to single quotes", async () => {
-      await expect(replace('"hello"', "")).resolves.toEqual("'hello'");
-      await expect(replace('"test"', "")).resolves.toEqual("'test'");
-      await expect(replace('"Hello World"', "")).resolves.toEqual(
-        "'Hello World'",
+      await expect(replace("“hello”", "")).resolves.toEqual("‘hello’");
+      await expect(replace("“test”", "")).resolves.toEqual("‘test’");
+      await expect(replace("“Hello World”", "")).resolves.toEqual(
+        "‘Hello World’",
       );
     });
 
     it("should convert double quotes and replace words", async () => {
-      await expect(replace('"color"', "")).resolves.toEqual("'colour'");
-      await expect(replace('"math"', "")).resolves.toEqual("'maths'");
-      await expect(replace('"Color"', "")).resolves.toEqual("'Colour'");
+      await expect(replace("“color”", "")).resolves.toEqual("‘colour’");
+      await expect(replace("“math”", "")).resolves.toEqual("‘maths’");
+      await expect(replace("“Color”", "")).resolves.toEqual("‘Colour’");
     });
 
     it("should handle multiple double quotes in a word", async () => {
       await expect(
-        replace('He said "hello" and "goodbye"', ""),
-      ).resolves.toEqual("He said 'hello' and 'goodbye'");
+        replace("He said “hello” and “goodbye”", ""),
+      ).resolves.toEqual("He said ‘hello’ and ‘goodbye’");
     });
 
     it("should not affect words without double quotes", async () => {

frontend/src/ts/test/british-english.ts

@@ -7,9 +7,16 @@ export async function replace(
   previousWord: string,
 ): Promise<string> {
   // Convert American-style double quotes to British-style single quotes
-  if (word.includes('"')) {
-    word = word.replace(/"/g, "'");
-  }
+  const us2ukQuotes = {
+    "“": "‘",
+    "”": "’",
+    "‘": "“",
+    "’": "”",
+  };
+  word = word.replace(
+    /[“”‘’]/g,
+    (char) => us2ukQuotes[char as keyof typeof us2ukQuotes],
+  );
 
   if (word.includes("-")) {
     //this handles hyphenated words (for example "cream-colored") to make sure

frontend/src/ts/test/words-generator.ts

@@ -42,10 +42,11 @@ export async function punctuateWord(
   currentWord: string,
   index: number,
   maxindex: number,
+  language: LanguageObject,
 ): Promise<string> {
   let word = currentWord;
 
-  const currentLanguage = Config.language.split("_")[0];
+  const currentLanguage = language.name.split("_")[0];
 
   const lastChar = Strings.getLastChar(previousWord);
 
@@ -140,22 +141,18 @@ export async function punctuateWord(
         }
       }
     }
-  } else if (
-    random() < 0.01 &&
-    lastChar !== "," &&
-    lastChar !== "." &&
-    currentLanguage !== "russian"
-  ) {
-    word = `"${word}"`;
-  } else if (
-    random() < 0.011 &&
-    lastChar !== "," &&
-    lastChar !== "." &&
-    currentLanguage !== "russian" &&
-    currentLanguage !== "ukrainian" &&
-    currentLanguage !== "slovak"
-  ) {
-    word = `'${word}'`;
+  } else if (random() < 0.01 && lastChar !== "," && lastChar !== ".") {
+    if (language?.quotationMarks !== undefined) {
+      word = `${language.quotationMarks.primary[0]}${word}${language.quotationMarks.primary[1]}`;
+    } else {
+      word = `"${word}"`;
+    }
+  } else if (random() < 0.011 && lastChar !== "," && lastChar !== ".") {
+    if (language?.quotationMarks !== undefined) {
+      word = `${language.quotationMarks.secondary[0]}${word}${language.quotationMarks.secondary[1]}`;
+    } else {
+      word = `'${word}'`;
+    }
   } else if (random() < 0.012 && lastChar !== "," && lastChar !== ".") {
     if (currentLanguage === "code") {
       const r = random();
@@ -941,6 +938,7 @@ export async function getNextWord(
       randomWord,
       wordIndex,
       wordsBound,
+      currentLanguage,
     );
   }
 

frontend/src/ts/utils/strings.ts

@@ -294,8 +294,23 @@ export function isWordRightToLeft(
 }
 
 export const CHAR_EQUIVALENCE_SETS = [
-  new Set(["’", "‘", "'", "ʼ", "׳", "ʻ", "᾽", "᾽"]),
-  new Set([`"`, "”", "“", "„"]),
+  new Set([
+    "’",
+    "‘",
+    "'",
+    "ʼ",
+    "׳",
+    "ʻ",
+    "᾽",
+    "᾽",
+    "‛",
+    "‚",
+    "‹",
+    "›",
+    "『",
+    "』",
+  ]),
+  new Set([`"`, "”", "“", "‟", "„", "⹂", "«", "»", "《", "》", "「", "」"]),
   new Set(["–", "—", "-", "‐"]),
   new Set([",", "‚"]),
 ];

frontend/static/languages/afrikaans_10k.json

@@ -1,5 +1,9 @@
 {
   "name": "afrikaans_10k",
+  "quotationMarks": {
+    "primary": ["“", "”"],
+    "secondary": ["‘", "’"]
+  },
   "words": [
     "aan",
     "aanbetref",

frontend/static/languages/afrikaans_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "afrikaans_1k",
+  "quotationMarks": {
+    "primary": ["“", "”"],
+    "secondary": ["‘", "’"]
+  },
   "noLazyMode": true,
   "words": [
     "sterker",

frontend/static/languages/albanian.json

@@ -1,5 +1,9 @@
 {
   "name": "albanian",
+  "quotationMarks": {
+    "primary": ["„", "”"],
+    "secondary": ["‘", "’"]
+  },
   "words": [
     "të",
     "e",

frontend/static/languages/albanian_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "albanian_1k",
+  "quotationMarks": {
+    "primary": ["„", "”"],
+    "secondary": ["‘", "’"]
+  },
   "words": [
     "I",
     "e tij",

frontend/static/languages/amharic.json

@@ -1,5 +1,9 @@
 {
   "name": "amharic",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["‹", "›"]
+  },
   "bcp47": "am-ET",
   "words": [
     "እግዚአብሔር",

frontend/static/languages/amharic_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "amharic_1k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["‹", "›"]
+  },
   "bcp47": "am-ET",
   "words": [
     "መለየት",

frontend/static/languages/amharic_5k.json

@@ -1,5 +1,9 @@
 {
   "name": "amharic_5k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["‹", "›"]
+  },
   "bcp47": "am-ET",
   "words": [
     "ሙዚቀኝነት",

frontend/static/languages/arabic.json

@@ -1,5 +1,9 @@
 {
   "name": "arabic",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "rightToLeft": true,
   "joiningScript": true,
   "bcp47": "ar-SA",

frontend/static/languages/arabic_10k.json

@@ -1,5 +1,9 @@
 {
   "name": "arabic_10k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "rightToLeft": true,
   "joiningScript": true,
   "bcp47": "ar-SA",

frontend/static/languages/arabic_egypt.json

@@ -1,5 +1,9 @@
 {
   "name": "arabic_egypt",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "rightToLeft": true,
   "joiningScript": true,
   "bcp47": "ar-EG",

frontend/static/languages/arabic_egypt_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "arabic_egypt_1k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "rightToLeft": true,
   "joiningScript": true,
   "bcp47": "ar-EG",

frontend/static/languages/arabic_morocco.json

@@ -1,5 +1,9 @@
 {
   "name": "arabic_morocco",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "rightToLeft": true,
   "joiningScript": true,
   "orderedByFrequency": false,

frontend/static/languages/armenian.json

@@ -1,5 +1,9 @@
 {
   "name": "armenian",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "noLazyMode": true,
   "orderedByFrequency": false,
   "words": [

frontend/static/languages/armenian_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "armenian_1k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "noLazyMode": true,
   "orderedByFrequency": false,
   "words": [

frontend/static/languages/armenian_western.json

@@ -1,5 +1,9 @@
 {
   "name": "armenian_western",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "bcp47": "hyw",
   "words": [
     "կանանց",

frontend/static/languages/armenian_western_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "armenian_western_1k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["", ""]
+  },
   "bcp47": "hyw",
   "words": [
     "թարգմանուած",

frontend/static/languages/azerbaijani.json

@@ -1,5 +1,9 @@
 {
   "name": "azerbaijani",
+  "quotationMarks": {
+    "primary": ["“", "”"],
+    "secondary": ["\"", "\""]
+  },
   "bcp47": "az-AZ",
   "words": [
     "dərs",

frontend/static/languages/azerbaijani_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "azerbaijani_1k",
+  "quotationMarks": {
+    "primary": ["“", "”"],
+    "secondary": ["\"", "\""]
+  },
   "bcp47": "az-AZ",
   "words": [
     "dərs",

frontend/static/languages/belarusian.json

@@ -1,5 +1,9 @@
 {
   "name": "belarusian",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["„", "“"]
+  },
   "noLazyMode": true,
   "bcp47": "be-BY",
   "words": [

frontend/static/languages/belarusian_100k.json

@@ -1,5 +1,9 @@
 {
   "name": "belarusian_100k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["„", "“"]
+  },
   "bcp47": "be-BY",
   "words": [
     "а",

frontend/static/languages/belarusian_10k.json

@@ -1,5 +1,9 @@
 {
   "name": "belarusian_10k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["„", "“"]
+  },
   "bcp47": "be-BY",
   "words": [
     "а",

frontend/static/languages/belarusian_1k.json

@@ -1,5 +1,9 @@
 {
   "name": "belarusian_1k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["„", "“"]
+  },
   "noLazyMode": true,
   "bcp47": "be-BY",
   "words": [

frontend/static/languages/belarusian_25k.json

@@ -1,5 +1,9 @@
 {
   "name": "belarusian_25k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["„", "“"]
+  },
   "bcp47": "be-BY",
   "words": [
     "а",

frontend/static/languages/belarusian_50k.json

@@ -1,5 +1,9 @@
 {
   "name": "belarusian_50k",
+  "quotationMarks": {
+    "primary": ["«", "»"],
+    "secondary": ["„", "“"]
+  },
   "bcp47": "be-BY",
   "words": [
     "а",