CLDR-json data extraction script

data/de/lexicon.txt

@@ -0,0 +1,19 @@
+Januar;N;MASC;NOM;SG;INAN
+Februar;N;MASC;NOM;SG;INAN
+März;N;MASC;NOM;SG;INAN
+April;N;MASC;NOM;SG;INAN
+Mai;N;MASC;NOM;SG;INAN
+Juni;N;MASC;NOM;SG;INAN
+Juli;N;MASC;NOM;SG;INAN
+August;N;MASC;NOM;SG;INAN
+September;N;MASC;NOM;SG;INAN
+Oktober;N;MASC;NOM;SG;INAN
+November;N;MASC;NOM;SG;INAN
+Dezember;N;MASC;NOM;SG;INAN
+Sonntag;N;MASC;NOM;SG;INAN
+Montag;N;MASC;NOM;SG;INAN
+Dienstag;N;MASC;NOM;SG;INAN
+Mittwoch;N;MASC;NOM;SG;INAN
+Donnerstag;N;MASC;NOM;SG;INAN
+Freitag;N;MASC;NOM;SG;INAN
+Samstag;N;MASC;NOM;SG;INAN

data/en/lexicon.txt

@@ -0,0 +1,19 @@
+January;N;MASC;NOM;SG;INAN
+February;N;MASC;NOM;SG;INAN
+March;N;MASC;NOM;SG;INAN
+April;N;MASC;NOM;SG;INAN
+May;N;MASC;NOM;SG;INAN
+June;N;MASC;NOM;SG;INAN
+July;N;MASC;NOM;SG;INAN
+August;N;MASC;NOM;SG;INAN
+September;N;MASC;NOM;SG;INAN
+October;N;MASC;NOM;SG;INAN
+November;N;MASC;NOM;SG;INAN
+December;N;MASC;NOM;SG;INAN
+Sunday;N;MASC;NOM;SG;INAN
+Monday;N;MASC;NOM;SG;INAN
+Tuesday;N;MASC;NOM;SG;INAN
+Wednesday;N;MASC;NOM;SG;INAN
+Thursday;N;MASC;NOM;SG;INAN
+Friday;N;MASC;NOM;SG;INAN
+Saturday;N;MASC;NOM;SG;INAN

data/es/lexicon.txt

@@ -0,0 +1,19 @@
+enero;N;MASC;NOM;SG;INAN
+febrero;N;MASC;NOM;SG;INAN
+marzo;N;MASC;NOM;SG;INAN
+abril;N;MASC;NOM;SG;INAN
+mayo;N;MASC;NOM;SG;INAN
+junio;N;MASC;NOM;SG;INAN
+julio;N;MASC;NOM;SG;INAN
+agosto;N;MASC;NOM;SG;INAN
+septiembre;N;MASC;NOM;SG;INAN
+octubre;N;MASC;NOM;SG;INAN
+noviembre;N;MASC;NOM;SG;INAN
+diciembre;N;MASC;NOM;SG;INAN
+domingo;N;MASC;NOM;SG;INAN
+lunes;N;MASC;NOM;SG;INAN
+martes;N;MASC;NOM;SG;INAN
+miércoles;N;MASC;NOM;SG;INAN
+jueves;N;MASC;NOM;SG;INAN
+viernes;N;MASC;NOM;SG;INAN
+sábado;N;MASC;NOM;SG;INAN

data/fr/lexicon.txt

@@ -0,0 +1,19 @@
+janvier;N;MASC;NOM;SG;INAN
+février;N;MASC;NOM;SG;INAN
+mars;N;MASC;NOM;SG;INAN
+avril;N;MASC;NOM;SG;INAN
+mai;N;MASC;NOM;SG;INAN
+juin;N;MASC;NOM;SG;INAN
+juillet;N;MASC;NOM;SG;INAN
+août;N;MASC;NOM;SG;INAN
+septembre;N;MASC;NOM;SG;INAN
+octobre;N;MASC;NOM;SG;INAN
+novembre;N;MASC;NOM;SG;INAN
+décembre;N;MASC;NOM;SG;INAN
+dimanche;N;MASC;NOM;SG;INAN
+lundi;N;MASC;NOM;SG;INAN
+mardi;N;MASC;NOM;SG;INAN
+mercredi;N;MASC;NOM;SG;INAN
+jeudi;N;MASC;NOM;SG;INAN
+vendredi;N;MASC;NOM;SG;INAN
+samedi;N;MASC;NOM;SG;INAN

data/sr/lexicon.txt

@@ -0,0 +1,19 @@
+јануар;N;MASC;NOM;SG;INAN
+фебруар;N;MASC;NOM;SG;INAN
+март;N;MASC;NOM;SG;INAN
+април;N;MASC;NOM;SG;INAN
+мај;N;MASC;NOM;SG;INAN
+јун;N;MASC;NOM;SG;INAN
+јул;N;MASC;NOM;SG;INAN
+август;N;MASC;NOM;SG;INAN
+септембар;N;MASC;NOM;SG;INAN
+октобар;N;MASC;NOM;SG;INAN
+новембар;N;MASC;NOM;SG;INAN
+децембар;N;MASC;NOM;SG;INAN
+недеља;N;FEM;NOM;SG;INAN
+понедељак;N;MASC;NOM;SG;INAN
+уторак;N;MASC;NOM;SG;INAN
+среда;N;FEM;NOM;SG;INAN
+четвртак;N;MASC;NOM;SG;INAN
+петак;N;MASC;NOM;SG;INAN
+субота;N;FEM;NOM;SG;INAN

data/tools/extract_cldr_data.py

@@ -0,0 +1,85 @@
+"""
+Extracts data from CLDR-JSON repository, e.g. nouns like month or day names.
+Script either creates a new inflection file, or appends data to existing one.
+The nomenclature is taken from https://unimorph.github.io/doc/unimorph-schema.pdf (see Appendix)
+
+Part of Speech;Gender;Case;Number;Animacy
+
+Run script from data folder.
+
+Before running the script clone cldr-json repository:
+
+gh repo clone unicode-org/cldr-json
+
+and install jsonpath-ng package:
+
+pip install jsonpath-ng
+"""
+
+import argparse
+import json
+import os
+
+from jsonpath_ng import jsonpath, parse
+
+
+def load_json(filename):
+    """Loads JSON data from the specified file.
+
+    Args:
+        filename: The name of the JSON file.
+
+    Returns:
+        The parsed JSON data.
+    """
+
+    try:
+        with open(filename, 'r', encoding='utf-8') as file:
+            return json.load(file)
+    except FileNotFoundError:
+        print(f"Error: File '{filename}' not found.")
+        return None
+
+
+def write_to_lexicon(output_file, language, json_data):
+    """Extracts specified data from cldr-json file
+       and writes it to the lexicon file.
+
+    Args:
+        output_file: name of the lexicon.
+        language: cldr-json file language.
+        json_data: parsed cldr-json data.
+    """
+    MONTH_NAMES_EXPRESSION = parse('main..dates.calendars.gregorian.months.format.wide.*')
+    DAY_NAMES_EXPRESSION = parse('main..dates.calendars.gregorian.days.format.wide.*')
+    EXPRESSIONS = [MONTH_NAMES_EXPRESSION, DAY_NAMES_EXPRESSION]
+
+    results = []
+    for expression in EXPRESSIONS:
+        match = expression.find(json_data)
+        for m in match:
+            results.append(m.value + ';N;MASC;NOM;SG;INAN\n')
+
+    full_filename = os.path.join(language, output_file)
+    try:
+        os.makedirs(os.path.dirname(full_filename), exist_ok=True)
+        with open(full_filename, 'a', encoding='utf-8') as file:
+            file.writelines(results)
+    except FileNotFoundError:
+        print(f"Error: file '{output_file}' can't be created.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Load and process CLDR-JSON files for given languages.')
+    parser.add_argument('--cldr_root', help='The path to CLDR-JSON data.', default='../../cldr-json/cldr-json/cldr-dates-full/main')
+    parser.add_argument('--input_file', help='Data file to read from, e.g. ca-gregorian.json.', default='ca-gregorian.json')
+    parser.add_argument('--output_file', help='Data file to create/append to, e.g. lexicon.txt.', default='lexicon.txt')
+    parser.add_argument('--language_list', nargs='+', default=['sr', 'en', 'de', 'es', 'fr'])
+    args = parser.parse_args()
+
+    for language in args.language_list:
+        full_filename = os.path.join(args.cldr_root, language, args.input_file)
+        data = load_json(full_filename)
+
+        if data:
+            write_to_lexicon(args.output_file, language, data)