Generate Rust code and FTL code for a bundle of messages

Cargo.lock

@@ -658,6 +658,15 @@ version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
 
+[[package]]
+name = "convert_case"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "cookie"
 version = "0.17.0"
@@ -2701,6 +2710,13 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "kifu-l10n"
+version = "0.1.0"
+dependencies = [
+ "messages-codegen",
+]
+
 [[package]]
 name = "kv-log-macro"
 version = "1.0.7"
@@ -2718,10 +2734,12 @@ dependencies = [
  "chrono-tz",
  "fixed_decimal",
  "fluent",
+ "fluent-ergonomics",
  "icu",
  "icu_locid",
  "icu_provider",
  "sys-locale",
+ "unic-langid",
 ]
 
 [[package]]
@@ -2899,6 +2917,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "messages-codegen"
+version = "0.1.0"
+dependencies = [
+ "convert_case",
+ "quote",
+ "serde 1.0.193",
+ "serde_yaml",
+ "syn 2.0.48",
+]
+
 [[package]]
 name = "mime"
 version = "0.2.6"
@@ -4154,6 +4183,19 @@ dependencies = [
  "serde 1.0.193",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a15e0ef66bf939a7c890a0bf6d5a733c70202225f9888a89ed5c62298b019129"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde 1.0.193",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sgf"
 version = "0.1.0"
@@ -5081,6 +5123,12 @@ dependencies = [
  "traitobject",
 ]
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b"
+
 [[package]]
 name = "url"
 version = "1.7.2"
@@ -5562,6 +5610,10 @@ version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dad7bb64b8ef9c0aa27b6da38b452b0ee9fd82beaf276a87dd796fb55cbae14e"
 
+[[package]]
+name = "xdg-test"
+version = "0.1.0"
+
 [[package]]
 name = "yoke"
 version = "0.7.3"

Cargo.toml

@@ -20,8 +20,10 @@ members = [
     "ifc",
     "kifu/core",
     "kifu/gtk",
+    "kifu/l10n",
     "l10n",
     "memorycache",
+    "messages-codegen",
     "nom-training",
     "result-extended",
     "screenplay",
@@ -29,4 +31,5 @@ members = [
     "timezone-testing",
     "tree",
     "visions/server",
+    "xdg-test",
 ]

flake.nix

@@ -71,6 +71,7 @@
               dashboard = attrs: { nativeBuildInputs = gtkNativeInputs; };
               fitnesstrax = import ./fitnesstrax/app/override.nix { gtkNativeInputs = gtkNativeInputs; };
               kifu-gtk = import ./kifu/gtk/override.nix { gtkNativeInputs = gtkNativeInputs; };
+              xdg-test = import ./xdg-test/override.nix { wrapGAppsHook4 = pkgs.wrapGAppsHook4; };
             };
           };
 
@@ -85,6 +86,7 @@
           file-service = cargo_nix.workspaceMembers.file-service.build;
           fitnesstrax = cargo_nix.workspaceMembers.fitnesstrax.build;
           kifu-gtk = cargo_nix.workspaceMembers.kifu-gtk.build;
+          xdg-test = cargo_nix.workspaceMembers.xdg-test.build;
 
           all = pkgs.symlinkJoin {
             name = "all";
@@ -94,6 +96,7 @@
                 file-service
                 fitnesstrax
                 kifu-gtk
+                xdg-test
             ];
           };
 

kifu/gtk/resources/en-US.ftl

@@ -0,0 +1,5 @@
+welcome = Welcome
+games-in-database = {count ->
+    [one] There is one game in the database
+    [other] There are {count} games in the database
+}

kifu/gtk/resources/eo.ftl

@@ -0,0 +1,5 @@
+welcome = Bonvenon
+games-in-database = {count ->
+    [one] Estas unu ludon en la datumbazo.
+    [other] Estas {count} ludojn en la datumbazo.
+}

kifu/gtk/src/messages.rs

@@ -1,21 +1,5 @@
 use l10n::{FluentArgs, FluentValue, Message};
 
-const ENGLISH_MESSAGES: &'static str = "
-welcome = Welcome
-games-in-database = {count ->
-    [one] There is one game in the database
-    [other] There are {count} games in the database
-}
-";
-
-const ESPERANTO_MESSAGES: &'static str = "
-welcome = Bonvenon
-games-in-database = {count ->
-    [one] Estas unu ludon en la datumbazo.
-    [other] Estas {count} ludojn en la datumbazo.
-}
-";
-
 enum Phrase {
     Welcome,
     GamesInDatabase(i32),

l10n/Cargo.toml

@@ -10,7 +10,10 @@ chrono              = { version = "0.4" }
 chrono-tz           = { version = "0.8" }
 fixed_decimal       = { version = "0.5.5", features = [ "ryu" ] }
 fluent              = { version = "0.16" }
+fluent-ergonomics   = { path = "../fluent-ergonomics" }
 icu                 = { version = "1" }
 icu_locid           = { version = "1" }
 icu_provider        = { version = "1" }
 sys-locale          = { version = "0.3" }
+unic-langid         = { version = "*" }
+

l10n/src/lib.rs

@@ -1,19 +1,17 @@
-use std::{ops::Deref, path::Path};
 use chrono::{Datelike, NaiveDate, Timelike};
-use chrono_tz::{Tz};
+use chrono_tz::Tz;
-use fixed_decimal::{FixedDecimal};
+use fixed_decimal::FixedDecimal;
-use icu::{
+use fluent::{FluentBundle, FluentResource};
-    datetime::options::length,
+use fluent_ergonomics::FluentErgo;
-    decimal::{FixedDecimalFormatter},
+use icu::{datetime::options::length, decimal::FixedDecimalFormatter, locid::Locale};
-    locid::Locale,
-};
 use icu_provider::DataLocale;
+use std::{collections::HashMap, ops::Deref, path::Path};
 use sys_locale::get_locale;
 
 // Re-exports. I'm doing these so that clients of this library don't have to go tracking down
 // additional structures
-pub use fluent::{FluentValue, FluentArgs};
 pub use fixed_decimal::FloatPrecision;
+pub use fluent::{FluentArgs, FluentValue};
 
 #[derive(Debug)]
 pub enum NonEmptyListError {
@@ -22,12 +20,12 @@ pub enum NonEmptyListError {
 
 pub struct NonEmptyList<A>(Vec<A>);
 
-impl <A> NonEmptyList<A> {
+impl<A> NonEmptyList<A> {
     fn new(elem: A) -> Self {
         Self(vec![elem])
     }
 
-    fn from_iter(iter: impl IntoIterator<Item = A>) -> Result<NonEmptyList<A> ,NonEmptyListError> {
+    fn from_iter(iter: impl IntoIterator<Item = A>) -> Result<NonEmptyList<A>, NonEmptyListError> {
         let lst = iter.into_iter().collect::<Vec<A>>();
         if lst.len() > 0 {
             Ok(NonEmptyList(lst))
@@ -39,7 +37,6 @@ impl <A> NonEmptyList<A> {
     fn first(&self) -> &A {
         &self.0[0]
     }
-
 }
 
 impl<A> Deref for NonEmptyList<A> {
@@ -50,7 +47,7 @@ impl<A> Deref for NonEmptyList<A> {
 }
 
 pub enum L10NError {
-    UnparsableLocale
+    UnparsableLocale,
 }
 
 impl From<icu::locid::Error> for L10NError {
@@ -68,28 +65,60 @@ impl From<icu::locid::Error> for L10NError {
 // Nobody wants to generate all of that code, though I have done so in the past, and manually
 // generating that code could be useful for illustration. I think I'm going to want to do code
 // generation from the source strings file, and then compile the enumeration into the code.
+// However, I have not found a mechanism in Fluent to identify all of the placeholders within a
+// message, so I'm not even sure that I can automate this code generation.
 pub trait Message {
     fn msgid(&self) -> &str;
     fn args(&self) -> Option<FluentArgs>;
 }
 
 pub struct L10N {
+    messages_root: std::path::PathBuf,
+    messages: FluentErgo,
+
     locales: NonEmptyList<Locale>,
     zone: chrono_tz::Tz,
 }
 
-impl Default for L10N {
+impl L10N {
-    fn default() -> Self {
+    fn new(messages_root: std::path::PathBuf) -> Self {
         let english = "en-US".parse::<Locale>().unwrap();
-        let sys_locale = get_locale().and_then(|locale_str| locale_str.parse::<Locale>().ok()).unwrap_or(english);
+        let sys_locale = get_locale()
+            .and_then(|locale_str| locale_str.parse::<Locale>().ok())
+            .unwrap_or(english.clone());
         let locales = NonEmptyList::new(sys_locale.clone());
         let zone = chrono_tz::UTC;
-        Self { locales, zone }
-    }
-}
 
-impl L10N {
+        /*
-    pub fn load_messages_from_file(&mut self, locale: String, path: &Path) -> Result<(), L10NError>{
+        let mut source_message_path = messages_root.clone();
+        source_message_path.push("en-US.ftl");
+        let english_phrases = FluentResource::try_new
+        */
+
+        let messages = {
+            let mut english_messages = messages_root.clone();
+            english_messages.push("en-US.ftl");
+
+            let langid: unic_langid::LanguageIdentifier = english.to_string().parse().unwrap();
+            let mut messages = FluentErgo::new(&[langid.clone()]);
+            let _ = messages.add_from_file(langid, &english_messages);
+
+            messages
+        };
+
+        Self {
+            messages_root,
+            messages,
+            locales,
+            zone,
+        }
+    }
+
+    pub fn load_messages_from_file(
+        &mut self,
+        locale: String,
+        path: &Path,
+    ) -> Result<(), L10NError> {
         unimplemented!()
     }
 
@@ -107,7 +136,10 @@ impl L10N {
     // changed any time the list of locales gets changed. Also, the system can just run through the
     // entire list of fallbacks.
     pub fn set_locales(&mut self, locales: NonEmptyList<&str>) -> Result<(), L10NError> {
-        let locales = locales.iter().map(|locale| Locale::try_from_bytes(locale.as_bytes())).collect::<Result<Vec<Locale>, icu::locid::Error>>()?;
+        let locales = locales
+            .iter()
+            .map(|locale| Locale::try_from_bytes(locale.as_bytes()))
+            .collect::<Result<Vec<Locale>, icu::locid::Error>>()?;
         self.locales = NonEmptyList(locales);
         Ok(())
     }
@@ -125,8 +157,8 @@ impl L10N {
     // parameters. In an ideal world, neither of these can be incorrect. Messages are all checked
     // at compile time, as are their parameters. That implies an enumeration, with one element per
     // message, and with each element knowing its parameters.
-    pub fn format_message(&self) -> String {
+    pub fn messages(&self) -> FluentErgo {
-        unimplemented!()
+        self.messages.clone()
     }
 
     pub fn format_date_time_utc(
@@ -225,8 +257,9 @@ impl L10N {
     pub fn format_f64(&self, value: f64, precision: FloatPrecision) -> String {
         let fdf = FixedDecimalFormatter::try_new(
             &self.locales.first().clone().into(),
-            Default::default()
+            Default::default(),
-            ).expect("locale should be present");
+        )
+        .expect("locale should be present");
 
         let number = FixedDecimal::try_from_f64(value, precision).unwrap();
 
@@ -270,9 +303,10 @@ impl From<DateTime> for icu::calendar::DateTime<icu::calendar::Gregorian> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use fluent::fluent_args;
 
     fn ref_l10n() -> L10N {
-        let mut l10n = L10N::default();
+        let mut l10n = L10N::new(std::path::PathBuf::from("./test_files"));
         // Make sure we know the locale before the test begins. Some systems, such as my own, are
         // not actually in English.
         l10n.set_locales(NonEmptyList::from_iter(vec!["en-US"]).unwrap());
@@ -353,23 +387,79 @@ mod tests {
     fn it_formats_a_number_according_to_locale() {
         let mut l10n = ref_l10n();
 
-        assert_eq!(
+        assert_eq!(l10n.format_f64(100.4, FloatPrecision::Floating), "100.4",);
-            l10n.format_f64(100.4, FloatPrecision::Floating),
-            "100.4",
-            );
         assert_eq!(
             l10n.format_f64(15000.4, FloatPrecision::Floating),
             "15,000.4",
-            );
+        );
 
         l10n.set_locales(NonEmptyList::from_iter(vec!["de-DE", "en-US"]).unwrap());
-        assert_eq!(
+        assert_eq!(l10n.format_f64(100.4, FloatPrecision::Floating), "100,4",);
-            l10n.format_f64(100.4, FloatPrecision::Floating),
-            "100,4",
-            );
         assert_eq!(
             l10n.format_f64(15000.4, FloatPrecision::Floating),
             "15.000,4",
-            );
+        );
     }
+
+    #[test]
+    fn it_can_load_message_files() {
+        let mut l10n = ref_l10n();
+        let messages = l10n.messages();
+
+        let args = fluent_args![
+            "name" => "Savanni"
+        ];
+        assert_eq!(
+            messages.tr("welcome", Some(&args)).unwrap(),
+            "Hello, Savanni"
+        );
+
+        let args = fluent_args![
+            "count" => 1
+        ];
+        assert_eq!(
+            messages.tr("games-in-database", Some(&args)).unwrap(),
+            "There is one game in the database"
+        );
+
+        let args = fluent_args![
+            "count" => 2
+        ];
+        assert_eq!(
+            messages.tr("games-in-database", Some(&args)).unwrap(),
+            "There are 2 games in the database"
+        );
+    }
+
+    /*
+    #[test]
+    fn it_can_change_languages_on_locale_change() {
+    }
+
+    #[test]
+    fn phrases_can_be_translated() {
+    }
+
+    #[test]
+    fn phrases_can_fall_back() {
+    }
+    */
+
+    /* Not really a unit test, more of a test to see what I could introspect within a fluent
+     * message. I was hoping that attributes would give me placeholder names, but that doesn't seem
+     * to be the case.
+    #[test]
+    fn messages() {
+        let langid_en = "en-US".parse().expect("Parsing failed.");
+        let resource = FluentResource::try_new(MESSAGES.to_owned()).unwrap();
+        let mut bundle = FluentBundle::new(vec![langid_en]);
+        bundle.add_resource(&resource).unwrap();
+
+        let msg = bundle.get_message("welcome").expect("message should exist");
+        for attr in msg.attributes() {
+            println!("attr: {:?}", attr);
+        }
+        assert!(false);
+    }
+    */
 }

l10n/test_files/en-US.ftl

@@ -0,0 +1,5 @@
+welcome = Hello, {$name}
+games-in-database = {$count ->
+    [one] There is one game in the database
+    *[other] There are {$count} games in the database
+}

messages-codegen/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "messages-codegen"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+proc-macro = true
+
+[dependencies]
+convert_case    = { version = "0.6" }
+quote           = { version = "*" }
+syn             = { version = "*" }
+serde_yaml      = { version = "*" }
+serde           = { version = "*", features = [ "derive" ] }

messages-codegen/src/lib.rs

@@ -0,0 +1,87 @@
+/* Given that fluent doesn't provide an ability to see all of the placeholders within a message,
+ * and that the placeholders may change depending on the values of others, I don't have a way to
+ * ensure that the translation code remains entirely connected to the source messages themselves.
+ *
+ * Given that I can't really enforce it, I can, perhaps, codegen it in a way that at least the two
+ * are right next to one another. A macro that generates the data structure and can write the
+ * source strings file.
+ *
+ * For overall file structure, I want the messages, L10N, and FluentErgo tied together.
+ */
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn;
+
+/* I want to write a macro that reads something like this:
+ * messages! {
+ *     Welcome(name: String) => "Hello, ${name}",
+ *     GamesInDatabase(count: usize) => "{$count ->
+ *     [one] There is one game in the database
+ *     *[other] There are ${count} games in the database
+ * }",
+ *
+ * It generates an enumeration with all of the named values (Welcome, GamesInDatabase), it
+ * generates corresponding structures (Welcome{ name: String }, GamesInDatabase{ count: usize }),
+ * and it generates two implementations. One is an implementation that can be used to write all of
+ * the strings after => into a file. The other is an implementation that can be imported into a
+ * program, but which *does not* contain the strings. That would look more like this:
+ *
+ * messages.rs:
+ *
+ * enum Messages {
+ *   Welcome(Welcome),
+ *   GamesInDatabase(GamesInDatabase),
+ * }
+ *
+ * struct Welcome{ name: String }
+ * struct GamesInDatabase{ count: usize }
+ *
+ * message_strings.ftl:
+ *
+ * welcome = "Hello, ${name}",
+ * games-in-database = "{$count ->
+ *     [one] There is one game in the database
+ *     *[other] There are ${count} games in the database
+ * }",
+ *
+ * messages.rs can be imported into the resulting program, and the names of the strings are thus a
+ * part of the program, but the strings themselves are still data files.
+ */
+
+/*
+#[macro_export]
+macro_rules! messages {
+    ($($name:ident($($arg:ident: $argtype:ty)*) => $message:literal,)+) => {
+        pub enum Messages {
+            $($name($name)),+
+        }
+
+        impl Messages {
+            fn gen_strings() -> Vec<String> {
+                vec![
+                    $(concat!(stringify!($name), " => ", $message).to_string(),)+
+                ]
+            }
+        }
+
+        $(pub struct $name {
+            $($arg: $argtype)*
+        })+
+    }
+}
+*/
+
+#[proc_macro_derive(Messages)]
+pub fn messages_macro(input: TokenStream) -> TokenStream {
+    let syn::DeriveInput { ident, .. } = syn::parse_macro_input! {input};
+
+    let gen = quote! {
+        impl Messages for #ident {
+            fn hello() {
+                println!("Hello from {}", stringify!(#ident));
+            }
+        }
+    };
+    gen.into()
+}

messages-codegen/src/main.rs

@@ -0,0 +1,135 @@
+use convert_case::{Case, Casing};
+use serde::Deserialize;
+use serde_yaml;
+use std::{collections::HashMap, fmt, fs::File};
+
+#[derive(Debug, Clone, Deserialize)]
+struct MessageJS {
+    #[serde(default)]
+    parameters: HashMap<String, String>,
+    content: String,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum Ty {
+    String,
+    Number,
+    Count,
+}
+
+impl fmt::Display for Ty {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        match self {
+            Ty::String => write!(f, "String")?,
+            Ty::Number => write!(f, "i32")?,
+            Ty::Count => write!(f, "usize")?,
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct Parameter {
+    name: String,
+    ty: Ty,
+}
+
+impl From<&str> for Parameter {
+    fn from(val: &str) -> Self {
+        Self {
+            name: "Nothing".to_owned(),
+            ty: Ty::String,
+        }
+    }
+}
+
+impl From<(String, String)> for Parameter {
+    fn from((name, val): (String, String)) -> Self {
+        let ty = match val.as_ref() {
+            "string" => Ty::String,
+            "number" => Ty::Number,
+            "count" => Ty::Count,
+            _ => panic!("invalid value"),
+        };
+        Self { name, ty }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct Message {
+    name: String,
+    parameters: Vec<Parameter>,
+    content: String,
+}
+
+impl Message {
+    fn from_js(name: String, source: MessageJS) -> Self {
+        Self {
+            name,
+            parameters: source
+                .parameters
+                .into_iter()
+                .map(|param| Parameter::from(param))
+                .collect(),
+            content: source.content,
+        }
+    }
+
+    fn rust_code(&self) -> String {
+        if self.parameters.is_empty() {
+            format!(r"pub struct {}; ", self.name.to_case(Case::Pascal))
+        } else {
+            let mut struct_strs = vec![];
+
+            struct_strs.push(format!("pub struct {} {{", self.name.to_case(Case::Pascal)));
+            let mut parameters: Vec<String> = self
+                .parameters
+                .iter()
+                .map(|param| format!("    {}: {},", param.name, param.ty))
+                .collect();
+
+            struct_strs.append(&mut parameters);
+            struct_strs.push("}".to_owned());
+            struct_strs.push("".to_owned());
+
+            struct_strs.push(format!(
+                "impl Message for {} {{",
+                self.name.to_case(Case::Pascal)
+            ));
+            struct_strs.push(format!("    pub fn localize(&self, bundle: &FluentBundle) -> String {{"));
+            struct_strs.push("        let mut args = FluentArgs::new();".to_owned());
+            let mut parameters: Vec<String> = self
+                .parameters
+                .iter()
+                .map(|param| format!("        args.set(\"{}\", self.{})", param.name, param.name))
+                .collect();
+            struct_strs.append(&mut parameters);
+            struct_strs.push("".to_owned());
+            struct_strs.push(format!("        let msg = bundle.get_message(\"{}\").unwrap();", self.name));
+            struct_strs.push(format!("        let mut errors = vec![]"));
+            struct_strs.push(format!("        msg.format_pattern(&msg.value().unwrap(), args, &mut errors)"));
+            struct_strs.push("    }".to_owned());
+            struct_strs.push("}".to_owned());
+
+            struct_strs.join("\n")
+        }
+    }
+
+    fn content(&self) -> String {
+        format!("{} = {}", self.name, self.content.clone())
+    }
+}
+
+fn main() {
+    let messages: HashMap<String, MessageJS> =
+        serde_yaml::from_reader(File::open("test-data/messages.yaml").unwrap()).unwrap();
+
+    let messages = messages
+        .into_iter()
+        .map(|(name, msg)| Message::from_js(name, msg));
+    for message in messages {
+        println!();
+        println!("{}", message.rust_code());
+        // println!("{}", message.content());
+    }
+}

messages-codegen/test-data/messages.yaml

@@ -0,0 +1,14 @@
+welcome:
+  content: "Welcome to Kifu"
+hello:
+  parameters:
+    name: string
+  content: "Hello, ${name}"
+games-in-database:
+  parameters:
+    count: count
+  content: |
+    {$count ->
+      [one] There is one game in the database
+      *[other] There are ${count} games in the database
+    }

xdg-test/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "xdg-test"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

xdg-test/override.nix

@@ -0,0 +1,7 @@
+{ wrapGAppsHook4 }:
+attrs: 
+let
+    gsettingsDir = "${attrs.crateName}-${attrs.version}";
+in {
+    nativeBuildInputs = [ wrapGAppsHook4 ];
+}

xdg-test/src/main.rs

@@ -0,0 +1,7 @@
+use std::env;
+
+fn main() {
+    let data_dirs = env::var("XDG_DATA_DIRS");
+
+    println!("{:?}", data_dirs);
+}