feat: re-enable and update python message format (#2855)

backend/data/src/main/kotlin/io/tolgee/formats/ExportMessageFormat.kt

@@ -5,6 +5,7 @@ import io.tolgee.formats.paramConvertors.out.IcuToCPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToI18nextPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToJavaPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToPhpPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.out.IcuToPythonPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.out.IcuToRubyPlaceholderConvertor
 
 @Suppress("unused") // it's exposed to the API
@@ -16,5 +17,5 @@ enum class ExportMessageFormat(val paramConvertorFactory: () -> FromIcuPlacehold
   RUBY_SPRINTF(paramConvertorFactory = { IcuToRubyPlaceholderConvertor() }),
   I18NEXT(paramConvertorFactory = { IcuToI18nextPlaceholderConvertor() }),
   ICU(paramConvertorFactory = { IcuToIcuPlaceholderConvertor() }),
-//  PYTHON_SPRINTF,
+  PYTHON_PERCENT(paramConvertorFactory = { IcuToPythonPlaceholderConvertor() }),
 }

backend/data/src/main/kotlin/io/tolgee/formats/importCommon/ImportFormat.kt

@@ -5,6 +5,7 @@ import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.I18nextToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.`in`.PythonToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor
 import io.tolgee.formats.po.`in`.PoToIcuMessageConvertor
 
@@ -106,7 +107,10 @@ enum class ImportFormat(
     ImportFileFormat.PO,
     messageConvertorOrNull = PoToIcuMessageConvertor { RubyToIcuPlaceholderConvertor() },
   ),
-//  PO_PYTHON(messageConvertorOrNull = BasePoToIcuMessageConvertor { PythonToIcuPlaceholderConvertor() }),
+  PO_PYTHON(
+    ImportFileFormat.PO,
+    messageConvertorOrNull = PoToIcuMessageConvertor { PythonToIcuPlaceholderConvertor() },
+  ),
 
   STRINGS(
     ImportFileFormat.STRINGS,

backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/in/PythonToIcuPlaceholderConvertor.kt

@@ -6,7 +6,7 @@ class PythonToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor {
   override val pluralArgName: String = "0"
 
   override val regex: Regex
-    get() = PYTHON_PARAM_REGEX
+    get() = PYTHON_PLACEHOLDER_REGEX
 
   private val baseToIcuPlaceholderConvertor = BaseToIcuPlaceholderConvertor()
 
@@ -18,7 +18,7 @@ class PythonToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor {
   }
 
   companion object {
-    val PYTHON_PARAM_REGEX =
+    val PYTHON_PLACEHOLDER_REGEX =
       """
       (?x)(
       %
@@ -30,5 +30,18 @@ class PythonToIcuPlaceholderConvertor : ToIcuPlaceholderConvertor {
       (?<specifier>[diouxXeEfFgGcrs%])
       )
       """.trimIndent().toRegex()
+
+    val PYTHON_DETECTION_REGEX =
+      """
+      (?x)(
+      (^|\W+)%
+      (?:\((?<argname>[\w-]+)\))?
+      (?<flags>[-+0\#]+)?
+      (?<width>[\d*]+)?
+      (?:\.(?<precision>\d+))?
+      (?<length>[hlL])?
+      (?<specifier>[diouxXeEfFgGcrs%])
+      )
+      """.trimIndent().toRegex()
   }
 }

backend/data/src/main/kotlin/io/tolgee/formats/paramConvertors/out/IcuToPythonPlaceholderConvertor.kt

@@ -1,25 +1,19 @@
 package io.tolgee.formats.paramConvertors.out
 
-import com.ibm.icu.text.MessagePattern
 import io.tolgee.formats.FromIcuPlaceholderConvertor
 import io.tolgee.formats.MessagePatternUtil
 import io.tolgee.formats.escapePercentSign
 
 class IcuToPythonPlaceholderConvertor : FromIcuPlaceholderConvertor {
-  private var argIndex = -1
-
-  override fun convert(node: MessagePatternUtil.ArgNode): String {
-    argIndex++
-    val argNumString = getArgNameString(node)
-    val type = node.argType
-
-    if (type == MessagePattern.ArgType.SIMPLE) {
-      when (node.typeName) {
-        "number" -> return convertNumber(node)
-      }
+  private val baseToCLikePlaceholderConvertor =
+    BaseToCLikePlaceholderConvertor(
+      defaultSpecifier = "s",
+    ) {
+      getArgNameString(it)
     }
 
-    return "%${argNumString}s"
+  override fun convert(node: MessagePatternUtil.ArgNode): String {
+    return baseToCLikePlaceholderConvertor.convert(node)
   }
 
   override fun convertText(
@@ -33,35 +27,10 @@ class IcuToPythonPlaceholderConvertor : FromIcuPlaceholderConvertor {
     node: MessagePatternUtil.MessageContentsNode,
     argName: String?,
   ): String {
-    return "%($argName)d"
-  }
-
-  private fun convertNumber(node: MessagePatternUtil.ArgNode): String {
-    if (node.simpleStyle?.trim() == "scientific") {
-      return "%${getArgNameString(node)}e"
-    }
-    val precision = getPrecision(node)
-    if (precision == 6) {
-      return "%${getArgNameString(node)}f"
-    }
-    if (precision != null) {
-      return "%${getArgNameString(node)}.${precision}f"
-    }
-
-    return "%${getArgNameString(node)}d"
-  }
-
-  private fun getPrecision(node: MessagePatternUtil.ArgNode): Int? {
-    val precisionMatch = ICU_PRECISION_REGEX.matchEntire(node.simpleStyle ?: "")
-    precisionMatch ?: return null
-    return precisionMatch.groups["precision"]?.value?.length
+    return if (argName != null) "%($argName)d" else "%d"
   }
 
   private fun getArgNameString(node: MessagePatternUtil.ArgNode): String {
     return "(${node.name})"
   }
-
-  companion object {
-    val ICU_PRECISION_REGEX = """.*\.(?<precision>0+)""".toRegex()
-  }
 }

backend/data/src/main/kotlin/io/tolgee/formats/po/in/PoFormatDetector.kt

@@ -5,6 +5,7 @@ import io.tolgee.formats.importCommon.ImportFormat
 import io.tolgee.formats.paramConvertors.`in`.CToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.JavaToIcuPlaceholderConvertor.Companion.JAVA_DETECTION_REGEX
 import io.tolgee.formats.paramConvertors.`in`.PhpToIcuPlaceholderConvertor
+import io.tolgee.formats.paramConvertors.`in`.PythonToIcuPlaceholderConvertor
 import io.tolgee.formats.paramConvertors.`in`.RubyToIcuPlaceholderConvertor
 
 class PoFormatDetector() {
@@ -47,13 +48,13 @@ class PoFormatDetector() {
               0.7,
             ),
           ),
-//        ImportFormat.PO_PYTHON to
-//          arrayOf(
-//            FormatDetectionUtil.regexFactor(
-//              PHP_PLACEHOLDER_REGEX,
-//              1.05,
-//            ),
-//          ),
+        ImportFormat.PO_PYTHON to
+          arrayOf(
+            FormatDetectionUtil.regexFactor(
+              PythonToIcuPlaceholderConvertor.PYTHON_DETECTION_REGEX,
+              1.0,
+            ),
+          ),
       )
   }
 
@@ -63,6 +64,7 @@ class PoFormatDetector() {
       "c-format" -> return ImportFormat.PO_C
       "java-format" -> return ImportFormat.PO_JAVA
       "icu-format" -> return ImportFormat.PO_ICU
+      "python-format" -> return ImportFormat.PO_PYTHON
       else -> null
     }
   }

backend/data/src/test/kotlin/io/tolgee/unit/formats/po/out/PoMessageFormatsExporterTest.kt

@@ -35,28 +35,28 @@ class PoMessageFormatsExporterTest {
     )
   }
 
-//  @Test
-//  fun python() {
-//    val exporter = getExporter(PoSupportedMessageFormat.PYTHON)
-//    val data = getExported(exporter)
-//    data.assertFile(
-//      "cs.po",
-//      """
-//    |msgid ""
-//    |msgstr ""
-//    |"Language: cs\n"
-//    |"MIME-Version: 1.0\n"
-//    |"Content-Type: text/plain; charset=UTF-8\n"
-//    |"Content-Transfer-Encoding: 8bit\n"
-//    |"Plural-Forms: nplurals = 3; plural = (n === 1 ? 0 : (n >= 2 && n <= 4) ? 1 : 2)\n"
-//    |"X-Generator: Tolgee\n"
-//    |
-//    |msgid "key3"
-//    |msgstr "%(2)d %(1)s %(0)s"
-//    |
-//      """.trimMargin(),
-//    )
-//  }
+  @Test
+  fun python() {
+    val exporter = getExporter(ExportMessageFormat.PYTHON_PERCENT)
+    val data = getExported(exporter)
+    data.assertFile(
+      "cs.po",
+      """
+    |msgid ""
+    |msgstr ""
+    |"Language: cs\n"
+    |"MIME-Version: 1.0\n"
+    |"Content-Type: text/plain; charset=UTF-8\n"
+    |"Content-Transfer-Encoding: 8bit\n"
+    |"Plural-Forms: nplurals = 3; plural = (n === 1 ? 0 : (n >= 2 && n <= 4) ? 1 : 2)\n"
+    |"X-Generator: Tolgee\n"
+    |
+    |msgid "key3"
+    |msgstr "%(2)d %(1)s %(0)s"
+    |
+      """.trimMargin(),
+    )
+  }
 
   @Test
   fun c() {

e2e/cypress/common/export.ts

@@ -164,7 +164,14 @@ export const testExportFormats = (
   });
 
   testFormatWithMessageFormats(
-    ['ICU', 'PHP Sprintf', 'C Sprintf', 'Ruby Sprintf', 'Java String.format'],
+    [
+      'ICU',
+      'PHP Sprintf',
+      'C Sprintf',
+      'Ruby Sprintf',
+      'Java String.format',
+      'Python Percent',
+    ],
     {
       format: 'Gettext (.po)',
       expectedParams: {
@@ -303,6 +310,7 @@ const messageFormatParamMap = {
   'C Sprintf': 'C_SPRINTF' as MessageFormat,
   'Java String.format': 'JAVA_STRING_FORMAT' as MessageFormat,
   'Ruby Sprintf': 'RUBY_SPRINTF' as MessageFormat,
+  'Python Percent': 'PYTHON_PERCENT' as MessageFormat,
 };
 
 type MessageFormat = components['schemas']['ExportParams']['messageFormat'];

webapp/src/service/apiSchema.generated.ts

@@ -1373,7 +1373,8 @@ export interface components {
         | "APPLE_SPRINTF"
         | "RUBY_SPRINTF"
         | "I18NEXT"
-        | "ICU";
+        | "ICU"
+        | "PYTHON_PERCENT";
       name: string;
       pruneBeforePublish: boolean;
       publicUrl?: string;
@@ -1478,7 +1479,8 @@ export interface components {
         | "APPLE_SPRINTF"
         | "RUBY_SPRINTF"
         | "I18NEXT"
-        | "ICU";
+        | "ICU"
+        | "PYTHON_PERCENT";
       name: string;
       /**
        * @description Whether the data in the CDN should be pruned before publishing new data.
@@ -2077,7 +2079,8 @@ export interface components {
         | "APPLE_SPRINTF"
         | "RUBY_SPRINTF"
         | "I18NEXT"
-        | "ICU";
+        | "ICU"
+        | "PYTHON_PERCENT";
       /**
        * @description Delimiter to structure file content.
        *
@@ -2203,6 +2206,7 @@ export interface components {
         | "PO_JAVA"
         | "PO_ICU"
         | "PO_RUBY"
+        | "PO_PYTHON"
         | "STRINGS"
         | "STRINGSDICT"
         | "APPLE_XLIFF"
@@ -10173,7 +10177,8 @@ export interface operations {
           | "APPLE_SPRINTF"
           | "RUBY_SPRINTF"
           | "I18NEXT"
-          | "ICU";
+          | "ICU"
+          | "PYTHON_PERCENT";
         /**
          * This is a template that defines the structure of the resulting .zip file content.
          *

webapp/src/views/projects/export/components/formatGroups.tsx

@@ -92,6 +92,7 @@ export const formatGroups: FormatGroup[] = [
           'JAVA_STRING_FORMAT',
           'ICU',
           'RUBY_SPRINTF',
+          'PYTHON_PERCENT',
         ],
       },
       {

webapp/src/views/projects/export/components/messageFormatTranslation.tsx

@@ -12,4 +12,5 @@ export const messageFormatTranslation: Record<MessageFormat, ReactNode> = {
   I18NEXT: <T keyName="export_form_message_format_i18next" />,
   ICU: <T keyName="export_form_message_format_icu" />,
   APPLE_SPRINTF: <T keyName="export_form_message_format_apple-sprintf" />,
+  PYTHON_PERCENT: <T keyName="export_form_message_format_python-percent" />,
 };