split arch-unspecific data to save space and bandwidth (#34)

.github/workflows/ci.yml

@@ -34,6 +34,7 @@ jobs:
             ninja
           wget https://github.com/fcitx-contrib/fcitx5-macos-prebuilder/releases/download/latest/marisa-${{ matrix.arch }}.tar.bz2
           sudo tar xjvf marisa-${{ matrix.arch }}.tar.bz2 -C /usr/local bin/marisa-build
+          pip install dirhash
 
       - name: Download and install Fcitx5.app
         run: |
@@ -43,12 +44,20 @@ jobs:
       - name: Build all
         run: ./all.sh ${{ matrix.arch }}
 
-      - name: Upload artifact
+      - name: Upload arch-specific artifact
         uses: actions/upload-artifact@v4
         with:
           name: artifact-${{ matrix.arch }}
           path: |
-            build/*.tar.bz2
+            build/*-${{ matrix.arch }}.tar.bz2
+
+      - name: Upload arch-unspecific artifact
+        if: ${{ matrix.os == 'macos-14' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: |
+            build/*-any.tar.bz2
 
       - name: Check validity
         run: ./check-validity.sh
@@ -68,6 +77,37 @@ jobs:
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3
 
+  compare:
+    needs: build
+    if: ${{ github.ref != 'refs/heads/master' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+
+      - name: Compare
+        run: python compare-arches.py > summary.md
+
+      - name: Find comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: "Arch comparison"
+
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body-path: summary.md
+          edit-mode: replace
+
   release:
     needs: build
     if: ${{ github.ref == 'refs/heads/master' }}

common.sh

@@ -49,11 +49,26 @@ f5m_install() {
   done
 }
 
+f5m_split_data() {
+  cd $DESTDIR$INSTALL_PREFIX
+  rm -rf ../data
+  mkdir -p ../data
+  rm -rf share/{icons,metainfo} # useless for macOS
+  for dir in "include" "share"; do
+    if [[ -d $dir ]]; then
+      mv $dir ../data/$dir
+    fi
+  done
+}
+
 # params: IMs to auto add on plugin install
 f5m_make_tarball() {
   cd $DESTDIR$INSTALL_PREFIX
   python $ROOT/generate-descriptor.py "$@"
   tar cjvf ../../../$name-$ARCH.tar.bz2 *
+
+  cd ../data
+  tar cjvf ../../../$name-any.tar.bz2 *
 }
 
 set -x

compare-arches.py

@@ -0,0 +1,43 @@
+import json
+
+plugins = {
+    "x86_64": {},
+    "arm64": {}
+}
+
+for arch in ("arm64", "x86_64"):
+    with open(f"meta-{arch}.json") as f:
+        meta = json.load(f)
+        for plugin in meta["plugins"]:
+            plugins[arch][plugin["name"]] = plugin
+
+all_plugins = sorted(set(plugins["x86_64"].keys()) | set(plugins["arm64"].keys()))
+
+print("## Arch comparison")
+print("-|x86_64|arm64|any")
+print("-|-|-|-")
+for plugin in all_plugins:
+    x86_64 = plugins["x86_64"].get(plugin)
+    arm64 = plugins["arm64"].get(plugin)
+    if not x86_64:
+        (x, a, d) = ("💀️", "❌", "❌")
+    elif not arm64:
+        (x, a, d) = ("❌", "💀️", "❌")
+    else:
+        if "version" in x86_64:
+            x = "✅"
+        else:
+            x = "❌"
+        if "version" in arm64:
+            a = "✅"
+        else:
+            a = "❌"
+        if "data_version" not in x86_64 or "data_version" not in arm64:
+            d = "💀️"
+        elif x86_64["data_version"] != arm64["data_version"]:
+            d = "❌"
+        else:
+            d = "✅"
+    print(f"{plugin}|{x}|{a}|{d}")
+
+print(f"\n{len(all_plugins)} plugins in total.")

generate-descriptor.py

@@ -1,35 +1,40 @@
 import json
 import os
-import subprocess
 import sys
+from dirhash import dirhash
 
 input_methods = sys.argv[1:]
 
 cwd = os.getcwd()
+data_dir = f"{cwd}/../data"
 plugin = cwd.split("/")[-3] # /path/to/rime/tmp/fcitx5
 files: list[str] = []
 
-for dirpath, _, filenames in os.walk(cwd):
-    if dirpath == "plugin":
-        continue
-    for filename in filenames:
-        files.append(f"{dirpath[len(cwd) + 1:]}/{filename}")
+for wd in (cwd, data_dir):
+    for dirpath, _, filenames in os.walk(wd):
+        if dirpath == f"{wd}/plugin":
+            continue
+        for filename in filenames:
+            files.append(f"{dirpath[len(wd) + 1:]}/{filename}")
 
+try:
+    version = dirhash(cwd, "md5", ignore=["plugin"])
+except:
+    # pure data plugin
+    version = None
 
-def getVersion():
-    # XXX: it causes unnecessary update when the plugin is not changed
-    result = subprocess.run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE, text=True)
-    return result.stdout.strip()
+data_version = dirhash(data_dir, "md5")
 
+plugin_dir = f"{data_dir}/plugin"
 
-version = getVersion()
-
-os.makedirs("plugin", exist_ok=True)
-with open(f"plugin/{plugin}.json", "w") as f:
+os.makedirs(plugin_dir, exist_ok=True)
+with open(f"{plugin_dir}/{plugin}.json", "w") as f:
     descriptor = {
-        "version": version,
+        "data_version": data_version,
         "files": files
     }
+    if version:
+        descriptor["version"] = version
     if input_methods:
         descriptor["input_methods"] = input_methods
     json.dump(descriptor, f)

generate-meta.py

@@ -10,15 +10,20 @@ def getArch():
 arch = getArch()
 plugins = []
 
-for script in os.listdir("scripts"):
-    plugin = script.split(".")[0]
-    descriptor = f"build/{plugin}/tmp/fcitx5/plugin/{plugin}.json"
+for plugin in os.listdir("build"):
+    if plugin.endswith(".tar.bz2"):
+        continue
+    descriptor = f"build/{plugin}/tmp/data/plugin/{plugin}.json"
     with open(descriptor, "r") as f:
-        version = json.load(f)["version"]
+        j = json.load(f)
+        version = j.get("version")
+        data_version = j["data_version"]
     plugins.append({
         "name": plugin,
-        "version": version
+        "data_version": data_version
     })
+    if version:
+        plugins[-1]["version"] = version
 
 with open(f"meta-{arch}.json", "w") as f:
     json.dump({

package-table-extra.py

@@ -0,0 +1,74 @@
+import os
+
+plugins = {
+    "array": [
+        "array30",
+        "array30-large",
+    ],
+    "boshiamy": [
+        "boshiamy"
+    ],
+    "cangjie": [
+        "cangjie5",
+        "cangjie3",
+        "cangjie-large",
+        "scj6"
+    ],
+    "cantonese": [
+        "cantonese",
+        "cantonhk",
+        "jyutping-table"
+    ],
+    "quick": [
+        "quick5",
+        "quick3",
+        "quick-classic",
+        "easy-large"
+    ],
+    "stroke": [
+        "t9",
+        "stroke5"
+    ],
+    "wu": [
+        "wu"
+    ],
+    "wubi86": [
+        "wubi-large"
+    ],
+    "wubi98": [
+        "wubi98",
+        "wubi98-single",
+        "wubi98-large",
+        "wubi98-pinyin"
+    ],
+    "zhengma": [
+        "zhengma",
+        "zhengma-large",
+        "zhengma-pinyin"
+    ]
+}
+
+
+def ensure(command: str):
+    if os.system(command) != 0:
+        exit(1)
+
+
+build_dir = os.getcwd()
+
+# split table-extra to plugins and keep directory structures so the package script can be reused
+for plugin, ims in plugins.items():
+    ensure(f"rm -rf {plugin}")
+    ensure(f"mkdir -p {plugin}/" + "tmp/data/share/fcitx5/{inputmethod,table}")
+    ensure(f"mkdir -p {plugin}/tmp/fcitx5")
+
+    # equivalent to f5m_split_data
+    for im in ims:
+        ensure(f"mv table-extra/tmp/fcitx5/share/fcitx5/inputmethod/{im}.conf {plugin}/tmp/data/share/fcitx5/inputmethod")
+        ensure(f"mv table-extra/tmp/fcitx5/share/fcitx5/table/{im}.main.dict {plugin}/tmp/data/share/fcitx5/table")
+
+    os.chdir(f"{plugin}/tmp/fcitx5")
+    ensure(f"python {build_dir}/../generate-descriptor.py {ims[0]}")
+    os.chdir("../data")
+    ensure(f"tar cjvf {build_dir}/{plugin}-any.tar.bz2 *")
+    os.chdir(build_dir)

scripts/anthy.sh

@@ -7,4 +7,5 @@ install_deps anthy-unicode
 f5m_configure -DENABLE_TEST=OFF
 f5m_build
 f5m_install anthy-unicode
+f5m_split_data
 f5m_make_tarball anthy

scripts/chinese-addons.sh

@@ -21,6 +21,10 @@ fi
 f5m_configure "${ARGS[@]}"
 f5m_build
 f5m_install libime opencc
+
+f5m_split_data
+
 # Install zh_CN.lm and zh_CN.lm.predict which are not in share
-tar xjvf $CACHE_DIR/libime-$ARCH.tar.bz2 -C $DESTDIR$INSTALL_PREFIX lib/libime
+tar xjvf $CACHE_DIR/libime-$ARCH.tar.bz2 -C $DESTDIR$INSTALL_PREFIX/../data lib/libime
+
 f5m_make_tarball pinyin

scripts/hallelujah.sh

@@ -6,4 +6,5 @@ install_deps marisa fmt json-c
 f5m_configure -DENABLE_TEST=OFF
 f5m_build
 f5m_install
+f5m_split_data
 f5m_make_tarball hallelujah

scripts/lua.sh

@@ -8,4 +8,5 @@ install_deps lua
 f5m_configure -DENABLE_TEST=OFF -DUSE_DLOPEN=OFF
 f5m_build
 f5m_install
+f5m_split_data
 f5m_make_tarball

scripts/rime.sh

@@ -29,4 +29,5 @@ cp default.yaml $rime_dir
 cd -
 cp -r $INSTALL_PREFIX/share/opencc $DESTDIR$rime_data_dir
 
+f5m_split_data
 f5m_make_tarball rime

scripts/skk.sh

@@ -36,4 +36,5 @@ type=file,file=\$FCITX_CONFIG_DIR/skk/user.dict,mode=readwrite
 type=file,file=\$XDG_DATA_DIR/skk-dict/SKK-JISYO.L,mode=readonly
 EOF
 
+f5m_split_data
 f5m_make_tarball skk

scripts/table-extra.sh

@@ -6,4 +6,9 @@ install_deps boost libime
 f5m_configure
 f5m_build
 f5m_install
-f5m_make_tarball table-extra
+
+cd $ROOT/build
+python $ROOT/package-table-extra.py
+
+# generate-meta.py will enumerate build/
+rm -r $ROOT/build/table-extra

scripts/thai.sh

@@ -6,4 +6,5 @@ install_deps libthai
 f5m_configure
 f5m_build
 f5m_install
+f5m_split_data
 f5m_make_tarball libthai

scripts/unikey.sh

@@ -5,4 +5,5 @@ set -e
 f5m_configure -DENABLE_QT=OFF -DENABLE_TEST=OFF
 f5m_build
 f5m_install
+f5m_split_data
 f5m_make_tarball unikey