From 39a6d4186557e51932b3ff948bd9d94eba8266a6 Mon Sep 17 00:00:00 2001 From: mario4tier Date: Mon, 13 Jan 2025 21:28:06 -0800 Subject: [PATCH] CI: Add better tracking for windows package tests --- .../ta-lib-0.6.4-windows-x86_32.msi.digest | 10 ++ .../ta-lib-0.6.4-windows-x86_32.zip.digest | 10 ++ .../ta-lib-0.6.4-windows-x86_64.msi.digest | 10 ++ .../ta-lib-0.6.4-windows-x86_64.zip.digest | 10 ++ scripts/package.py | 115 ++++++++++++++---- scripts/utilities/common.py | 8 +- 6 files changed, 134 insertions(+), 29 deletions(-) create mode 100644 dist/digests/ta-lib-0.6.4-windows-x86_32.msi.digest create mode 100644 dist/digests/ta-lib-0.6.4-windows-x86_32.zip.digest create mode 100644 dist/digests/ta-lib-0.6.4-windows-x86_64.msi.digest create mode 100644 dist/digests/ta-lib-0.6.4-windows-x86_64.zip.digest diff --git a/dist/digests/ta-lib-0.6.4-windows-x86_32.msi.digest b/dist/digests/ta-lib-0.6.4-windows-x86_32.msi.digest new file mode 100644 index 00000000..6529d18d --- /dev/null +++ b/dist/digests/ta-lib-0.6.4-windows-x86_32.msi.digest @@ -0,0 +1,10 @@ +{ + "asset_file_name": "ta-lib-0.6.4-windows-x86_32.msi", + "sources_digest": "f56bc6c0c0d72f1ee648897220e3ec3f", + "builder_id": "mario4tier", + "built_success": "True", + "package_md5": "3f07f119fb7e63614b31958832edeb12", + "gen_code_pass": "Disabled", + "ta_regtest_pass": "Disabled", + "dist_test_pass": "True" +} \ No newline at end of file diff --git a/dist/digests/ta-lib-0.6.4-windows-x86_32.zip.digest b/dist/digests/ta-lib-0.6.4-windows-x86_32.zip.digest new file mode 100644 index 00000000..232355e5 --- /dev/null +++ b/dist/digests/ta-lib-0.6.4-windows-x86_32.zip.digest @@ -0,0 +1,10 @@ +{ + "asset_file_name": "ta-lib-0.6.4-windows-x86_32.zip", + "sources_digest": "f56bc6c0c0d72f1ee648897220e3ec3f", + "builder_id": "mario4tier", + "built_success": "True", + "package_md5": "17f56e378508201998021766e0c5e00d", + "gen_code_pass": "Disabled", + "ta_regtest_pass": "Disabled", + "dist_test_pass": "True" +} \ No newline at end of file diff --git a/dist/digests/ta-lib-0.6.4-windows-x86_64.msi.digest b/dist/digests/ta-lib-0.6.4-windows-x86_64.msi.digest new file mode 100644 index 00000000..e9576ba1 --- /dev/null +++ b/dist/digests/ta-lib-0.6.4-windows-x86_64.msi.digest @@ -0,0 +1,10 @@ +{ + "asset_file_name": "ta-lib-0.6.4-windows-x86_64.msi", + "sources_digest": "f56bc6c0c0d72f1ee648897220e3ec3f", + "builder_id": "mario4tier", + "built_success": "True", + "package_md5": "b8c36d7628599d85e269de3d7166c748", + "gen_code_pass": "Disabled", + "ta_regtest_pass": "Disabled", + "dist_test_pass": "True" +} \ No newline at end of file diff --git a/dist/digests/ta-lib-0.6.4-windows-x86_64.zip.digest b/dist/digests/ta-lib-0.6.4-windows-x86_64.zip.digest new file mode 100644 index 00000000..cd03e450 --- /dev/null +++ b/dist/digests/ta-lib-0.6.4-windows-x86_64.zip.digest @@ -0,0 +1,10 @@ +{ + "asset_file_name": "ta-lib-0.6.4-windows-x86_64.zip", + "sources_digest": "f56bc6c0c0d72f1ee648897220e3ec3f", + "builder_id": "mario4tier", + "built_success": "True", + "package_md5": "11b450075ad34007cdc8f90be7ede036", + "gen_code_pass": "Disabled", + "ta_regtest_pass": "Disabled", + "dist_test_pass": "True" +} \ No newline at end of file diff --git a/scripts/package.py b/scripts/package.py index 008778ed..edc70271 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -44,7 +44,7 @@ from utilities.package_digest import PackageDigest from utilities.windows import call_vcvarsall from utilities.versions import sync_sources_digest, sync_versions -from utilities.common import are_generated_files_git_changed, compare_dir, copy_file_list, create_temp_dir, get_git_bot_user_name, get_git_user_name, get_src_generated_files, is_arm64_toolchain_installed, is_cmake_installed, is_debian_based, is_dotnet_installed, is_i386_toolchain_installed, is_nightly_github_action, is_redhat_based, is_rpmbuild_installed, is_ubuntu, is_dotnet_installed, is_wix_installed, is_x86_64_toolchain_installed, run_command, run_command_term, verify_git_repo, run_command_sudo +from utilities.common import are_generated_files_git_changed, compare_dir, copy_file_list, create_temp_dir, get_git_bot_user_name, get_git_user_name, get_src_generated_files, is_arm64_toolchain_installed, is_cmake_installed, is_debian_based, is_dotnet_installed, is_i386_toolchain_installed, is_msbuild_installed, is_nightly_github_action, is_redhat_based, is_rpmbuild_installed, is_ubuntu, is_dotnet_installed, is_wix_installed, is_x86_64_toolchain_installed, run_command, run_command_term, verify_git_repo, run_command_sudo from utilities.files import compare_msi_files, compare_tar_gz_files, compare_zip_files, create_rtf_from_txt, create_zip_file, compare_deb_files, force_delete, force_delete_glob, path_join def delete_other_versions(target_dir: str, file_pattern: str, new_version: str ): @@ -139,13 +139,25 @@ def find_asset_with_ext(target_dir, version: str, extension: str) -> str: return os.path.basename(filepath) -def package_windows_zip(root_dir: str, version: str, platform: str) -> dict: - result: dict = {"processed": True, "build_valid": False} - - file_name_prefix = f'ta-lib-{version}-windows-{platform}' - asset_file_name = f'{file_name_prefix}.zip' +def package_windows_zip(root_dir: str, asset_file_name: str, version: str, platform: str) -> dict: + result: dict = {"build_valid": False} result["asset_file_name"] = asset_file_name + # Validate the asset_file_name + if not asset_file_name.endswith('.zip'): + print(f"Error: Invalid asset_file_name {asset_file_name}. Expected a .zip file.") + return + if version not in asset_file_name: + print(f"Error: Invalid asset_file_name {asset_file_name}. Expected version {version}.") + return + + file_name_prefix = asset_file_name[:-4] + + # Check dependencies. + if not is_msbuild_installed(): + print("Error: MSBuild not found. It is required to build the package.") + sys.exit(1) + # Clean-up dist_dir = path_join(root_dir, 'dist') delete_other_versions(dist_dir,"ta-lib-*.zip",version) @@ -155,6 +167,16 @@ def package_windows_zip(root_dir: str, version: str, platform: str) -> dict: force_delete_glob(temp_dir, "ta-lib-*") + # Delete previous package digest files. + digests_dir = os.path.join(dist_dir, 'digests') + delete_other_versions(digests_dir, "*.digest", version) + + if is_build_skipping_allowed(root_dir, asset_file_name, version, sources_digest, builder_id): + result["build_valid"] = True + result["existed"] = True + result["copied"] = False + return result + # Build the libraries build_dir = do_cmake_reconfigure(root_dir, '-G Ninja -DBUILD_DEV_TOOLS=OFF -DCMAKE_BUILD_TYPE=Release') do_cmake_build(build_dir) @@ -204,7 +226,8 @@ def package_windows_zip(root_dir: str, version: str, platform: str) -> dict: print(f"Error creating zip file: {e}") return - # TODO Add testing of the temporary package here. + # TODO Add some real "end-user installation" testing. Now just pretend is is OK... + result["dist_test_pass"] = True # Temporary zip is verified OK, so copy it into dist, but only if its content is different. os.makedirs(dist_dir, exist_ok=True) @@ -221,8 +244,30 @@ def package_windows_zip(root_dir: str, version: str, platform: str) -> dict: result["copied"] = package_copied return result -def package_windows_msi(root_dir: str, version: str, platform: str, force: bool) -> dict: +def package_windows_msi(root_dir: str, asset_file_name: str, version: str, platform: str, force: bool) -> dict: result: dict = {"build_valid": False} + result["asset_file_name"] = asset_file_name + + # Validate the asset_file_name + if not asset_file_name.endswith('.msi'): + print(f"Error: Invalid asset_file_name {asset_file_name}. Expected a .msi file.") + return + if version not in asset_file_name: + print(f"Error: Invalid asset_file_name {asset_file_name}. Expected version {version}.") + return + + # Check dependencies. + if not is_msbuild_installed(): + print("Error: MSBuild not found. It is required to build the package.") + sys.exit(1) + + if not is_dotnet_installed(): + print("Error: .NET Framework not found. It is required to build the MSI.") + return result + + if not is_wix_installed(): + print("Error: WiX Toolset not found. It is required to build the MSI.") + return result # Clean-up dist_dir = path_join(root_dir, 'dist') @@ -233,26 +278,35 @@ def package_windows_msi(root_dir: str, version: str, platform: str, force: bool) force_delete_glob(temp_dir, "ta-lib-*") + # Delete previous package digest files. + digests_dir = os.path.join(dist_dir, 'digests') + delete_other_versions(digests_dir, "*.digest", version) + + if is_build_skipping_allowed(root_dir, asset_file_name, version, sources_digest, builder_id): + result["build_valid"] = True + result["existed"] = True + result["copied"] = False + return result + # MSI supports only .rtf for license, so generate it into root_dir. license_txt = path_join(root_dir,"LICENSE") license_rtf = path_join(root_dir,"LICENSE.rtf") create_rtf_from_txt(license_txt,license_rtf) - if not is_dotnet_installed(): - print("Error: .NET Framework not found. It is required to build the MSI.") - return result - - if not is_wix_installed(): - print("Error: WiX Toolset not found. It is required to build the MSI.") - return result - build_dir = do_cmake_reconfigure(root_dir, '-G Ninja -DCPACK_GENERATOR=WIX -DBUILD_DEV_TOOLS=OFF -DCMAKE_BUILD_TYPE=Release') do_cmake_build(build_dir) # Build the libraries do_cpack_build(build_dir) # Create the .msi - asset_file_name = find_asset_with_ext(build_dir, version, "msi") - temp_dist_file = path_join(build_dir, asset_file_name) + built_asset_file_name = find_asset_with_ext(build_dir, version, "msi") + if built_asset_file_name != asset_file_name: + print(f"Error: Expected file name {asset_file_name}, but got {built_asset_file_name}") + return result + # TODO Add some real "end-user installation" testing. Now just pretend is is OK... + result["dist_test_pass"] = True + + # Temporary msi is verified OK, so copy it into dist, but only if its content is different. + temp_dist_file = path_join(build_dir, asset_file_name) dist_dir = path_join(root_dir, 'dist') dist_file = path_join(dist_dir, asset_file_name) package_existed = os.path.exists(dist_file) @@ -264,8 +318,7 @@ def package_windows_msi(root_dir: str, version: str, platform: str, force: bool) os.rename(temp_dist_file, dist_file) package_copied = True - result["build_valid"] = True - result["asset_file_name"] = asset_file_name + result["build_valid"] = True result["existed"] = package_existed result["copied"] = package_copied return result @@ -840,24 +893,27 @@ def package_windows_platform(root_dir: str, version: str, platform: str) -> dict elif platform == "arm_32": vcvarsall_args = ["amd64_arm"] + zip_asset_file_name = f'ta-lib-{version}-windows-{platform}.zip' + msi_asset_file_name = f'ta-lib-{version}-windows-{platform}.msi' + call_vcvarsall(root_dir, vcvarsall_args) results = { "zip_results": { "build_valid": False, "processed": False, - "asset_file_name": f"{platform}.zip", # Default, will change. + "asset_file_name": zip_asset_file_name, }, "msi_results": { "build_valid": False, "processed": False, - "asset_file_name": f"{platform}.msi", # Default, will change. + "asset_file_name": msi_asset_file_name, } } - zip_results = package_windows_zip(root_dir, version, platform) + zip_results = package_windows_zip(root_dir, zip_asset_file_name, version, platform) results["zip_results"].update(zip_results) results["zip_results"]["processed"] = True if not results["zip_results"]["build_valid"]: - print(f'Error: Packaging dist/{results["zip_results"]["asset_file_name"]} failed') + print(f'Error: Packaging dist/{zip_asset_file_name} failed') sys.exit(1) # The zip file is better at detecting if the *content* is different. @@ -873,11 +929,11 @@ def package_windows_platform(root_dir: str, version: str, platform: str) -> dict if not is_wix_installed(): print("Warning: WiX Toolset not found. MSI packaging skipped.") else: - msi_results = package_windows_msi(root_dir, version, platform, force_msi_overwrite) + msi_results = package_windows_msi(root_dir, msi_asset_file_name, version, platform, force_msi_overwrite) results["msi_results"].update(msi_results) results["msi_results"]["processed"] = True if not results["msi_results"]["build_valid"]: - print(f'Error: Packaging dist/{results["msi_results"]["asset_file_name"]} failed') + print(f'Error: Packaging dist/{msi_asset_file_name} failed') sys.exit(1) return results @@ -890,6 +946,11 @@ def package_all_windows(root_dir: str, version: str, sources_digest: str, builde #results_arm_64 = package_windows_platform(root_dir, version, "arm_64") #results_arm_32 = package_windows_platform(root_dir, version, "arm_32") + update_package_digest(root_dir, results_x86_64["zip_results"], sources_digest, builder_id) + update_package_digest(root_dir, results_x86_64["msi_results"], sources_digest, builder_id) + update_package_digest(root_dir, results_x86_32["zip_results"], sources_digest, builder_id) + update_package_digest(root_dir, results_x86_32["msi_results"], sources_digest, builder_id) + print(f"\n***********") print(f"* Summary *") print(f"***********") @@ -919,7 +980,7 @@ def package_all_windows(root_dir: str, version: str, sources_digest: str, builde builder_id = get_git_user_name() if host_platform == "linux": - package_all_linux(root_dir,version,sources_digest,builder_id,sudo_pwd) + package_all_linux(root_dir, version, sources_digest, builder_id, sudo_pwd) elif host_platform == "win32": arch = platform.architecture()[0] if arch == '64bit': diff --git a/scripts/utilities/common.py b/scripts/utilities/common.py index 718d3532..19ea5c29 100644 --- a/scripts/utilities/common.py +++ b/scripts/utilities/common.py @@ -81,9 +81,13 @@ def is_wix_installed() -> bool: return False def is_msbuild_installed() -> bool: - if sys.platform == 'Windows': + if sys.platform == "win32": try: - result = subprocess.run(['vswhere', '-latest', '-products', '*', '-requires', 'Microsoft.Component.MSBuild', '-find', 'MSBuild\\**\\Bin\\MSBuild.exe'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + vswhere_path = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" + if not os.path.exists(vswhere_path): + return False + + result = subprocess.run([vswhere_path, '-latest', '-products', '*', '-requires', 'Microsoft.Component.MSBuild', '-find', 'MSBuild\\**\\Bin\\MSBuild.exe'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) msbuild_path = result.stdout.decode().strip() if msbuild_path: subprocess.run([msbuild_path, '-version'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)