Impoving UTF-8 BOM handling for collection info in lib2inpx

Moving to go 1.20, droppng deprecated packages (pkg/errors, ioutils) and updating all dependencies. Dropping support for 32 bit Windows build
rupor-github · Mar 15, 2023 · 5d66a3b · 5d66a3b
1 parent 8979e9b
commit 5d66a3b
Show file tree

Hide file tree

Showing 27 changed files with 119 additions and 827 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,45 +11,22 @@
 *.log
 *.svg
 
-*.[568vq]
-[568vq].out
-
-.undodir/
-.glide/
-.idea/
-.vscode/
-build/
-bin/
-_obj/
-_test/
-
 # build specific
 src/*/misc/version.go
 
-pkg/
-!src/**/pkg/
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-_testmain.go
-.session.vim
-.set-project-env
-conf/mmb_*
+lib2inpx-*.7z
+lib2inpx-*.zip
+lib2inpx-*.xz
+lib2inpx-*.gz
 
-# Our staff
-
-lib2inpx-win32.7z
-lib2inpx-win64.7z
-lib2inpx-linux_i686-glibc_*.tar.xz
-lib2inpx-linux_x86_64-glibc_*.tar.xz
+bin/
+build/
+[Bb]in_*/
+[Dd]ebug*/
+[Rr]elease*/
 
-[Bb]in_win32
-[Bb]in_win64
-[Bb]in_linux_i686
-[Bb]in_linux_x86_64
+mariadb-??.?.*-linux-glibc_*-x86_64.tar.gz
+lib2inpx-*.tar.xz
 
 mariadb-??.?.*-linux-glibc_*-x86_64.tar.gz
 mariadb-??.?.*-linux-glibc_*-x86_64/**
@@ -68,16 +45,3 @@ mysql-?.?.*-win32.zip
 mysql-?.?.*-win32/**
 mysql-?.?.*-winx64.zip
 mysql-?.?.*-winx64/**
-
-[Dd]ebug32/
-[Dd]ebug64/
-[Dd]ebug_win32/
-[Dd]ebug_win64/
-[Rr]elease32/
-[Rr]elease64/
-[Rr]elease_win32/
-[Rr]elease_win64/
-[Dd]ebug_linux_i686/
-[Dd]ebug_linux_x86_64/
-[Rr]elease_linux_i686/
-[Rr]elease_linux_x86_64/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,7 +13,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
 
 # Project version number
 set(PRJ_VERSION_MAJOR 9)
-set(PRJ_VERSION_MINOR 45)
+set(PRJ_VERSION_MINOR 46)
 
 if(WIN32 AND NOT DEFINED ENV{MSYSTEM})
     message(FATAL_ERROR "Currently unsuppored environment. Use MINGW for Windows builds. CMake willl exit now.")
@@ -49,6 +49,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     message(STATUS "** Debug mode")
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -D_XML_DEBUG")
     set(TEST_RUN_ARGS "-v")
+    set(GO_MOD_ARGS "-mod=mod")
 else()
     set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -s")
     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s")
@@ -84,7 +85,8 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJECT_BINARY_DIR}/inpx
 configure_file("${PROJECT_SOURCE_DIR}/cmake/version.h.in" "${PROJECT_BINARY_DIR}/version.h")
 
 set(Boost_USE_STATIC_LIBS ON)
-set(Boost_USE_STATIC_RUNTIME ON)
+# set(Boost_USE_STATIC_RUNTIME ON)
+
 find_package(Boost 1.58.0 REQUIRED COMPONENTS "date_time" "program_options" "filesystem")
 find_package(Threads)
 if(DEFINED ENV{MYSQL_PROPER})
@@ -152,7 +154,7 @@ install(FILES ${MYSQL_LIB_DIR}/../share/english/errmsg.sys DESTINATION ${PROJECT
 # go targets are always out of date, "go build" will figure it out
 ########################################################################################################
 
-find_package(Go 1.13 REQUIRED)
+find_package(Go 1.20 REQUIRED)
 
 if(CMAKE_CROSSCOMPILING)
     # Set Go target environments
@@ -165,11 +167,20 @@ if(CMAKE_CROSSCOMPILING)
     endif()
 endif()
 
+# Make sure we setup go properly
+if (DEFINED ENV{GOPATH})
+    set(GO_PATH "$ENV{GOPATH}")
+else()
+    set(GO_PATH "/tmp/gopkg_${CMAKE_PROJECT_NAME}")
+    execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${GO_PATH})
+endif()
+message(STATUS "Using go modules with GOPATH=${GO_PATH}")
+
 configure_file("${PROJECT_SOURCE_DIR}/cmake/version.go.in" "${PROJECT_SOURCE_DIR}/src/inpxcreator/misc/version.go")
 
 add_custom_target(bin_libget2 ALL
     DEPENDS gen_git_ver
-    COMMAND GOPATH=${PROJECT_SOURCE_DIR} GO111MODULE=on GOOS=${CROSS_GOOS} GOARCH=${CROSS_GOARCH}
+    COMMAND GOPATH=${GO_PATH} GOOS=${CROSS_GOOS} GOARCH=${CROSS_GOARCH} CGO_ENABLED=0
         ${GO_EXECUTABLE} build -o ${PROJECT_BINARY_DIR}/libget2${CMAKE_EXECUTABLE_SUFFIX}
         -ldflags='-X main.LastGitCommit=\$\(shell cat ${PROJECT_BINARY_DIR}/inpxc_git_ver\)'
         ${GO_MOD_ARGS}
@@ -180,7 +191,7 @@ install(PROGRAMS ${PROJECT_BINARY_DIR}/libget2${CMAKE_EXECUTABLE_SUFFIX} DESTINA
 
 add_custom_target(bin_libmerge ALL
     DEPENDS gen_git_ver
-    COMMAND GOPATH=${PROJECT_SOURCE_DIR} GO111MODULE=on GOOS=${CROSS_GOOS} GOARCH=${CROSS_GOARCH}
+    COMMAND GOPATH=${GO_PATH} GOOS=${CROSS_GOOS} GOARCH=${CROSS_GOARCH} CGO_ENABLED=0
         ${GO_EXECUTABLE} build -o ${PROJECT_BINARY_DIR}/libmerge${CMAKE_EXECUTABLE_SUFFIX}
         -ldflags='-X main.LastGitCommit=\$\(shell cat ${PROJECT_BINARY_DIR}/inpxc_git_ver\)'
         ${GO_MOD_ARGS}
@@ -193,30 +204,16 @@ install(PROGRAMS ${PROJECT_BINARY_DIR}/libmerge${CMAKE_EXECUTABLE_SUFFIX} DESTIN
 # Go dependency management
 ########################################################################################################
 
-add_custom_target(deps_print
-    SOURCES ${PROJECT_SOURCE_DIR}/src/inpxcreator/go.mod
-    COMMAND GOPATH=${PROJECT_SOURCE_DIR} GO111MODULE=on ${GO_EXECUTABLE} mod graph
-    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/src/inpxcreator"
-    COMMENT "Printing dependencies (go mod graph)"
-    VERBATIM)
-
 add_custom_target(deps_tidy
     SOURCES ${PROJECT_SOURCE_DIR}/src/inpxcreator/go.mod
-    COMMAND GOPATH=${PROJECT_SOURCE_DIR} GO111MODULE=on ${GO_EXECUTABLE} mod tidy
+    COMMAND GOPATH=${GO_PATH} ${GO_EXECUTABLE} mod tidy -go=${GO_LANG_VERSION}
     WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/src/inpxcreator"
     COMMENT "Cleaning dependencies (go mod tidy)"
     VERBATIM)
 
-add_custom_target(deps_verify
-    SOURCES ${PROJECT_SOURCE_DIR}/src/inpxcreator/go.mod
-    COMMAND GOPATH=${PROJECT_SOURCE_DIR} GO111MODULE=on ${GO_EXECUTABLE} mod verify
-    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/src/inpxcreator"
-    COMMENT "Verifying dependencies (go mod verify)"
-    VERBATIM)
-
 add_custom_target(deps_vendor
     SOURCES ${PROJECT_SOURCE_DIR}/src/inpxcreator/go.mod
-    COMMAND GOPATH=${PROJECT_SOURCE_DIR} GO111MODULE=on ${GO_EXECUTABLE} mod vendor
+    COMMAND GOPATH=${GO_PATH} ${GO_EXECUTABLE} mod vendor
     WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/src/inpxcreator"
     COMMENT "Creating vendor directory (go mod vendor)"
     VERBATIM)
diff --git a/build-release.sh b/build-release.sh
@@ -33,7 +33,7 @@ RED='[38;5;01m'
 YELLOW='[38;5;03m'
 
 readonly ALL_OFF BOLD BLUE GREEN RED YELLOW
-ARCH_INSTALLS="${ARCH_INSTALLS:-win32 win64 linux}"
+ARCH_INSTALLS="${ARCH_INSTALLS:-win64 linux}"
 
 if [ ! -d ${HOME}/result ]; then
 	print_error "directory for the result is not available. Exiting..."

diff --git a/cmake/FindGlide.cmake b/cmake/FindGlide.cmake
diff --git a/cmake/FindGo.cmake b/cmake/FindGo.cmake
@@ -5,44 +5,33 @@
 # The module defines the following variables:
 #   GO_FOUND - true if the Go was found
 #   GO_EXECUTABLE - path to the executable
+#   GOFMT_EXECUTABLE - path to gofmt executable
 #   GO_VERSION - Go version number
+#   GO_LANG_VERSION - Go version number without patch, only major and minor versions. Used with "go mod tidy"
 #   GO_PLATFORM - i.e. linux
 #   GO_ARCH - i.e. amd64
 # Example usage:
 #   find_package(Go 1.2 REQUIRED)
 
-find_program(GO_EXECUTABLE go PATHS $ENV{HOME}/go ENV GOROOT GOPATH PATH_SUFFIXES bin)
+find_program(GO_EXECUTABLE go PATHS ENV GOROOT GOPATH PATH_SUFFIXES bin)
 if(GO_EXECUTABLE)
-    if(NOT CMAKE_CROSSCOMPILING AND WIN32 AND DEFINED ENV{MSYSTEM})
-        if(NOT GO_ROOT)
-            if($ENV{MSYSTEM} STREQUAL "MINGW32")
-                set(GO_ROOT "/mingw32/lib/go")
-            elseif($ENV{MSYSTEM} STREQUAL "MINGW64")
-                set(GO_ROOT "/mingw64/lib/go")
-            else()
-                message(FATAL_ERROR "Currently unsuppored environment. CMake willl exit now.")
-            endif()
-        endif()
-    else()
-	if (DEFINED ENV{GOROOT})
- 		set(GO_ROOT "$ENV{GOROOT}")
-	else()
- 		set(GO_ROOT "/usr/lib/go")
-	endif()
-    endif()
-    execute_process(COMMAND sh -c "${GO_EXECUTABLE} version" OUTPUT_VARIABLE GO_VERSION_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if(GO_VERSION_OUTPUT MATCHES "go([0-9]+\\.[0-9]+\\.?[0-9]*)[a-zA-Z0-9]* ([^/]+)/(.*)")
+	execute_process(COMMAND ${GO_EXECUTABLE} version OUTPUT_VARIABLE GO_VERSION_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(GO_VERSION_OUTPUT MATCHES "go version go([0-9]+\\.[0-9]+\\.?[0-9]*)[a-zA-Z0-9]* ([^/]+)/(.*)")
         set(GO_VERSION ${CMAKE_MATCH_1})
         set(GO_PLATFORM ${CMAKE_MATCH_2})
         set(GO_ARCH ${CMAKE_MATCH_3})
+        string(REGEX MATCH "([0-9]+\\.[0-9]+)" GO_LANG_VERSION ${GO_VERSION})
     elseif(GO_VERSION_OUTPUT MATCHES "go version devel .* ([^/]+)/(.*)$")
         set(GO_VERSION "99-devel")
+        set(GO_LANG_VERSION, "99-devel")
         set(GO_PLATFORM ${CMAKE_MATCH_1})
         set(GO_ARCH ${CMAKE_MATCH_2})
         message("WARNING: Development version of Go being used, can't determine compatibility.")
     endif()
+    get_filename_component(path ${GO_EXECUTABLE} DIRECTORY)
+    set(GOFMT_EXECUTABLE ${path}/gofmt${CMAKE_EXECUTABLE_SUFFIX})
 endif()
 mark_as_advanced(GO_EXECUTABLE)
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Go REQUIRED_VARS GO_EXECUTABLE GO_VERSION GO_PLATFORM GO_ARCH VERSION_VAR GO_VERSION)
+find_package_handle_standard_args(Go REQUIRED_VARS GO_EXECUTABLE GOFMT_EXECUTABLE GO_PLATFORM GO_ARCH GO_VERSION GO_LANG_VERSION VERSION_VAR GO_VERSION)
diff --git a/cmake/FindGodep.cmake b/cmake/FindGodep.cmake
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
@@ -1,47 +1,41 @@
 FROM ubuntu:16.04
 
+# Synology is still running very old glibc - 2.5 as of v7. So I have to use old verion of Ubuntu for linux build...
+
 RUN apt-get -qqy update &&\
 apt-get -qqy install tzdata wget curl build-essential git git-extras automake pkg-config libcurl4-openssl-dev zlib1g-dev libaio-dev libboost-all-dev p7zip-full gdb mingw-w64 binutils-mingw-w64 mingw-w64-tools libz-mingw-w64-dev
 
 WORKDIR /root/
 
 # ---------- We either patch cmake to support up to date boost (/usr/share/cmake-3.5/Modules/FindBoost.cmake) or build fresh one
-RUN curl -LO https://cmake.org/files/v3.15/cmake-3.15.4.tar.gz \
-    && tar -xvf cmake-3.15.4.tar.gz \
-    && cd cmake-3.15.4 \
+RUN curl -LO https://cmake.org/files/v3.25/cmake-3.25.3.tar.gz \
+    && tar -xvf cmake-3.25.3.tar.gz \
+    && cd cmake-3.25.3 \
     && ./bootstrap --system-curl \
     && make install \
     && cd .. \
-    && rm -rf cmake-3.15.4.tar.gz cmake-3.15.4
+    && rm -rf cmake-3.25.3.tar.gz cmake-3.25.3
 
 # ---------- mingw does not have expat
 RUN git clone https://github.com/libexpat/libexpat.git \
     && cd libexpat \
-    && git checkout R_2_2_6 \
+    && git checkout R_2_5_0 \
     && cd expat \
     && ./buildconf.sh \
-    && ./configure --host=i686-w64-mingw32 --prefix=/usr/i686-w64-mingw32 --without-xmlwf --without-docbook \
-    && make install \
-    && make clean \
     && ./configure --host=x86_64-w64-mingw32 --prefix=/usr/x86_64-w64-mingw32 --without-xmlwf --without-docbook \
     && make install \
     && cd ../.. \
     && rm -rf libexpat \
-    && rm /usr/i686-w64-mingw32/lib/*.la \
     && rm /usr/x86_64-w64-mingw32/lib/*.la
 
 # ---------- mingw does not have static iconv
-RUN curl -LO https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz \
-    && tar -xvf libiconv-1.15.tar.gz \
-    && cd libiconv-1.15 \
-    && ./configure --host=i686-w64-mingw32 --prefix=/usr/i686-w64-mingw32 --enable-static \
-    && make install \
-    && make clean \
+RUN curl -LO https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.17.tar.gz \
+    && tar -xvf libiconv-1.17.tar.gz \
+    && cd libiconv-1.17 \
     && ./configure --host=x86_64-w64-mingw32 --prefix=/usr/x86_64-w64-mingw32 --enable-static \
     && make install \
     && cd .. \
-    && rm -rf libiconv-1.15.tar.gz libiconv-1.15 \
-    && rm /usr/i686-w64-mingw32/lib/*.la \
+    && rm -rf libiconv-1.17.tar.gz libiconv-1.17 \
     && rm /usr/x86_64-w64-mingw32/lib/*.la
 
 # ---------- mingw does not have boost (and we cannot cross build anything older than 1.63)
@@ -64,9 +58,6 @@ RUN ln -s /usr/bin/x86_64-w64-mingw32-windres /usr/bin/windres \
     && ./b2 --without-python --without-serialization --user-config=user-config.jam --debug-configuration toolset=gcc-5.3.1 link=shared,static target-os=windows address-model=64 pch=off threading=multi threadapi=win32 variant=release --prefix=/usr/x86_64-w64-mingw32 --layout=tagged --without-python --without-mpi -sICONV_PATH=/usr/x86_64-w64-mingw32 -sICONV_LINK="-L/usr/x86_64-w64-mingw32/lib -liconv" install \
     && ./b2 --without-python --without-serialization --user-config=user-config.jam --debug-configuration toolset=gcc-5.3.1 link=shared,static target-os=windows address-model=64 pch=off threading=multi threadapi=win32 variant=release --prefix=/usr/x86_64-w64-mingw32 --layout=tagged --without-python --without-mpi -sICONV_PATH=/usr/x86_64-w64-mingw32 -sICONV_LINK="-L/usr/x86_64-w64-mingw32/lib -liconv" clean \
     && rm -rf bin.v2 stage \
-    && echo "using gcc : 5.3.1 : i686-w64-mingw32-g++ ;" > user-config.jam \
-    && ./b2 --user-config=user-config.jam --debug-configuration toolset=gcc-5.3.1 link=shared,static target-os=windows address-model=32 pch=off threading=multi threadapi=win32 variant=release --prefix=/usr/i686-w64-mingw32 --layout=tagged --without-python --without-mpi --without-serialization -sICONV_PATH=/usr/i686-w64-mingw32 -sICONV_LINK="-L/usr/i686-w64-mingw32/lib -liconv" install \
-    && ./b2 --user-config=user-config.jam --debug-configuration toolset=gcc-5.3.1 link=shared,static target-os=windows address-model=32 pch=off threading=multi threadapi=win32 variant=release --prefix=/usr/i686-w64-mingw32 --layout=tagged --without-python --without-mpi --without-serialization -sICONV_PATH=/usr/i686-w64-mingw32 -sICONV_LINK="-L/usr/i686-w64-mingw32/lib -liconv" clean \
     && cd .. \
     && rm -rf boost_1_63_0.tar.gz boost_1_63_0
 
@@ -76,9 +67,9 @@ RUN ./get-maria.sh
 
 #  ---------- download golang
 
-RUN curl -LO https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz \
-   && tar -xvf go1.14.2.linux-amd64.tar.gz \
-   && rm go1.14.2.linux-amd64.tar.gz \
+RUN curl -LO https://go.dev/dl/go1.20.2.linux-amd64.tar.gz \
+   && tar -xvf go1.20.2.linux-amd64.tar.gz \
+   && rm go1.20.2.linux-amd64.tar.gz \
    && sed -i -e '$a export PATH=$PATH:/root/go/bin' .bashrc \
    && sed -i -e '$a export GOROOT=/root/go' .bashrc
 

diff --git a/docker/container.txt b/docker/container.txt
@@ -1,14 +1,4 @@
-docker build --squash -t inpxc:latest . -f ./Dockerfile.build
-
-docker save -o ../../inpxc.tar inpxc
-docker load -i ../../inpxc.tar
-
-docker run --name inpxcreator -v E:/projects/books/InpxCreator:/root/result -it inpxc:latest
-docker run -e TZ=America/New_York --name inpxcreator -v E:/projects/books/InpxCreator:/root/result -it inpxc:latest /bin/bash
-
---------------------------------------
-
 podman build -t inpxc:latest . -f ./Dockerfile.build
 
-podman run -it --name inpxcreator -v /mnt/e/projects/books/InpxCreator:/root/result --rm inpxc:latest
-podman run -it -e TZ=America/New_York --name inpxcreator -v /mnt/e/projects/books/InpxCreator:/root/result inpxc:latest /bin/bash
+podman run -it --network host -v ~/projects/github.com/rupor-github:/root/result --rm inpxc:latest
+podman run -it --network host -e TZ=America/New_York -v  ~/projects/github.com/rupor-github/InpxCreator:/root/result inpxc:latest /bin/bash
diff --git a/docker/get-maria.sh b/docker/get-maria.sh
@@ -8,7 +8,7 @@ ver_minor=26
 declare -A checkmap
 from_url="https://downloads.mariadb.com/MariaDB"
 
-ARCH_INSTALLS="${ARCH_INSTALLS:-win32 win64 linux}"
+ARCH_INSTALLS="${ARCH_INSTALLS:-win64 linux}"
 
 for _mingw in ${ARCH_INSTALLS}; do
 	case ${_mingw} in