From 88b201b37f710975563ed48be1b702b7839d4304 Mon Sep 17 00:00:00 2001 From: richardg867 Date: Tue, 15 Mar 2022 19:52:17 -0300 Subject: [PATCH] Jenkins: Update build script with Windows dependency management, FAudio on Linux --- .ci/build.sh | 174 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 147 insertions(+), 27 deletions(-) diff --git a/.ci/build.sh b/.ci/build.sh index 493f42789..e9f9075a4 100644 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -34,7 +34,7 @@ # # Define common functions. -alias is_windows='[ ! -z "$MSYSTEM" ]' +alias is_windows='[ -n "$MSYSTEM" ]' alias is_mac='uname -s | grep -q Darwin' make_tar() { @@ -62,7 +62,7 @@ make_tar() { fi # Make tar verbose if requested. - [ ! -z "$VERBOSE" ] && local compression_flag="$compression_flag -v" + [ -n "$VERBOSE" ] && local compression_flag="$compression_flag -v" # tar is notorious for having many diverging implementations. For instance, # the flags we use to strip UID/GID metadata can be --owner/group (GNU), @@ -136,7 +136,7 @@ done cmake_flags_extra= # Check if mandatory arguments were specified. -if [ -z "$package_name" -a -z "$tarball_name" ] || [ ! -z "$package_name" -a -z "$arch" ] +if [ -z "$package_name" -a -z "$tarball_name" ] || [ -n "$package_name" -a -z "$arch" ] then echo '[!] Usage: build.sh -b {package_name} {architecture} [-t] [cmake_flags...]' echo ' build.sh -s {source_tarball_name}' @@ -147,7 +147,7 @@ fi cd "$(dirname "$0")/.." # Make source tarball if requested. -if [ ! -z "$tarball_name" ] +if [ -n "$tarball_name" ] then echo [-] Making source tarball [$tarball_name] @@ -211,6 +211,106 @@ then fi echo [-] Using MSYSTEM [$MSYSTEM] + # Update keyring, as the package signing keys sometimes change. + echo [-] Updating package databases and keyring + yes | pacman -Sy --needed msys2-keyring + + # Query installed packages. + pacman -Qe > pacman.txt + + # Download the specified versions of architecture-specific dependencies. + echo -n [-] Downloading dependencies: + pkg_dir="/var/cache/pacman/pkg" + repo_base="https://repo.msys2.org/mingw/$(echo $MSYSTEM | tr '[:upper:]' '[:lower:]')" + pkgs="" + while IFS=" " read pkg version + do + prefixed_pkg="$MINGW_PACKAGE_PREFIX-$pkg" + installed_version=$(grep -E "^$prefixed_pkg " pacman.txt | cut -d " " -f 2) + if [ "$installed_version" != "$version" ] # installed_version will be empty if not installed + then + echo -n " [$pkg" + + # Download package if not already present in the local cache. + pkg_tar="$prefixed_pkg-$version-any.pkg.tar" + if [ -s "$pkg_dir/$pkg_tar.xz" ] + then + pkg_fn="$pkg_tar.xz" + pkg_dest="$pkg_dir/$pkg_fn" + else + pkg_fn="$pkg_tar.zst" + pkg_dest="$pkg_dir/$pkg_fn" + if [ ! -s "$pkg_dest" ] + then + if ! wget -qO "$pkg_dest" "$repo_base/$pkg_fn" + then + rm -f "$pkg_dest" + pkg_fn="$pkg_tar.xz" + pkg_dest="$pkg_dir/$pkg_fn" + wget -qO "$pkg_dest" "$repo_base/$pkg_fn" || rm -f "$pkg_dest" + fi + if [ -s "$pkg_dest" ] + then + wget -qO "$pkg_dest.sig" "$repo_base/$pkg_fn.sig" || rm -f "$pkg_dest.sig" + [ ! -s "$pkg_dest.sig" ] && rm -f "$pkg_dest.sig" + fi + fi + fi + + # Check if the cached package is valid. + if [ -s "$pkg_dest" ] + then + # Add cached zst package. + pkgs="$pkgs $pkg_fn" + else + # Not valid, remove if it exists. + rm -f "$pkg_dest" "$pkg_dest.sig" + echo -n " FAIL" + fi + echo -n "]" + fi + done < <(cat .ci/dependencies_msys.txt | tr -d '\r') + [ -z "$pkgs" ] && echo -n none required + echo + + # Install the downloaded architecture-specific dependencies. + echo [-] Installing dependencies through pacman + if [ -n "$pkgs" ] + then + pushd "$pkg_dir" + yes | pacman -U --needed $pkgs + if [ $? -ne 0 ] + then + # Install packages individually if installing them all together failed. + for pkg in pkgs + do + yes | pacman -U --needed "$pkg" + done + fi + popd + + # Query installed packages again. + pacman -Qe > pacman.txt + fi + + # Install the latest versions for any missing packages (if the specified version couldn't be installed). + pkgs="make git" + while IFS=" " read pkg version + do + prefixed_pkg="$MINGW_PACKAGE_PREFIX-$pkg" + grep -qE "^$prefixed_pkg " pacman.txt || pkgs="$pkgs $prefixed_pkg" + done < <(cat .ci/dependencies_msys.txt | tr -d '\r') + rm -f pacman.txt + yes | pacman -S --needed $pkgs + if [ $? -ne 0 ] + then + # Install packages individually if installing them all together failed. + for pkg in pkgs + do + yes | pacman -S --needed "$pkg" + done + fi + # Point CMake to the toolchain file. cmake_flags_extra="$cmake_flags_extra -D \"CMAKE_TOOLCHAIN_FILE=cmake/$toolchain.cmake\"" elif is_mac @@ -227,7 +327,7 @@ else esac # Establish general dependencies. - pkgs="cmake pkg-config git imagemagick wget p7zip-full wayland-protocols tar gzip" + pkgs="cmake pkg-config git imagemagick wget p7zip-full wayland-protocols tar gzip file" if [ "$(dpkg --print-architecture)" = "$arch_deb" ] then pkgs="$pkgs build-essential" @@ -242,7 +342,7 @@ else # ...and the ones we do want listed. Non-dev packages fill missing spots on the list. libpkgs="" longest_libpkg=0 - for pkg in libc6-dev libstdc++6 libopenal-dev libfreetype6-dev libx11-dev libsdl2-dev libpng-dev librtmidi-dev qtdeclarative5-dev libwayland-dev libevdev-dev libglib2.0-dev libslirp-dev + for pkg in libc6-dev libstdc++6 libopenal-dev libfreetype6-dev libx11-dev libsdl2-dev libpng-dev librtmidi-dev qtdeclarative5-dev libwayland-dev libevdev-dev libglib2.0-dev libslirp-dev libfaudio-dev libaudio-dev libjack-jackd2-dev libpipewire-0.3-dev libsamplerate0-dev libsndio-dev do libpkgs="$libpkgs $pkg:$arch_deb" length=$(echo -n $pkg | sed 's/-dev$//' | sed "s/qtdeclarative/qt/" | wc -c) @@ -298,9 +398,6 @@ EOF # Link against the system libslirp instead of compiling ours. cmake_flags_extra="$cmake_flags_extra -D SLIRP_EXTERNAL=ON" - - # Use OpenAL for Linux builds before FAudio builds are set up. - cmake_flags_extra="$cmake_flags_extra -D OPENAL=ON" fi # Clean workspace. @@ -327,16 +424,16 @@ if [ "$CI" = "true" ] then # Backup strategy when running under Jenkins. [ -z "$git_hash" ] && git_hash=$(echo $GIT_COMMIT | cut -c 1-8) -elif [ ! -z "$git_hash" ] +elif [ -n "$git_hash" ] then # Append + to denote a dirty tree. git diff --quiet 2> /dev/null || git_hash="$git_hash+" fi -[ ! -z "$git_hash" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_GIT_HASH=$git_hash\"" +[ -n "$git_hash" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_GIT_HASH=$git_hash\"" # Add copyright year. year=$(date +%Y) -[ ! -z "$year" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_COPYRIGHT_YEAR=$year\"" +[ -n "$year" ] && cmake_flags_extra="$cmake_flags_extra -D \"EMU_COPYRIGHT_YEAR=$year\"" # Run CMake. echo [-] Running CMake with flags [$cmake_flags $cmake_flags_extra] @@ -432,19 +529,42 @@ then else cwd_root=$(pwd) - # Build openal-soft 1.21.1 manually to fix audio issues. This is a temporary - # workaround until a newer version of openal-soft trickles down to Debian repos. - if [ -d "openal-soft-1.21.1" ] + if grep -q "OPENAL:BOOL=ON" build/CMakeCache.txt then - rm -rf openal-soft-1.21.1/build/* + # Build openal-soft 1.21.1 manually to fix audio issues. This is a temporary + # workaround until a newer version of openal-soft trickles down to Debian repos. + if [ -d "openal-soft-1.21.1" ] + then + rm -rf openal-soft-1.21.1/build/* + else + wget -qO - https://github.com/kcat/openal-soft/archive/refs/tags/1.21.1.tar.gz | tar zxf - + fi + cd openal-soft-1.21.1/build + [ -e Makefile ] && make clean + cmake -G "Unix Makefiles" -D "CMAKE_TOOLCHAIN_FILE=$cwd_root/toolchain.cmake" -D "CMAKE_INSTALL_PREFIX=$cwd_root/archive_tmp/usr" .. + make -j$(nproc) install || exit 99 + cd "$cwd_root" + + # Build SDL2 without sound systems. + sdl_ss=OFF else - wget -qO - https://github.com/kcat/openal-soft/archive/refs/tags/1.21.1.tar.gz | tar zxf - + # Build FAudio 22.03 manually to remove the dependency on GStreamer. This is a temporary + # workaround until a newer version of FAudio trickles down to Debian repos. + if [ -d "FAudio-22.03" ] + then + rm -rf FAudio-22.03/build + else + wget -qO - https://github.com/FNA-XNA/FAudio/archive/refs/tags/22.03.tar.gz | tar zxf - + fi + mkdir FAudio-22.03/build + cd FAudio-22.03/build + cmake -G "Unix Makefiles" -D "CMAKE_TOOLCHAIN_FILE=$cwd_root/toolchain.cmake" -D "CMAKE_INSTALL_PREFIX=$cwd_root/archive_tmp/usr" .. + make -j$(nproc) install || exit 99 + cd "$cwd_root" + + # Build SDL2 with sound systems. + sdl_ss=ON fi - cd openal-soft-1.21.1/build - [ -e Makefile ] && make clean - cmake -G "Unix Makefiles" -D "CMAKE_TOOLCHAIN_FILE=$cwd_root/toolchain.cmake" -D "CMAKE_INSTALL_PREFIX=$cwd_root/archive_tmp/usr" .. - make -j$(nproc) install || exit 99 - cd "$cwd_root" # Build rtmidi without JACK support to remove the dependency on libjack. if [ -d "rtmidi-4.0.0" ] @@ -468,10 +588,10 @@ else rm -rf sdlbuild mkdir sdlbuild cd sdlbuild - cmake -G "Unix Makefiles" -D SDL_DISKAUDIO=OFF -D SDL_DIRECTFB_SHARED=OFF -D SDL_OPENGL=OFF -D SDL_OPENGLES=OFF -D SDL_OSS=OFF -D SDL_ALSA=OFF \ - -D SDL_ALSA_SHARED=OFF -D SDL_JACK=OFF -D SDL_JACK_SHARED=OFF -D SDL_ESD=OFF -D SDL_ESD_SHARED=OFF -D SDL_PIPEWIRE=OFF -D SDL_PIPEWIRE_SHARED=OFF \ - -D SDL_PULSEAUDIO=OFF -D SDL_PULSEAUDIO_SHARED=OFF -D SDL_ARTS=OFF -D SDL_ARTS_SHARED=OFF -D SDL_NAS=OFF -D SDL_NAS_SHARED=OFF -D SDL_SNDIO=OFF \ - -D SDL_SNDIO_SHARED=OFF -D SDL_FUSIONSOUND=OFF -D SDL_FUSIONSOUND_SHARED=OFF -D SDL_LIBSAMPLERATE=OFF -D SDL_LIBSAMPLERATE_SHARED=OFF -D SDL_X11=OFF \ + cmake -G "Unix Makefiles" -D SDL_DISKAUDIO=OFF -D SDL_DIRECTFB_SHARED=OFF -D SDL_OPENGL=OFF -D SDL_OPENGLES=OFF -D SDL_OSS=OFF -D SDL_ALSA=$sdl_ss \ + -D SDL_ALSA_SHARED=$sdl_ss -D SDL_JACK=$sdl_ss -D SDL_JACK_SHARED=$sdl_ss -D SDL_ESD=OFF -D SDL_ESD_SHARED=OFF -D SDL_PIPEWIRE=$sdl_ss -D SDL_PIPEWIRE_SHARED=$sdl_ss \ + -D SDL_PULSEAUDIO=$sdl_ss -D SDL_PULSEAUDIO_SHARED=$sdl_ss -D SDL_ARTS=OFF -D SDL_ARTS_SHARED=OFF -D SDL_NAS=$sdl_ss -D SDL_NAS_SHARED=$sdl_ss -D SDL_SNDIO=$sdl_ss \ + -D SDL_SNDIO_SHARED=$sdl_ss -D SDL_FUSIONSOUND=OFF -D SDL_FUSIONSOUND_SHARED=OFF -D SDL_LIBSAMPLERATE=$sdl_ss -D SDL_LIBSAMPLERATE_SHARED=$sdl_ss -D SDL_X11=OFF \ -D SDL_X11_SHARED=OFF -D SDL_WAYLAND=OFF -D SDL_WAYLAND_SHARED=OFF -D SDL_WAYLAND_LIBDECOR=OFF -D SDL_WAYLAND_LIBDECOR_SHARED=OFF \ -D SDL_WAYLAND_QT_TOUCH=OFF -D SDL_RPI=OFF -D SDL_VIVANTE=OFF -D SDL_VULKAN=OFF -D SDL_KMSDRM=OFF -D SDL_KMSDRM_SHARED=OFF -D SDL_OFFSCREEN=OFF \ -D SDL_HIDAPI_JOYSTICK=ON -D SDL_VIRTUAL_JOYSTICK=ON -D SDL_SHARED=ON -D SDL_STATIC=OFF -S "$cwd_root/SDL2-2.0.20" \ @@ -544,7 +664,7 @@ else project_version=$(grep -oP '#define\s+EMU_VERSION\s+"\K([^"]+)' "build/src/include/$project_lower/version.h" 2> /dev/null) [ -z "$project_version" ] && project_version=unknown build_num=$(grep -oP '#define\s+EMU_BUILD_NUM\s+\K([0-9]+)' "build/src/include/$project_lower/version.h" 2> /dev/null) - [ ! -z "$build_num" -a "$build_num" != "0" ] && project_version="$project_version-b$build_num" + [ -n "$build_num" -a "$build_num" != "0" ] && project_version="$project_version-b$build_num" # Download appimage-builder if necessary. [ ! -e "appimage-builder.AppImage" ] && wget -qO appimage-builder.AppImage \