36b5e11de6
Wraps `wget` command with sleep and mutiple retry support for resilient network downloads and replaces git clone with direct tarball downloads where possible. Closes-Bug: #2098417 Change-Id: I49435f5c1a7c0628e4eb7e824322d4e1f2938588
265 lines
10 KiB
Bash
Executable File
265 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -ex
|
|
WORKDIR=$(readlink -f $0 | xargs dirname)
|
|
source ${WORKDIR}/common.sh
|
|
|
|
IRONIC_LIB_SOURCE=${IRONIC_LIB_SOURCE:-}
|
|
|
|
TC_RELEASE="15.x"
|
|
QEMU_RELEASE="5.2.0"
|
|
LSHW_RELEASE="B.02.18"
|
|
BIOSDEVNAME_RELEASE="0.7.2"
|
|
IPMITOOL_GIT_HASH="19d78782d795d0cf4ceefe655f616210c9143e62"
|
|
|
|
CHROOT_CMD="sudo chroot $BUILDDIR /usr/bin/env -i PATH=$CHROOT_PATH http_proxy=$http_proxy https_proxy=$https_proxy no_proxy=$no_proxy"
|
|
|
|
DOWNLOAD_RETRY_MAX=${DOWNLOAD_RETRY_MAX:-5}
|
|
DOWNLOAD_RETRY_DELAY=${DOWNLOAD_RETRY_DELAY:-10}
|
|
|
|
function download_with_retry() {
|
|
local source_url=$1
|
|
local destination_path=$2
|
|
local attempts=1
|
|
|
|
declare -A tar_flags=(
|
|
["tar.gz"]="z"
|
|
["tgz"]="z"
|
|
["tar.bz2"]="j"
|
|
["tbz2"]="j"
|
|
["tar.xz"]="J"
|
|
["txz"]="J"
|
|
)
|
|
|
|
echo "Downloading $source_url to $destination_path"
|
|
while [ $attempts -le $DOWNLOAD_RETRY_MAX ]; do
|
|
for ext in "${!tar_flags[@]}"; do
|
|
if [[ "$source_url" =~ \.${ext}$ ]]; then
|
|
mkdir -p "$destination_path"
|
|
if wget --timeout=30 --tries=3 -O - "$source_url" | tar -x${tar_flags[$ext]} -C "$destination_path" --strip-components=1 -f -; then
|
|
echo "Successfully downloaded $source_url on attempt $attempts"
|
|
return 0
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ ! "$source_url" =~ \.tar\.[a-z]+$ ]]; then
|
|
if wget --timeout=30 --tries=3 "$source_url" -O "${destination_path}"; then
|
|
echo "Successfully downloaded $source_url on attempt $attempts"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
echo "Download attempt $attempts failed for $source_url, retrying in $DOWNLOAD_RETRY_DELAY seconds..."
|
|
sleep $DOWNLOAD_RETRY_DELAY
|
|
attempts=$((attempts + 1))
|
|
done
|
|
|
|
echo "Failed to download $source_url after $DOWNLOAD_RETRY_MAX attempts"
|
|
return 1
|
|
}
|
|
|
|
function tce_load_with_retry() {
|
|
local package=$1
|
|
local attempts=1
|
|
|
|
echo "Loading package $package with tce-load"
|
|
while [ $attempts -le $DOWNLOAD_RETRY_MAX ]; do
|
|
if sudo chroot --userspec=$TC:$STAFF $BUILDDIR /usr/bin/env -i PATH=$CHROOT_PATH http_proxy=$http_proxy https_proxy=$https_proxy no_proxy=$no_proxy tce-load -wci $package; then
|
|
echo "Successfully loaded $package on attempt $attempts"
|
|
return 0
|
|
fi
|
|
|
|
echo "tce-load attempt $attempts failed for $package, retrying in $DOWNLOAD_RETRY_DELAY seconds..."
|
|
sleep $DOWNLOAD_RETRY_DELAY
|
|
attempts=$((attempts + 1))
|
|
done
|
|
|
|
echo "Failed to load $package with tce-load after $DOWNLOAD_RETRY_MAX attempts"
|
|
return 1
|
|
}
|
|
|
|
echo "Building tinyipa:"
|
|
|
|
# Ensure we have an extended sudo to prevent the need to enter a password over
|
|
# and over again.
|
|
sudo -v
|
|
|
|
# If an old build directory exists remove it
|
|
if [ -d "$BUILDDIR" ]; then
|
|
sudo rm -rf "$BUILDDIR"
|
|
fi
|
|
|
|
##############################################
|
|
# Download and Cache Tiny Core Files
|
|
##############################################
|
|
|
|
# Find a working TC mirror if none is explicitly provided
|
|
choose_tc_mirror
|
|
|
|
cd $WORKDIR/build_files
|
|
download_with_retry "$TINYCORE_MIRROR_URL/$TC_RELEASE/x86_64/release/distribution_files/corepure64.gz" "corepure64.gz"
|
|
download_with_retry "$TINYCORE_MIRROR_URL/$TC_RELEASE/x86_64/release/distribution_files/vmlinuz64" "vmlinuz64"
|
|
cd $WORKDIR
|
|
|
|
########################################################
|
|
# Build Required Python Dependencies in a Build Directory
|
|
########################################################
|
|
|
|
# Make directory for building in
|
|
mkdir "$BUILDDIR"
|
|
|
|
# Extract rootfs from .gz file
|
|
( cd "$BUILDDIR" && zcat $WORKDIR/build_files/corepure64.gz | sudo cpio -i -H newc -d )
|
|
|
|
# Configure mirror
|
|
sudo sh -c "echo $TINYCORE_MIRROR_URL > $BUILDDIR/opt/tcemirror"
|
|
|
|
# Download Qemu-utils, Biosdevname and IPMItool source
|
|
download_with_retry "https://download.qemu.org/qemu-${QEMU_RELEASE}.tar.xz" "${BUILDDIR}/tmp/qemu"
|
|
download_with_retry "https://github.com/lyonel/lshw/archive/refs/tags/${LSHW_RELEASE}.tar.gz" "${BUILDDIR}/tmp/lshw"
|
|
if $TINYIPA_REQUIRE_BIOSDEVNAME; then
|
|
download_with_retry "https://linux.dell.com/biosdevname/biosdevname-${BIOSDEVNAME_RELEASE}/biosdevname-${BIOSDEVNAME_RELEASE}.tar.gz" "${BUILDDIR}/tmp/biosdevname"
|
|
fi
|
|
if $TINYIPA_REQUIRE_IPMITOOL; then
|
|
git clone https://codeberg.org/IPMITool/ipmitool.git "${BUILDDIR}/tmp/ipmitool-src"
|
|
cd "${BUILDDIR}/tmp/ipmitool-src"
|
|
git reset $IPMITOOL_GIT_HASH --hard
|
|
cd -
|
|
fi
|
|
|
|
# Create directory for python local mirror
|
|
mkdir -p "$BUILDDIR/tmp/localpip"
|
|
|
|
# Download IPA and requirements
|
|
IPA_SOURCE_DIR=${IPA_SOURCE_DIR:-/opt/stack/ironic-python-agent}
|
|
cd $IPA_SOURCE_DIR
|
|
rm -rf *.egg-info
|
|
pwd
|
|
|
|
PYTHON_COMMAND="python3"
|
|
$PYTHON_COMMAND setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
|
|
|
|
ls $BUILDDIR/tmp/localpip || true
|
|
cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt
|
|
|
|
if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
|
|
IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST"
|
|
for PKGDIR in "${PKGDIRS[@]}"; do
|
|
PKG=$(cd "$PKGDIR" ; $PYTHON_COMMAND setup.py --name)
|
|
pushd "$PKGDIR"
|
|
rm -rf *.egg-info
|
|
$PYTHON_COMMAND setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
|
|
if [[ -r requirements.txt ]]; then
|
|
cp requirements.txt $BUILDDIR/tmp/${PKG}-requirements.txt
|
|
fi
|
|
popd
|
|
done
|
|
fi
|
|
|
|
$WORKDIR/generate_tox_constraints.sh upper-constraints.txt
|
|
cp upper-constraints.txt $BUILDDIR/tmp/upper-constraints.txt
|
|
echo Using upper-constraints:
|
|
cat upper-constraints.txt
|
|
cd $WORKDIR
|
|
|
|
sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf
|
|
|
|
trap "sudo umount $BUILDDIR/proc; sudo umount $BUILDDIR/dev/pts" EXIT
|
|
sudo mount --bind /proc $BUILDDIR/proc
|
|
sudo mount --bind /dev/pts $BUILDDIR/dev/pts
|
|
|
|
if [ -d /opt/stack/new ]; then
|
|
CI_DIR=/opt/stack/new
|
|
elif [ -d /opt/stack ]; then
|
|
CI_DIR=/opt/stack
|
|
else
|
|
CI_DIR=
|
|
fi
|
|
|
|
if [ -n "$CI_DIR" ]; then
|
|
# Running in CI environment, make checkouts available
|
|
$CHROOT_CMD mkdir -p $CI_DIR
|
|
for project in $(ls $CI_DIR); do
|
|
if grep -q "$project" $BUILDDIR/tmp/upper-constraints.txt &&
|
|
[ -d "$CI_DIR/$project/.git" ]; then
|
|
sudo cp -R "$CI_DIR/$project" $BUILDDIR/$CI_DIR/
|
|
fi
|
|
done
|
|
fi
|
|
|
|
$CHROOT_CMD mkdir -m777 /etc/sysconfig/tcedir
|
|
$CHROOT_CMD touch /etc/sysconfig/tcuser
|
|
$CHROOT_CMD chmod a+rwx /etc/sysconfig/tcuser
|
|
|
|
mkdir $BUILDDIR/tmp/overrides
|
|
cp $WORKDIR/build_files/fakeuname $BUILDDIR/tmp/overrides/uname
|
|
|
|
sudo cp $WORKDIR/build_files/ntpdate $BUILDDIR/bin/ntpdate
|
|
|
|
PY_REQS="buildreqs_python3.lst"
|
|
|
|
# NOTE(rpittau) change ownership of the tce info dir to prevent writing issues
|
|
sudo chown $TC:$STAFF $BUILDDIR/usr/local/tce.installed
|
|
|
|
while read line; do
|
|
tce_load_with_retry "$line"
|
|
done < <(paste $WORKDIR/build_files/$PY_REQS $WORKDIR/build_files/buildreqs.lst)
|
|
|
|
TINYIPA_PYTHON_EXE="python3.9"
|
|
|
|
PIP_COMMAND="$TINYIPA_PYTHON_EXE -m pip"
|
|
|
|
# Build python wheels
|
|
$CHROOT_CMD ${TINYIPA_PYTHON_EXE} -m ensurepip
|
|
$CHROOT_CMD ${PIP_COMMAND} install --upgrade pip==${PIP_VERSION} wheel
|
|
$CHROOT_CMD ${PIP_COMMAND} install pbr
|
|
$CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt
|
|
|
|
if [ -n "$PYTHON_EXTRA_SOURCES_DIR_LIST" ]; then
|
|
IFS="," read -ra PKGDIRS <<< "$PYTHON_EXTRA_SOURCES_DIR_LIST"
|
|
for PKGDIR in "${PKGDIRS[@]}"; do
|
|
PKG=$(cd "$PKGDIR" ; $PYTHON_COMMAND setup.py --name)
|
|
if [[ -r $BUILDDIR/tmp/${PKG}-requirements.txt ]]; then
|
|
$CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/${PKG}-requirements.txt
|
|
fi
|
|
$CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ${PKG}
|
|
done
|
|
fi
|
|
|
|
$CHROOT_CMD ${PIP_COMMAND} wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
|
|
echo Resulting wheels:
|
|
ls -1 $BUILDDIR/tmp/wheels
|
|
|
|
# Build qemu-utils
|
|
rm -rf $WORKDIR/build_files/qemu-utils.tcz
|
|
$CHROOT_CMD /bin/sh -c "cd /tmp/qemu && CFLAGS=-Wno-error ./configure --disable-system --disable-user --disable-linux-user --disable-bsd-user --disable-guest-agent --disable-blobs --enable-tools --python=/usr/local/bin/$TINYIPA_PYTHON_EXE && make && make install DESTDIR=/tmp/qemu-utils"
|
|
find $BUILDDIR/tmp/qemu-utils/ -type f -executable | xargs file | awk -F ':' '/ELF/ {print $1}' | sudo xargs strip
|
|
cd $WORKDIR/build_files && mksquashfs $BUILDDIR/tmp/qemu-utils qemu-utils.tcz && md5sum qemu-utils.tcz > qemu-utils.tcz.md5.txt
|
|
# Create qemu-utils.tcz.dep
|
|
echo "glib2.tcz" > qemu-utils.tcz.dep
|
|
|
|
# Build lshw
|
|
rm -rf $WORKDIR/build_files/lshw.tcz
|
|
# NOTE(mjturek): We touch src/lshw.1 and clear src/po/Makefile to avoid building the man pages, as they aren't used and require large dependencies to build.
|
|
$CHROOT_CMD /bin/sh -c "cd /tmp/lshw && touch src/lshw.1 && echo install: > src/po/Makefile && make && make install DESTDIR=/tmp/lshw-installed"
|
|
find $BUILDDIR/tmp/lshw-installed/ -type f -executable | xargs file | awk -F ':' '/ELF/ {print $1}' | sudo xargs strip
|
|
cd $WORKDIR/build_files && mksquashfs $BUILDDIR/tmp/lshw-installed lshw.tcz && md5sum lshw.tcz > lshw.tcz.md5.txt
|
|
|
|
# Build biosdevname
|
|
if $TINYIPA_REQUIRE_BIOSDEVNAME; then
|
|
rm -rf $WORKDIR/build_files/biosdevname.tcz
|
|
$CHROOT_CMD /bin/sh -c "cd /tmp/biosdevname-* && ./configure && make && make install DESTDIR=/tmp/biosdevname-installed"
|
|
find $BUILDDIR/tmp/biosdevname-installed/ -type f -executable | xargs file | awk -F ':' '/ELF/ {print $1}' | sudo xargs strip
|
|
cd $WORKDIR/build_files && mksquashfs $BUILDDIR/tmp/biosdevname-installed biosdevname.tcz && md5sum biosdevname.tcz > biosdevname.tcz.md5.txt
|
|
fi
|
|
|
|
if $TINYIPA_REQUIRE_IPMITOOL; then
|
|
rm -rf $WORKDIR/build_files/ipmitool.tcz
|
|
# NOTE(TheJulia): Explicitly add the libtool path since /usr/local/ is not in path from the chroot.
|
|
$CHROOT_CMD /bin/sh -c "cd /tmp/ipmitool-src && env LIBTOOL='/usr/local/bin/libtool' ./bootstrap && ./configure && make && make install DESTDIR=/tmp/ipmitool"
|
|
find $BUILDDIR/tmp/ipmitool/ -type f -executable | xargs file | awk -F ':' '/ELF/ {print $1}' | sudo xargs strip
|
|
cd $WORKDIR/build_files && mksquashfs $BUILDDIR/tmp/ipmitool ipmitool.tcz && md5sum ipmitool.tcz > ipmitool.tcz.md5.txt
|
|
fi
|