JBR-4451 Make bash scripts safer

This commit is contained in:
Vladislav Rassokhin
2022-05-18 02:24:51 +07:00
committed by jbrbot
parent 32d46af9f4
commit b941d12dfa
19 changed files with 288 additions and 40 deletions

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# This script creates a Docker image suitable for building AArch64 variant
# of the JetBrains Runtime "dev" version.

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# This script creates a Docker image suitable for building musl AArch64 variant
# of the JetBrains Runtime version 17.

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# This script creates a Docker image suitable for building musl-x64 variant
# of the JetBrains Runtime version 17.

View File

@@ -1,12 +1,15 @@
#!/bin/bash -x
#!/bin/bash
function do_maketest() {
set -euo pipefail
set -x
function check_bundle_type_maketest() {
# check whether last char is 't', if so remove it
if [ "${bundle_type: -1}" == "t" ]; then
echo ${bundle_type%?}
return 1
bundle_type="${bundle_type%?}"
do_maketest=1
else
echo ${bundle_type}
return 0
do_maketest=0
fi
}
@@ -16,20 +19,23 @@ function getVersionProp() {
while getopts ":i?" o; do
case "${o}" in
i)
i="incremental build"
INC_BUILD=1
;;
i) INC_BUILD=1 ;;
esac
done
shift $((OPTIND-1))
if [[ $# -lt 2 ]]; then
echo "Required at least two arguments: build_number bundle_type"
exit 1
fi
build_number=$1
bundle_type=$2
architecture=$3 # aarch64 or x64
# shellcheck disable=SC2034
architecture=${3:-x64} # aarch64 or x64
check_bundle_type_maketest
bundle_type=$(do_maketest)
do_maketest=$?
tag_prefix="jdk-"
OPENJDK_TAG=$(git log --simplify-by-decoration --decorate=short --pretty=short | grep "$tag_prefix" | cut -d "(" -f2 | cut -d ")" -f1 | awk '{print $2}' | tr -d ',' | sort -t "-" -k 2 -g | tail -n 1)
VERSION_FEATURE=$(getVersionProp "DEFAULT_VERSION_FEATURE")
@@ -61,6 +67,10 @@ export TZ
SOURCE_DATE_EPOCH="$(git log -1 --pretty=%ct)"
export SOURCE_DATE_EPOCH
COPYRIGHT_YEAR=""
BUILD_TIME=""
TOUCH_TIME=""
REPRODUCIBLE_TAR_OPTS=""
case "$OS_NAME" in
Linux)
COPYRIGHT_YEAR="$(date --utc --date=@$SOURCE_DATE_EPOCH +%Y)"

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -54,6 +57,7 @@ function create_image_bundle {
__modules=$4
libc_type_suffix=''
fastdebug_infix=''
if is_musl; then libc_type_suffix='musl-' ; fi
@@ -109,7 +113,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
make clean CONF=$RELEASE_NAME || do_exit $?
fi
@@ -129,6 +133,8 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
cp $JCEF_PATH/jmods/* $JSDK_MODS_DIR # $JSDK/jmods is not changed
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -54,6 +57,7 @@ function create_image_bundle {
__modules=$4
libc_type_suffix=''
fastdebug_infix=''
if is_musl; then libc_type_suffix='musl-' ; fi
@@ -109,7 +113,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
make clean CONF=$RELEASE_NAME || do_exit $?
fi
@@ -130,6 +134,8 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
jbr_name_postfix="_${bundle_type}"
[ "$bundle_type" != "fd" ] && jbrsdk_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -43,6 +46,7 @@ function create_image_bundle {
__modules=$4
libc_type_suffix=''
fastdebug_infix=''
if is_musl; then libc_type_suffix='musl-' ; fi
@@ -97,7 +101,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
make clean CONF=$RELEASE_NAME || do_exit $?
fi
@@ -113,6 +117,8 @@ chmod -R a+r $JSDK
if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -20,7 +23,6 @@
source jb/project/tools/common/scripts/common.sh
JCEF_PATH=${JCEF_PATH:=./jcef_mac}
architecture=${architecture:=x64}
BOOT_JDK=${BOOT_JDK:=$(/usr/libexec/java_home -v 17)}
function do_configure {
@@ -71,11 +73,13 @@ function create_image_bundle {
__modules_path=$3
__modules=$4
fastdebug_infix=''
tmp=.bundle.$$.tmp
mkdir "$tmp" || do_exit $?
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
JBR=${__bundle_name}-${JBSDK_VERSION}-osx-${architecture}-${fastdebug_infix}b${build_number}
JBR=${__bundle_name}-${JBSDK_VERSION}-osx-${architecture}-${fastdebug_infix:-}b${build_number}
JRE_CONTENTS=$tmp/$__arch_name/Contents
mkdir -p "$JRE_CONTENTS" || do_exit $?
@@ -134,7 +138,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
make clean CONF=$RELEASE_NAME || do_exit $?
fi
@@ -152,6 +156,8 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
cp $JCEF_PATH/jmods/* $JSDK_MODS_DIR # $JSDK/jmods is not changed
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
APP_DIRECTORY=$1
APPL_USER=$2

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
APPLICATION_PATH=$1
APP_NAME=$2

View File

@@ -1,7 +1,8 @@
#!/bin/bash -x
#!/bin/bash
#immediately exit script with an error if a command fails
set -euo pipefail
set -x
export COPY_EXTENDED_ATTRIBUTES_DISABLE=true
export COPYFILE_DISABLE=true

View File

@@ -0,0 +1,153 @@
#!/bin/bash
set -euo pipefail
while getopts ":t" o; do
case "${o}" in
t)
t="With Teamcity tests info"
TC_PRINT=1
;;
esac
done
shift $((OPTIND-1))
NEWFILEPATH=$1
CONFIGID=$2
BUILDID=$3
TOKEN=$4
#
# Get the size of new artifact
#
unameOut="$(uname -s)"
case "${unameOut}" in
Linux*)
NEWFILESIZE=$(stat -c%s "$NEWFILEPATH")
;;
Darwin*)
NEWFILESIZE=$(stat -f%z "$NEWFILEPATH")
;;
CYGWIN*)
NEWFILESIZE=$(stat -c%s$4
#
# Get the size of new artifact
#
"$NEWFILEPATH")
;;
MINGW*)
NEWFILESIZE=$(stat -c%s "$NEWFILEPATH")
;;
*)
echo "Unknown machine: ${unameOut}"
exit 1
esac
FILENAME=$(basename ${NEWFILEPATH})
#
# Get pattern of artifact name
# Base filename pattern: <BUNDLE_TYPE>-<JDK_VERSION>-<OS>-<ARCH>-b<BUILD>.tar.gz: jbr_dcevm-17.0.2-osx-x64-b1234.tar.gz
# BUNDLE_TYPE: jbr, jbrsdk, jbr_dcevm, jbrsdk_jcef etc.
# OS_ARCH_PATTERN - <os_architecture>: osx-x64, linux-aarch64, windows-x64 etc.
BUNDLE_TYPE=jbrsdk
OS_ARCH_PATTERN=""
FILE_EXTENSION=tar.gz
re='(jbr[a-z_]*).*-[0-9_\.]+-(.+)-b[0-9]+(.+)'
if [[ $FILENAME =~ $re ]]; then
BUNDLE_TYPE=${BASH_REMATCH[1]}
OS_ARCH_PATTERN=${BASH_REMATCH[2]}
FILE_EXTENSION=${BASH_REMATCH[3]}
fi
if [ $TC_PRINT -eq 1 ]; then
testname_file_ext=`echo $FILE_EXTENSION | sed 's/\./_/g'`
testname=$BUNDLE_TYPE"_"$OS_ARCH_PATTERN$testname_file_ext
echo \#\#teamcity[testStarted name=\'$testname\']
fi
echo "BUNDLE_TYPE: " $BUNDLE_TYPE
echo "OS_ARCH_PATTERN: " $OS_ARCH_PATTERN
echo "FILE_EXTENSION: " $FILE_EXTENSION
echo "New size of $FILENAME = $NEWFILESIZE bytes."
function test_failed_msg() {
if [ $3 -eq 1 ]; then
echo \#\#teamcity[testFailed name=\'$1\' message=\'$2\']
fi
}
function test_finished_msg() {
if [ $2 -eq 1 ]; then
echo \#\#teamcity[testFinished name=\'$1\']
fi
}
#
# Get previous successful build ID
# Example:
# CONFIGID=IntellijCustomJdk_Jdk17_Master_LinuxX64jcef
# BUILDID=12345678
#
# expected return value
# id="123".number="567"
#
CURL_RESPONSE=$(curl --header "Authorization: Bearer $TOKEN" "https://buildserver.labs.intellij.net/app/rest/builds/?locator=buildType:(id:$CONFIGID),status:success,count:1,finishDate:(build:$BUILDID,condition:before)")
re='id=\"([0-9]+)\".+number=\"([0-9\.]+)\"'
# ID: Previous successful build id
ID=0
if [[ $CURL_RESPONSE =~ $re ]]; then
ID=${BASH_REMATCH[1]}
echo "BUILD Number: ${BASH_REMATCH[2]}"
else
msg="ERROR: can't find previous build"
echo $msg
echo $CURL_RESPONSE
test_failed_msg $testname $msg $TC_PRINT
test_finished_msg $testname $TC_PRINT
exit 1
fi
#
# Get artifacts from previous successful build
#
# expected return value
# name="jbrsdk_jcef*.tar.gz size="123'
#
CURL_RESPONSE=$(curl --header "Authorization: Bearer $TOKEN" "https://buildserver.labs.intellij.net/app/rest/builds/$ID?fields=id,number,artifacts(file(name,size))")
echo "Atrifacts of previous build of $CONFIGID :"
echo $CURL_RESPONSE
# Find binary size (in response) with reg exp
re='name=\"('$BUNDLE_TYPE'[^\"]+'${OS_ARCH_PATTERN}'[^\"]+'${FILE_EXTENSION}')\" size=\"([0-9]+)\"'
if [[ $CURL_RESPONSE =~ $re ]]; then
OLDFILENAME=${BASH_REMATCH[1]}
echo "Prev artifact name: $OLDFILENAME"
OLDFILESIZE=${BASH_REMATCH[2]}
echo "Prev artifact size = $OLDFILESIZE"
let allowedSize=OLDFILESIZE+OLDFILESIZE/20 # use 5% threshold
echo "Allowed size = $allowedSize"
if [[ "$NEWFILESIZE" -gt "$allowedSize" ]]; then
msg="ERROR: new size is significally greater than prev size (need to investigate)"
echo $msg
test_failed_msg $testname $msg $TC_PRINT
test_finished_msg $testname $TC_PRINT
exit 1
else
echo "PASSED"
test_finished_msg $testname $TC_PRINT
fi
else
msg="ERROR: can't find string with size in xml response:"
echo $msg
echo $CURL_RESPONSE
test_failed_msg $testname $msg $TC_PRINT
test_finished_msg $testname $TC_PRINT
exit 1
fi

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
usage ()
{

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -46,6 +49,8 @@ function create_image_bundle {
__modules_path=$3
__modules=$4
fastdebug_infix=''
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
echo Running jlink ...
@@ -80,7 +85,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
if [ $do_maketest -eq 1 ]; then
make LOG=info CONF=$RELEASE_NAME clean images test-image || do_exit $?
@@ -111,6 +116,8 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
cp $JCEF_PATH/jmods/* $JSDK_MODS_DIR # $JSDK/jmods is not unchanged
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -44,6 +47,8 @@ function create_image_bundle {
__modules_path=$3
__modules=$4
fastdebug_infix=''
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
echo Running jlink ...
@@ -78,7 +83,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
if [ $do_maketest -eq 1 ]; then
make LOG=info CONF=$RELEASE_NAME clean images test-image || do_exit $?
@@ -109,6 +114,8 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
cp $JCEF_PATH/jmods/* ${JSDK_MODS_DIR} # $JSDK/jmods is not unchanged
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -29,8 +32,8 @@ function do_configure {
--with-toolchain-version=$TOOLCHAIN_VERSION \
--with-boot-jdk=$BOOT_JDK \
--disable-ccache \
$STATIC_CONF_ARGS \
--enable-cds=yes \
$STATIC_CONF_ARGS \
$REPRODUCIBLE_BUILD_OPTS \
|| do_exit $?
}
@@ -41,6 +44,8 @@ function create_image_bundle {
__modules_path=$3
__modules=$4
fastdebug_infix=''
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
echo Running jlink ...
@@ -74,7 +79,7 @@ case "$bundle_type" in
;;
esac
if [ -z "$INC_BUILD" ]; then
if [ -z "${INC_BUILD:-}" ]; then
do_configure || do_exit $?
if [ $do_maketest -eq 1 ]; then
make LOG=info CONF=$RELEASE_NAME clean images test-image || do_exit $?
@@ -100,6 +105,8 @@ if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "fd" ]; then
cp $JCEF_PATH/jmods/* ${JSDK_MODS_DIR} # $JSDK/jmods is not unchanged
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
# create runtime image bundle

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -19,6 +22,8 @@ function pack_jbr {
__bundle_name=$1
__arch_name=$2
fastdebug_infix=''
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
JBR=${__bundle_name}-${JBSDK_VERSION}-windows-aarch64-${fastdebug_infix}b${build_number}
@@ -36,6 +41,8 @@ BASE_DIR=.
if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "dcevm" ] || [ "$bundle_type" == "fd" ]; then
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
pack_jbr jbr${jbr_name_postfix} jbr

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -19,6 +22,8 @@ function pack_jbr {
__bundle_name=$1
__arch_name=$2
fastdebug_infix=''
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
JBR=${__bundle_name}-${JBSDK_VERSION}-windows-x64-${fastdebug_infix}b${build_number}
@@ -36,6 +41,8 @@ BASE_DIR=.
if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "dcevm" ] || [ "$bundle_type" == "fd" ]; then
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
pack_jbr jbr${jbr_name_postfix} jbr

View File

@@ -1,4 +1,7 @@
#!/bin/bash -x
#!/bin/bash
set -euo pipefail
set -x
# The following parameters must be specified:
# build_number - specifies the number of JetBrainsRuntime build
@@ -15,6 +18,8 @@ function pack_jbr {
__bundle_name=$1
__arch_name=$2
fastdebug_infix=''
[ "$bundle_type" == "fd" ] && [ "$__arch_name" == "$JBRSDK_BUNDLE" ] && __bundle_name=$__arch_name && fastdebug_infix="fastdebug-"
JBR=${__bundle_name}-${JBSDK_VERSION}-windows-x86-${fastdebug_infix}b${build_number}
@@ -32,6 +37,8 @@ BASE_DIR=.
if [ "$bundle_type" == "jcef" ] || [ "$bundle_type" == "dcevm" ] || [ "$bundle_type" == "fd" ]; then
jbr_name_postfix="_${bundle_type}"
else
jbr_name_postfix=""
fi
pack_jbr jbr${jbr_name_postfix} jbr