path: root/test-manifest2.sh

#!/bin/bash

set -e

export LANG=C

# some defaults
GIT_REF_DIR=`pwd`/ref-snapshots/
ABE_BRANCH=
ABE_PATH="" # if empty, then it gets a local checkout below
DISPLAY_REPORT=false
DISPLAY_SUMMARY=false
QUIET=false
ONLY_CHECK_REFERENCE=false
BUILD_TARGET=aarch64-linux-gnu

validate_path()
{
    if ! readlink -e $1; then
        echo "Path $1 does not exist" >&2
        exit 1
    fi
}

read_var()
{
   local artifact_list=$1
   local var=$2
   grep "^${var}=" "${artifact_list}" | cut -d = -f 2-
}


# parse command line options
while [[ $# -gt 0 ]]; do
   OPT=$1
   shift
   case "$OPT" in
     --abe-path) ABE_PATH="`validate_path $1`"; shift;;
     --ref-snapshots) GIT_REF_DIR="`validate_path $1`"; shift;;
     --abe-branch) ABE_BRANCH=$1; shift;;
     --abe-flags) ABE_FLAGS=$1; shift;;
     --abe-configure-flags) ABE_CONF_FLAGS=$1; shift;;
     --target) BUILD_TARGET=$1; shift;;
     --display[_-]report) DISPLAY_REPORT=true;;
     --display[_-]summary) DISPLAY_SUMMARY=true;;
     --only-check-reference) ONLY_CHECK_REFERENCE=true;;
     --quiet) QUIET=true;;
     --debug) set -x;;
     *) echo "Unrecognised option: $OPT" >&2; exit 1;;
   esac
done

# find the location of the abe-tests directory
ABE_TESTS_PATH=`perl -e 'use File::Basename; use Cwd q(abs_path); print dirname(abs_path(shift))' "$0"`

# suppress return code of 1 when there are no matches, to avoid spurious shell
# aborts due to set -e
xgrep()
{
    (
        set +e
        grep "$@"
        RESULT=$?
        if [ $? -lt 2 ]; then
            exit 0
        else
            exit 2
        fi
     )
}

# if --quiet was used, then we hide most of the output of abe using this
# function.
log_to_file()
{
    if $QUIET; then
        tee "$1" | grep --line-buffered -P '^(NOTE|WARNING|ERROR|FIXME):'
    else
        tee "$1"
    fi
}

# if the date rolls over during the test, we can end up with log differences
# because the timestamp is used in filenames. Assuming that this test takes
# less than 24 hours, we can just convert tomorrow's date to today's date to
# hide log differences when this happens.
TODAY="`date +%Y%m%d`"
TOMORROW="`date -d tomorrow +%Y%m%d`"

postprocess_log()
{
   DIRNAME=$1
   sed -e 's@/'"$DIRNAME"'/@/<builddir>/@g' | sed -e 's@\(/[^ ./]*.git\)[^ /]*@\1<stuff>@g; s@builds/tmp.[0-9]*\>@builds/tmp.<pid>@g; '"s/$TODAY/<date>/g; s/$TOMORROW/<date>/g"
}

result_pass()
{
    echo PASSED: "$*" | tee --append $TOP/results_summary.txt $TOP/results_report.txt
}

result_fail()
{
    EXTRA_FILE=$1
    shift
    echo FAILED: "$* See $(basename $EXTRA_FILE)." | tee --append $TOP/results_summary.txt $TOP/results_report.txt
    sed -e 's/^/>>> /' "$EXTRA_FILE" >> $TOP/results_report.txt

}

result_fail_no_file()
{
    echo FAILED: "$*" | tee --append $TOP/results_summary.txt $TOP/results_report.txt
}

extract_run()
{
   FILE=$1
   DIRNAME=$2
   LINE=$(grep -n '^NOTE: Checkout all took' "$FILE" | head -n 1 | cut -f1 -d:)
   # we can ignore the manifest files warning here because the comparison is
   # less sophisticated than the one in this test and reports false positives
   if [ -z "$LINE" ]; then LINE=1; fi
   tail -n "+$LINE" "$FILE" | xgrep -E '^(DRYRUN:|RUN:|ERROR[ (]|FIXME\(|WARNING:) ' | postprocess_log "$DIRNAME" | xgrep -v '^WARNING: Manifest files are different'
}

compare_run_lines()
{
    FILE1=$1
    FILE2=$2
    OUTPUT=$3
    (
        set +e
        diff -u "$FILE1" "$FILE2" > "$OUTPUT"
        if [ $? -eq 0 ]; then
            result_pass run lines diff.
        else
            result_fail "$OUTPUT" run lines diff.
        fi
    )
}

extract_revisions()
{
   FILE=$1
   perl -e 'while (<>) { if (m/^NOTE: Configuring (.*?) *$/) { $component = $1; } if (m@^(?:DRY)?RUN:.*/bin/bash ('"$SNAPSHOTS"'/([^/]+\.git)[^ ]+/)configure SHELL=/bin/bash@) { print "$component: $2 "; system("cd $1 && git rev-parse HEAD") } }' "$FILE"
}

compare_revisions()
{
    FILE1=$1
    FILE2=$2
    OUTPUT=$3
    WHEN=$3
    (
    set +e
    diff -u "$FILE1" "$FILE2" > "$OUTPUT"
    if [ $? -eq 0 ]; then
        result_pass revisions diff.
    else
        result_fail "$OUTPUT" "revisions list differs $WHEN".
    fi
    )
}

add_extra_revisions()
{
   FILE=$1
   perl -e 'while (<>) { if (m@^RUN:.*/bin/bash ('"$SNAPSHOTS"'/([^/]+\.git)[^ ]+/)configure SHELL=/bin/bash@) { print "$2: "; system(qq(cd $1 && touch abe-test-dummy && git add abe-test-dummy && git -c user.name="ABE Tests" commit -m dummy)); } }' "$FILE"
}

check_revisions_in_manifest()
{
   PHASE=$1
   REVISIONS=$2
   MANI=$3
   REPORT=$4
   FAILED=false
   while read; do
     # We get input in the form:
     #   COMPONENT[optional SUBCOMPONENT]: REPO.git SHA1
     # Extract COMPONENT:
     COMPONENT=`echo "$REPLY" | sed -e 's/:.*//; s/ .*//'`
     REV=`echo $REPLY | sed -e 's/.* //'`
     if ! grep -q "${COMPONENT}_revision=$REV" "$MANI"; then
       FAILED=true
       echo "${COMPONENT}_revision=$REV not found in $(basename $MANI)" >> "$REPORT" 2>&1
     fi
  done < "$REVISIONS"
  if $FAILED; then
    result_fail "$REPORT" "Build revisions did not match manifest revisions in phase $PHASE."
  else
    result_pass "Build revisions matched manifest revisions in phase $PHASE."
  fi
}

strip_unused_content_from_manifest()
{
    IN="$1"
    OUT="$2"
    grep -B1000000 -F -x ' # Everything below this line is only for informational purposes for developers' <"$IN" >"$OUT"
}

scan_manifest_for_build_dir()
{
    FILE=$1
    BUILDDIR=$2
    PHASE=$3
    DIR_REFS=$TOP/scan_builddir$PHASE.txt
    if grep -F "$BUILDDIR" "$FILE" > $DIR_REFS; then
        result_fail $TOP/$DIR_REFS Phase $PHASE manifest contains reference to build dir.
      else
        result_pass Phase $PHASE manifest does not contain references to build dir.
    fi
}

wrap_abe()
{
    PHASE=$1
    ABE_LOG_FILE=$2
    shift 2
    (
        export ORIG_PATH="$PATH"
        # we handle errors explicitly here
        set +e
        set -o pipefail
        PATH="$ABE_TESTS_PATH/wrappers:$PATH"
        export REPORT_FILE_TEMPLATE=$TOP/wrap-failures-$PHASE/
        mkdir -p "$REPORT_FILE_TEMPLATE"
        "$@" 2>&1 | log_to_file "$ABE_LOG_FILE"
        if [ $? -eq 0 ]; then
            result_pass Phase $PHASE abe succeeded.
          else
            tail -n 100 "$ABE_LOG_FILE" > "$ABE_LOG_FILE".tail
            result_fail "$ABE_LOG_FILE".tail Phase $PHASE abe returned non-zero exit status.
        fi
    )
}

check_wrap_fails()
{
    PHASE=$1
    echo "======= CHECKING FOR FAILURES IN WRAPPED COMMANDS (PHASE $PHASE)"
    (
        FAILED=0
        shopt -s nullglob
        for i in $TOP/wrap-failures-$PHASE/*; do
            if [ -f "$i" ]; then
                cmd=`basename $i`
                result_fail_no_file "Phase $PHASE use of $cmd resulted in non-zero return code."
                FAILED=1
            fi
        done
        if [ $FAILED -eq 0 ]; then
            result_pass "Phase $PHASE monitored commands did not fail."
        fi
    )
}

check_log()
{
    PHASE=$1
    PATTERN=$2
    INFILE=$3
    OUTFILE=$4
    MSG_FAIL=$5
    MSG_PASS=$6
    (
        set +e
        grep "$PATTERN" "$INFILE" > "$OUTFILE"
        if [ $? -ne 1 ]; then
            result_fail $OUTFILE Phase $PHASE $MSG_FAIL.
        else
            result_pass Phase $PHASE $MSG_PASS
        fi
   )

}

check_artifacts()
{
    local PHASE=$1
    local ARTIFACTS=$2
    local error

    error=0
    while read; do
        local object=$(echo "${REPLY}" | cut -d = -f 1)
        local path=$(echo "${REPLY}" | cut -d = -f 2-)
        
        if [ "${path:0:1}" != "/" ]; then
            error=1
            echo "Artifact '${object}' has relative path '${path}'" >> $TOP/artifact-errors$PHASE.txt
        fi
        if [ ! -f ${path} ]; then
            error=1
            echo "Artifact '${object}' has non-existent path '${path}'" >> $TOP/artifact-errors$PHASE.txt
        fi
    done < "${ARTIFACTS}"

    if [ "${error}" -eq 0 ]; then
        result_pass Phase $PHASE Artifacts.
    else
        result_fail $TOP/artifact-errors$PHASE.txt Phase $PHASE Artifacts are invalid.
    fi
}

if [ -d "$GIT_REF_DIR" ]; then
    GIT_REF_OPT=--with-git-reference-dir=$GIT_REF_DIR
    echo "======= FOUND GIT REFERENCE DIR $GIT_REF_DIR"
  else
    unset GIT_REF_OPT
fi

rm -Rf full-manifest-test
mkdir -p full-manifest-test

cd full-manifest-test

TOP="`pwd`"
SNAPSHOTS="$TOP/snapshots"
# for speed when making tarballs
export XZ_OPT=-0

# make sure that some failure is reported if set -e kills the script.
trap '{
    if [ -z "$REACHED_THE_END" ]; then
        result_fail_no_file Abe-tests encountered unhandled error condition.
    fi
}' EXIT


if [[ -z "$ABE_PATH" ]]; then
  git clone git://git.linaro.org/toolchain/abe
  ABE_PATH=$TOP/abe
  (cd $ABE_PATH && git checkout "${ABE_BRANCH:-master}")
else
  if [[ ! -z "$ABE_BRANCH" ]]; then
    echo "--abe-branch and --abe-path cannot be used together" >&1
    exit 1
  fi
fi

if ! $ONLY_CHECK_REFERENCE; then

echo ======= MANIFEST TEST PHASE 1

mkdir -p build
pushd build
echo "======= CONFIGURE ABE (PHASE 1)"
$ABE_PATH/configure ${GIT_REF_OPT:+"$GIT_REF_OPT"} ${ABE_CONF_FLAGS:+$ABE_CONF_FLAGS}
echo "======= BUILD TOOLCHAIN (PHASE 1)"

wrap_abe 1 $TOP/test-mani-build1.txt				\
	$ABE_PATH/abe.sh --snapshots $SNAPSHOTS			\
	--list-artifacts "artifacts1.txt"			\
	--target "$BUILD_TARGET"				\
	--build all						\
	${ABE_FLAGS:+$ABE_FLAGS}				\
	--tarbin

popd

echo "======= TOOLCHAIN BUILD COMPLETE (PHASE 1)"

echo "======= FINDING GIT REVISIONS (PHASE 1)"
extract_revisions $TOP/test-mani-build1.txt > $TOP/revisions1.txt

echo "======= ADDING EXTRA REVISIONS (PHASE 1)"
add_extra_revisions $TOP/test-mani-build1.txt

echo "======= VALIDATING ARTIFACTS (PHASE 1)"
check_artifacts 1 $TOP/build/artifacts1.txt

echo "======= FINDING MANIFEST FROM PHASE 1"

mkdir -p test
pushd test
TARBALL="$(read_var $TOP/build/artifacts1.txt toolchain)"
tar axf "$TARBALL"
MANIFEST="$(read_var $TOP/build/artifacts1.txt manifest)"
cp "$MANIFEST" "$TOP/orig-manifest1.txt"
popd

echo "======= FOUND MANIFEST AT $MANIFEST (PHASE 1)"

echo "======= RENAMING BUILD DIRECTORY"
mv build build.orig

echo "======= VALIDATING MANIFEST FORMAT (PHASE 1)"
if $ABE_TESTS_PATH/validate-manifest.pl "$TOP/orig-manifest1.txt" &> "$TOP/validate1.txt"; then
    result_pass phase 1 manifest validated successfully.
else
    result_fail "$TOP/validate1.txt" phase 1 manifest did not validate.
fi

echo "======= CREATING STRIPPED MANIFEST (PHASE 1)"
strip_unused_content_from_manifest "$TOP/orig-manifest1.txt" "$TOP/stripped-manifest1.txt"

echo "======= MANIFEST TEST PHASE 2"

mkdir -p build2
pushd build2
echo "======= CONFIGURE ABE (PHASE 2)"
$ABE_PATH/configure ${GIT_REF_OPT:+"$GIT_REF_OPT"} ${ABE_CONF_FLAGS:+$ABE_CONF_FLAGS}
echo "======= BUILD TOOLCHAIN (PHASE 2)"
wrap_abe 2 $TOP/test-mani-build2.txt					\
	$ABE_PATH/abe.sh --disable update --snapshots $SNAPSHOTS	\
	--list-artifacts "artifacts2.txt"			\
	--manifest "$TOP/stripped-manifest1.txt"			\
	--build all --tarbin

popd

echo "======= TOOLCHAIN BUILD COMPLETE (PHASE 2)"

echo "======= FINDING GIT REVISIONS (PHASE 2)"
extract_revisions $TOP/test-mani-build2.txt > $TOP/revisions2.txt

echo "======= COMPARING GIT REVISIONS"
compare_revisions revisions1.txt revisions2.txt "$TOP/revisions_diff.txt" "between phase 1 and phase 2 builds"

echo "======= ANALYSING BUILD COMMANDS"

extract_run test-mani-build1.txt build >$TOP/run_lines_1.txt
extract_run test-mani-build2.txt build2 >$TOP/run_lines_2.txt

compare_run_lines $TOP/run_lines_1.txt $TOP/run_lines_2.txt $TOP/run_lines_diff.txt

echo "======= VALIDATING ARTIFACTS (PHASE 2)"
check_artifacts 2 $TOP/build2/artifacts2.txt


echo "======= FINDING MANIFESTS FROM PHASE 2"

mkdir -p test2
pushd test2

TARBALL2="$(read_var $TOP/build2/artifacts2.txt toolchain)"
tar axf "$TARBALL2"
MANIFEST2="$(read_var $TOP/build2/artifacts2.txt manifest)"
cp "$MANIFEST2" "$TOP/orig-manifest2.txt"

echo "======= VALIDATING MANIFEST FORMAT (PHASE 2)"
if $ABE_TESTS_PATH/validate-manifest.pl "$TOP/orig-manifest2.txt" &> "$TOP/validate2.txt"; then
    result_pass phase 2 manifest validated successfully.
else
    result_fail "$TOP/validate2.txt" phase 2 manifest did not validate.
fi

echo "======= CREATING STRIPPED MANIFEST (PHASE 2)"
strip_unused_content_from_manifest "$TOP/orig-manifest2.txt" "$TOP/stripped-manifest2.txt"

echo "======= CHECKING MANIFESTS"
scan_manifest_for_build_dir "$TOP/stripped-manifest1.txt" "$TOP/build" 1
scan_manifest_for_build_dir "$TOP/stripped-manifest2.txt" "$TOP/build" 2
scan_manifest_for_build_dir "$TOP/stripped-manifest2.txt" "$TOP/build2" 2

(
    set +e

    diff -u "$TOP/stripped-manifest1.txt" "$TOP/stripped-manifest2.txt" > $TOP/strict_mani_diff.txt
    if [ $? -eq 0 ]; then
        result_pass strict manifest diff.
    else
        result_fail $TOP/strict_mani_diff.txt strict manifest diff.
    fi

    diff -u <(sort "$TOP/stripped-manifest1.txt") <(sort "$TOP/stripped-manifest2.txt") > $TOP/relaxed_mani_diff.txt
    if [ $? -eq 0 ]; then
        result_pass relaxed manifest diff.
    else
        result_fail $TOP/relaxed_mani_diff.txt relaxed manifest diff.
    fi

    diff -u <(sort -u "$TOP/stripped-manifest1.txt") <(sort -u "$TOP/stripped-manifest2.txt") > $TOP/super_relaxed_mani_diff.txt
    if [ $? -eq 0 ]; then
        result_pass super relaxed manifest diff.
    else
        result_fail $TOP/super_relaxed_mani_diff.txt super relaxed manifest diff.
    fi
)

echo "======= CHECKING REVISIONS IN MANIFEST FILES"
check_revisions_in_manifest 1                \
        "$TOP/revisions1.txt"                \
        "$TOP/stripped-manifest1.txt"        \
        "$TOP/revisions-report1.txt"
check_revisions_in_manifest 2                \
        "$TOP/revisions2.txt"                \
        "$TOP/stripped-manifest2.txt"        \
        "$TOP/revisions-report2.txt"


echo "======= CHECKING BUILD LOGS"
check_log 1 '^ERROR ' $TOP/test-mani-build1.txt $TOP/error_list_1.txt 'found Abe ERRORs' 'no errors reported in build'
check_log 2 '^ERROR ' $TOP/test-mani-build2.txt $TOP/error_list_2.txt 'found Abe ERRORs' 'no errors reported in build'

check_log 1 '\.sh: line [0-9]\+' $TOP/test-mani-build1.txt $TOP/shell_error_1.txt 'found shell errors' 'no shell errors found'
check_log 2 '\.sh: line [0-9]\+' $TOP/test-mani-build2.txt $TOP/shell_error_2.txt 'found shell errors' 'no shell errors found'

echo "======= CHECKING FILE LISTS"
(cd $TOP/test;  find . | sort > $TOP/files_1.txt)
(cd $TOP/test2; find . | sort > $TOP/files_2.txt)

(
    set +e
    diff -u <(sed -e "s/$TODAY/<date>/g; s/$TOMORROW/<date>/g" $TOP/files_1.txt) <(sed -e "s/$TODAY/<date>/g; s/$TOMORROW/<date>/g" $TOP/files_2.txt) > $TOP/files_diff.txt
    if [ $? -eq 0 ]; then
        result_pass files list diff.
    else
        result_fail $TOP/files_diff.txt files list diff.
    fi
)

check_wrap_fails 1
check_wrap_fails 2

fi

echo "======= TESTING ARCHIVED MANIFESTS"

ARCHIVE_TOP="$ABE_TESTS_PATH/test-data/ref-1.1/"

mkdir -p build.archive1
pushd build.archive1
echo "======= CONFIGURE ABE (ARCHIVE 1)"
$ABE_PATH/configure ${GIT_REF_OPT:+"$GIT_REF_OPT"} ${ABE_CONF_FLAGS:+$ABE_CONF_FLAGS}
echo "======= BUILD TOOLCHAIN (ARCHIVE 1)"
wrap_abe archive1 $TOP/test-mani-checkout-archive1.txt			\
	$ABE_PATH/abe.sh --snapshots $SNAPSHOTS				\
	--list-artifacts "artifacts-checkout-archive1.txt"		\
	--manifest "$ARCHIVE_TOP/orig-manifest1.txt"  			\
	--checkout all
wrap_abe archive1 $TOP/test-mani-build-archive1.txt			\
	$ABE_PATH/abe.sh --disable update --snapshots $SNAPSHOTS	\
	--list-artifacts "artifacts-build-archive1.txt"			\
	--manifest "$ARCHIVE_TOP/orig-manifest1.txt"  			\
	--build all --tarbin --dryrun

popd

echo "======= ANALYSING BUILD COMMANDS"
# only look at configure lines for now
extract_run $TOP/test-mani-build-archive1.txt build.archive1 |sed -e 's/^DRYRUN:/RUN:/' | xgrep configure >$TOP/run_lines_archive1.txt
xgrep configure "$ARCHIVE_TOP/run_lines_1.txt" > $TOP/run_lines_ref_archive1.txt

# too many paths in here, disabled for now
#compare_run_lines $TOP/run_lines_ref_archive1.txt $TOP/run_lines_archive1.txt $TOP/run_lines_diff-archive1.txt
extract_revisions $TOP/test-mani-build-archive1.txt > $TOP/revisions-archive1.txt
compare_revisions "$ARCHIVE_TOP/revisions1.txt" $TOP/revisions-archive1.txt "$TOP/revisions_diff-archive1.txt" "in archive1 build."
#check_revisions_in_manifest archive1		\
#        "$TOP/revisions-archive1.txt"		\
#        "$ARCHIVE_TOP/orig-manifest1.txt"	\
#        "$TOP/revisions-report-archive1.txt"

# from here, we don't need debug, and it just messes up the output anyway
set +x

echo "======= TESTS COMPLETE"

echo
echo "          ==============================="
echo "          ======= RESULTS SUMMARY ======="
echo "          ==============================="
echo

num_fails="`xgrep -c ^FAILED $TOP/results_summary.txt`"
num_tests="`xgrep -c ^ $TOP/results_summary.txt`"

echo
echo "Summary of results in $TOP/results_summary.txt"
echo "Full report in in $TOP/results_report.txt"
echo

if $DISPLAY_SUMMARY; then
    echo "Summary follows:"
    cat "$TOP/results_summary.txt"
fi

if $DISPLAY_REPORT; then
    echo "Report follows:"
    cat "$TOP/results_report.txt"
fi

REACHED_THE_END=1

echo

if [[ $num_fails -gt 0 ]]; then
   echo "Failed $num_fails/$num_tests tests."
   exit 1
else
   echo "Passed $num_tests tests."
   exit 0
fi