diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 29667327..4509b4b8 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -213,6 +213,42 @@ Sample ansible.cfg is there to ensure collection does not need to be installed. ansible-playbook -i localhost, examples/iso_info.yml -v ``` +## Running integration tests + +Some test objects (ISO image, VirtualDisk, VM) needs to be created before we run integration tests. +Use command: + +```bash +ansible-playbook -i localhost, tests/integration/prepare/prepare_vm.yml -v +ansible-playbook -i localhost, tests/integration/prepare/prepare_iso.yml -v +ansible-playbook -i localhost, tests/integration/prepare/prepare_examples.yml -v +``` + +The `ansible-test integration` will try to run all integration test, and will fail on first problematic test. +It does allow you to continue from the failed test. + +You might want to automatically continue running remaining tests. +A few failed tests can be reviewed and retried later. +The `./ci-infra/helpers/run-tests.sh` was made for this. +If N tests fail in first pass, the `run-tests.sh` will retry only those N test in second pass. +The script is used like: + +```bash +source ci-infra/local-dev/env-host-4.sh +./ci-infra/helpers/run-tests.sh outdir +``` + +File `tests.txt` is optional input. +It contains one test name per line. +If ommited, all tests from `tests/integration` are run. + +After run, the scripts create in `outd` directory: + +- directory `log-${timestamp}` directory containing a log file for each run test +- file `status.txt` contains OK/ERR/PEND/SKIP status for each run test + +On next run, only tests that have PEND/ERR status are retried. + ## Creating a release Releases are automatically created when a tag is created with a name matching @@ -231,7 +267,7 @@ ScaleComputing does setup new VSNS, with suitable HyperCore version installed. Steps: - Request / reserve static IP address from Alex - - either replacing existing static IP or using next in series 105.11.20x + - either replacing existing static IP or using next in series 10.5.11.20x - create empty VM with 1 virtio disk, type other, tag hc3nested, 16GB ram, 4 cores. - image new vSNS node using test iso image (vs. release - this may change in upcoming releases) - (optional) Save it as template VM, example name `vsns9213-unconfigured` @@ -240,6 +276,7 @@ Steps: - `sudo singleNodeCluster=1 scclusterinit` - Save it as template VM, example name `vsns9213-template` - Create a final vSNS from template VM, example name `vsns9213-ci` + - keep same MAC address? - Add vSNS login URL to Azure OIDC redirectUris - ensure ip address is added to entraAD (azure) app registration for OIDC integration (ask Dave if needed) - "app_display_name": "Scale Computing HC3", diff --git a/ci-infra/helpers/run-tests.sh b/ci-infra/helpers/run-tests.sh index 293992f1..860ad7f4 100755 --- a/ci-infra/helpers/run-tests.sh +++ b/ci-infra/helpers/run-tests.sh @@ -5,6 +5,7 @@ # Input: # - outdir will contain status/progress file, and log files of individual tests # - (optional) test-names.txt contains integration tests to be run, one per line. +# If not given, all tests in tests/integration/targets/ are run. # Output: # - logs are one test per file, in outdir/log-timestamp/ # - list of succeded/failed tests are in outdir/status.txt @@ -12,6 +13,8 @@ # - PEND (or contains just test name), test will be run # - SKIP means skip this test # - OK or ERR are set after test is run +# On second run, only PEND tests are run. +# If there are no PEND tests, the ERR tests are retried. set -ue # set -v @@ -40,14 +43,29 @@ then echo "ERROR file content $TSTATUS" 1>&2 exit 1 fi +/bin/cp "$TSTATUS" "$OUTD2/status.txt" TEST_NAMES=$(grep "^PEND" "$TSTATUS" | awk '{print $2}') +if [ -z "$TEST_NAMES" ] +then + echo "No PEND tests, retrying ERR tests" + sed -i 's/^ERR/PEND/' "$TSTATUS" +fi +TEST_NAMES=$(grep "^PEND" "$TSTATUS" | awk '{print $2}') +if [ -z "$TEST_NAMES" ] +then + echo "No tests to run, exiting." + exit 0 +fi # shellcheck disable=SC2086 echo "Pending tests: "$TEST_NAMES # shellcheck disable=SC2086 +TEST_COUNT=$(echo $TEST_NAMES | wc -w) + +ii=0 for TN in $TEST_NAMES do - echo "Running test $TN" + echo "Running test $TN ($ii/$TEST_COUNT)" ( echo ansible-test integration --local "$TN" echo "======================" @@ -67,4 +85,5 @@ do ) >"$OUTD2/$TN.log" 2>&1 res=$(grep $'\t'"$TN\$" "$TSTATUS" | awk '{print $1}') echo " result $res $TN" + ii=$((ii+1)) done