From 65f6aa0a603d8964bcf1983d72e00b0025249ae6 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 9 Feb 2022 08:10:21 -0800 Subject: [PATCH] Add scripts for automated build and testing (#10194) --- .../arduino/template_project/boards.json | 36 +++++++--- .../arduino/base-box/test-config.json | 30 --------- apps/microtvm/reference-vm/base-box-tool.py | 66 +++++++++++-------- .../scripts/reference_vm_build.sh | 34 ++++++++++ .../scripts/reference_vm_release.sh | 43 ++++++++++++ .../reference-vm/scripts/reference_vm_test.sh | 37 +++++++++++ apps/microtvm/reference-vm/scripts/utils.sh | 26 ++++++++ .../zephyr/base-box/test-config.json | 18 ----- .../zephyr/template_project/boards.json | 40 ++++++++--- 9 files changed, 236 insertions(+), 94 deletions(-) delete mode 100644 apps/microtvm/reference-vm/arduino/base-box/test-config.json create mode 100755 apps/microtvm/reference-vm/scripts/reference_vm_build.sh create mode 100755 apps/microtvm/reference-vm/scripts/reference_vm_release.sh create mode 100755 apps/microtvm/reference-vm/scripts/reference_vm_test.sh create mode 100755 apps/microtvm/reference-vm/scripts/utils.sh delete mode 100644 apps/microtvm/reference-vm/zephyr/base-box/test-config.json diff --git a/apps/microtvm/arduino/template_project/boards.json b/apps/microtvm/arduino/template_project/boards.json index 595d56b5f615..8f039f0680e7 100644 --- a/apps/microtvm/arduino/template_project/boards.json +++ b/apps/microtvm/arduino/template_project/boards.json @@ -3,57 +3,75 @@ "package": "arduino", "architecture": "sam", "board": "arduino_due_x_dbg", - "model": "sam3x8e" + "model": "sam3x8e", + "vid_hex": "2341", + "pid_hex": "003d" }, "feathers2": { "package": "esp32", "architecture": "esp32", "board": "feathers2", "model": "esp32", - "note": "Due to the way the Feather S2 bootloader works, compilation behaves fine but uploads cannot be done automatically." + "note": "Due to the way the Feather S2 bootloader works, compilation behaves fine but uploads cannot be done automatically.", + "vid_hex": "303a", + "pid_hex": "0002" }, "metrom4": { "package": "adafruit", "architecture": "samd", "board": "adafruit_metro_m4", - "model": "atsamd51" + "model": "atsamd51", + "vid_hex": "", + "pid_hex": "" }, "spresense": { "package": "SPRESENSE", "architecture": "spresense", "board": "spresense", "model": "cxd5602gg", - "note": "Spresense only works as of its v2.3.0 sdk." + "note": "Spresense only works as of its v2.3.0 sdk.", + "vid_hex": "10c4", + "pid_hex": "ea60" }, "nano33ble": { "package": "arduino", "architecture": "mbed_nano", "board": "nano33ble", - "model": "nrf52840" + "model": "nrf52840", + "vid_hex": "2341", + "pid_hex": "805a" }, "pybadge": { "package": "adafruit", "architecture": "samd", "board": "adafruit_pybadge_m4", - "model": "atsamd51" + "model": "atsamd51", + "vid_hex": "", + "pid_hex": "" }, "teensy40": { "package": "teensy", "architecture": "avr", "board": "teensy40", "model": "imxrt1060", - "note": "The Teensy boards are listed here for completeness, but they won't work until https://github.com/arduino/arduino-cli/issues/700 is finished." + "note": "The Teensy boards are listed here for completeness, but they won't work until https://github.com/arduino/arduino-cli/issues/700 is finished.", + "vid_hex": "16c0", + "pid_hex": "0478" }, "teensy41": { "package": "teensy", "architecture": "avr", "board": "teensy41", - "model": "imxrt1060" + "model": "imxrt1060", + "vid_hex": "16c0", + "pid_hex": "0478" }, "wioterminal": { "package": "Seeeduino", "architecture": "samd", "board": "seeed_wio_terminal", - "model": "atsamd51" + "model": "atsamd51", + "vid_hex": "2886", + "pid_hex": "802d" } } diff --git a/apps/microtvm/reference-vm/arduino/base-box/test-config.json b/apps/microtvm/reference-vm/arduino/base-box/test-config.json deleted file mode 100644 index 80cc17f56847..000000000000 --- a/apps/microtvm/reference-vm/arduino/base-box/test-config.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "due": { - "vid_hex": "2341", - "pid_hex": "003d" - }, - "feathers2": { - "vid_hex": "303a", - "pid_hex": "0002" - }, - "nano33ble": { - "vid_hex": "2341", - "pid_hex": "805a" - }, - "spresense": { - "vid_hex": "10c4", - "pid_hex": "ea60" - }, - "teensy40": { - "vid_hex": "16c0", - "pid_hex": "0478" - }, - "teensy41": { - "vid_hex": "16c0", - "pid_hex": "0478" - }, - "wioterminal": { - "vid_hex": "2886", - "pid_hex": "802d" - } -} diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index 5a22ab0d8819..fa649afa5c87 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -18,6 +18,7 @@ import argparse +from ast import arg import copy import json import logging @@ -27,6 +28,7 @@ import shutil import subprocess import sys +import pathlib _LOG = logging.getLogger(__name__) @@ -61,25 +63,15 @@ # List of identifying strings for microTVM boards for testing. -# TODO add a way to declare supported boards to ProjectAPI +with open(pathlib.Path(THIS_DIR) / ".." / "zephyr" / "template_project" / "boards.json") as f: + zephyr_boards = json.load(f) + +with open(pathlib.Path(THIS_DIR) / ".." / "arduino" / "template_project" / "boards.json") as f: + arduino_boards = json.load(f) + ALL_MICROTVM_BOARDS = { - "arduino": ( - "due", - "feathers2", - "metrom4", - "nano33ble", - "pybadge", - "spresense", - "teensy40", - "teensy41", - "wioterminal", - ), - "zephyr": ( - "nucleo_f746zg", - "stm32f746g_disco", - "nrf5340dk_nrf5340_cpuapp", - "mps2_an521", - ), + "arduino": arduino_boards.keys(), + "zephyr": zephyr_boards.keys(), } @@ -290,7 +282,9 @@ def build_command(args): SKIP_COPY_PATHS = [".vagrant", "base-box"] -def do_build_release_test_vm(release_test_dir, user_box_dir, base_box_dir, provider_name): +def do_build_release_test_vm( + release_test_dir, user_box_dir: pathlib.Path, base_box_dir: pathlib.Path, provider_name +): if os.path.exists(release_test_dir): try: subprocess.check_call(["vagrant", "destroy", "-f"], cwd=release_test_dir) @@ -378,10 +372,10 @@ def _quote_cmd(cmd): def test_command(args): - user_box_dir = os.path.join(THIS_DIR, args.platform) - base_box_dir = os.path.join(THIS_DIR, args.platform, "base-box") - test_config_file = os.path.join(base_box_dir, "test-config.json") - with open(test_config_file) as f: + user_box_dir = pathlib.Path(THIS_DIR) / args.platform + base_box_dir = user_box_dir / "base-box" + boards_file = pathlib.Path(THIS_DIR) / ".." / args.platform / "template_project" / "boards.json" + with open(boards_file) as f: test_config = json.load(f) # select microTVM test config @@ -390,7 +384,7 @@ def test_command(args): for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): assert key in microtvm_test_config and isinstance( microtvm_test_config[key], expected_type - ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" + ), f"Expected key {key} of type {expected_type} in {boards_file}: {test_config!r}" microtvm_test_config["vid_hex"] = microtvm_test_config["vid_hex"].lower() microtvm_test_config["pid_hex"] = microtvm_test_config["pid_hex"].lower() @@ -443,7 +437,10 @@ def test_command(args): def release_command(args): - vm_name = f"tlcpack/microtvm-{args.platform}-{args.platform_version}" + if args.release_full_name: + vm_name = args.release_full_name + else: + vm_name = f"tlcpack/microtvm-{args.platform}-{args.platform_version}" if not args.skip_creating_release_version: subprocess.check_call( @@ -565,14 +562,29 @@ def parse_args(): ) parser_release.add_argument( "--platform-version", - required=True, + required=False, help=( "For Zephyr, the platform version to release, in the form 'x.y'. " "For Arduino, the version of arduino-cli that's being used, in the form 'x.y.z'." ), ) + parser_release.add_argument( + "--release-full-name", + required=False, + type=str, + default=None, + help=( + "If set, it will use this as the full release name and version for the box. " + "If this set, it will ignore `--platform-version` and `--release-version`." + ), + ) + + args = parser.parse_args() + + if args.action == "release" and not args.release_full_name: + parser.error("--platform-version is requireed.") - return parser.parse_args() + return args def main(): diff --git a/apps/microtvm/reference-vm/scripts/reference_vm_build.sh b/apps/microtvm/reference-vm/scripts/reference_vm_build.sh new file mode 100755 index 000000000000..bac31a26cf78 --- /dev/null +++ b/apps/microtvm/reference-vm/scripts/reference_vm_build.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Usage: apps/microtvm/reference-vm/scripts/reference_vm_build.sh +# + +if [ "$#" -lt 1 -o "$1" == "--help" -o "$1" == "-h" ]; then + echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_build.sh " + exit -1 +fi + +PLATFORM=$1 +shift + +cd "$(dirname "$0")" +source "./utils.sh" || exit 2 +cd ${RVM_BASE_PATH} + +${BASE_BOX_TOOL} --provider=virtualbox build ${PLATFORM} diff --git a/apps/microtvm/reference-vm/scripts/reference_vm_release.sh b/apps/microtvm/reference-vm/scripts/reference_vm_release.sh new file mode 100755 index 000000000000..8719e2c05a9f --- /dev/null +++ b/apps/microtvm/reference-vm/scripts/reference_vm_release.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Usage: apps/microtvm/reference-vm/scripts/reference_vm_release.sh +# + +if [ "$#" -lt 3 -o "$1" == "--help" -o "$1" == "-h" ]; then + echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_release.sh " + exit -1 +fi + +PLATFORM=$1 +shift + +RELEASE_NAME=$1 +shift + +RELEASE_VERSION=$1 +shift + +cd "$(dirname "$0")" +source "./utils.sh" || exit 2 +cd ${RVM_BASE_PATH} + +${BASE_BOX_TOOL} --provider=virtualbox release ${PLATFORM} \ + --release-full-name=${RELEASE_NAME} \ + --release-version=${RELEASE_VERSION} \ + --skip-creating-release-version diff --git a/apps/microtvm/reference-vm/scripts/reference_vm_test.sh b/apps/microtvm/reference-vm/scripts/reference_vm_test.sh new file mode 100755 index 000000000000..25031cc0ccd8 --- /dev/null +++ b/apps/microtvm/reference-vm/scripts/reference_vm_test.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Usage: apps/microtvm/reference-vm/scripts/reference_vm_test.sh +# + +if [ "$#" -lt 2 -o "$1" == "--help" -o "$1" == "-h" ]; then + echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_test.sh " + exit -1 +fi + +PLATFORM=$1 +shift + +BOARD=$1 +shift + +cd "$(dirname "$0")" +source "./utils.sh" || exit 2 +cd ${RVM_BASE_PATH} + +${BASE_BOX_TOOL} --provider=virtualbox test ${PLATFORM} --microtvm-board=${BOARD} diff --git a/apps/microtvm/reference-vm/scripts/utils.sh b/apps/microtvm/reference-vm/scripts/utils.sh new file mode 100755 index 000000000000..27b929da2316 --- /dev/null +++ b/apps/microtvm/reference-vm/scripts/utils.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") + +function get_repo_root() { + cd "${SCRIPTS_DIR}" && git rev-parse --show-toplevel +} + +BASE_BOX_TOOL="./base-box-tool.py" +RVM_BASE_PATH="$(get_repo_root)"/apps/microtvm/reference-vm diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json deleted file mode 100644 index 6e0cfa5e2527..000000000000 --- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "nucleo_f746zg": { - "vid_hex": "0483", - "pid_hex": "374b" - }, - "stm32f746g_disco": { - "vid_hex": "0483", - "pid_hex": "374b" - }, - "nrf5340dk_nrf5340_cpuapp": { - "vid_hex": "1366", - "pid_hex": "1055" - }, - "mps2_an521": { - "vid_hex": "", - "pid_hex": "" - } -} diff --git a/apps/microtvm/zephyr/template_project/boards.json b/apps/microtvm/zephyr/template_project/boards.json index 18e393897f04..543a3421683a 100644 --- a/apps/microtvm/zephyr/template_project/boards.json +++ b/apps/microtvm/zephyr/template_project/boards.json @@ -3,60 +3,80 @@ "board": "mimxrt1050_evk", "model": "imxrt10xx", "is_qemu": false, - "fpu": true + "fpu": true, + "vid_hex": "", + "pid_hex": "" }, "mps2_an521": { "board": "mps2_an521", "model": "mps2_an521", "is_qemu": true, - "fpu": false + "fpu": false, + "vid_hex": "", + "pid_hex": "" }, "nrf5340dk_nrf5340_cpuapp": { "board": "nrf5340dk_nrf5340_cpuapp", "model": "nrf5340dk", "is_qemu": false, - "fpu": true + "fpu": true, + "vid_hex": "1366", + "pid_hex": "1055" }, "nucleo_f746zg": { "board": "nucleo_f746zg", "model": "stm32f746xx", "is_qemu": false, - "fpu": true + "fpu": true, + "vid_hex": "0483", + "pid_hex": "374b" }, "nucleo_l4r5zi": { "board": "nucleo_l4r5zi", "model": "stm32l4r5zi", "is_qemu": false, - "fpu": true + "fpu": true, + "vid_hex": "0483", + "pid_hex": "374b" }, "qemu_cortex_r5": { "board": "qemu_cortex_r5", "model": "zynq_mp_r5", "is_qemu": true, - "fpu": true + "fpu": true, + "vid_hex": "", + "pid_hex": "" }, "qemu_riscv32": { "board": "qemu_riscv32", "model": "host", "is_qemu": true, - "fpu": false + "fpu": false, + "vid_hex": "", + "pid_hex": "" }, "qemu_riscv64": { "board": "qemu_riscv64", "model": "host", "is_qemu": true, - "fpu": true + "fpu": true, + "vid_hex": "", + "pid_hex": "" }, "qemu_x86": { "board": "qemu_x86", "model": "host", "is_qemu": true, - "fpu": true + "fpu": true, + "vid_hex": "", + "pid_hex": "" }, "stm32f746g_disco": { "board": "stm32f746g_disco", "model": "stm32f746xx", "is_qemu": false, - "fpu": true + "fpu": true, + "vid_hex": "0483", + "pid_hex": "374b" } }