[microTVM] TVMCon 2021 Zephyr Demo with CMSIS-NN (apache#10144)

This is adds a Zephyr Demo showing how to integrate TVM directly into your embedded application. It runs a keyword spotting model with the Zephyr RTOS using CMSIS-NN with the Ahead-of-Time (AOT) executor and the stack allocation strategy.

Jenkinsfile

@@ -369,6 +369,10 @@ stage('Build') {
               script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh",
               label: 'Run microTVM tests',
             )
+            sh (
+              script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh",
+              label: 'Run microTVM demos',
+            )
             junit 'build/pytest-results/*.xml'
           }
         }

apps/microtvm/zephyr_cmsisnn/.gitignore

@@ -0,0 +1,2 @@
+build
+*.tflite

apps/microtvm/zephyr_cmsisnn/CMakeLists.txt

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.20.0)
+set(BOARD mps3_an547)
+set(EMU_PLATFORM armfvp)
+
+find_package(Zephyr)
+project(my_zephyr_app)
+
+if (NOT DEFINED CMSIS_PATH)
+    set(CMSIS_PATH /opt/arm/ethosu/cmsis/)
+endif()
+
+set(TVMC_COMMAND python3 -m tvm.driver.tvmc)
+set(TVMC_ARGS 
+    --target="cmsis-nn -mcpu=cortex-m55, c" # CMSIS-NN and C targets
+    --runtime=crt # C Runtime
+    --executor=aot # Ahead-of-Time Executor
+    --executor-aot-unpacked-api=1 # Direct internal calls to operators
+    --executor-aot-interface-api=c # Expose C interface to the model
+    --pass-config=tir.disable_vectorize=1 # Disable vectorizer for C output
+    --output-format=mlf # Output Model Library Format tarball
+)
+
+set(TVM_RUNTIME
+    ${CMAKE_CURRENT_BINARY_DIR}/runtime/src/runtime/crt/common/crt_backend_api.c
+    ${CMAKE_CURRENT_BINARY_DIR}/runtime/src/runtime/crt/memory/stack_allocator.c
+)
+set(CODEGEN_OUTPUT 
+    ${CMAKE_CURRENT_BINARY_DIR}/codegen/host/src/default_lib0.c
+    ${CMAKE_CURRENT_BINARY_DIR}/codegen/host/src/default_lib1.c
+    ${CMAKE_CURRENT_BINARY_DIR}/codegen/host/src/default_lib2.c
+)
+set(DATA_FILES 
+    ${CMAKE_CURRENT_BINARY_DIR}/inputs.c 
+    ${CMAKE_CURRENT_BINARY_DIR}/outputs.c 
+    ${CMAKE_CURRENT_BINARY_DIR}/labels.c
+)
+set(CMSIS_SOURCES
+    ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1_x_n_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_s8_fast.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_s8_s16.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mat_mult_nt_t_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_vec_mat_mult_t_s8.c
+    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_with_offset.c
+)
+
+add_custom_command(
+    OUTPUT ${TVM_RUNTIME}
+    OUTPUT ${CODEGEN_OUTPUT} 
+    COMMAND ${TVMC_COMMAND} compile ${TVMC_ARGS} ${CMAKE_CURRENT_SOURCE_DIR}/model/cnn_s_quantized.tflite
+    COMMAND tar xf ${CMAKE_CURRENT_BINARY_DIR}/module.tar
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+add_custom_command(
+    OUTPUT ${DATA_FILES}
+    COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/model/convert_input.py ${CMAKE_CURRENT_SOURCE_DIR}/model/input.txt ${CMAKE_CURRENT_BINARY_DIR}
+    COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/model/convert_labels.py ${CMAKE_CURRENT_SOURCE_DIR}/model/labels.txt ${CMAKE_CURRENT_BINARY_DIR}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+target_sources(app PRIVATE 
+    src/main.c
+    ${TVM_RUNTIME}
+    ${CODEGEN_OUTPUT}
+    ${DATA_FILES}
+    ${CMSIS_SOURCES}
+)
+target_include_directories(app 
+    PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include 
+    PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/runtime/include ${CMAKE_CURRENT_BINARY_DIR}/codegen/host/include
+    PUBLIC ${CMSIS_PATH}/CMSIS/NN/Include/ ${CMSIS_PATH}/CMSIS/DSP/Include
+)

apps/microtvm/zephyr_cmsisnn/README.md

@@ -0,0 +1,43 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# Zephyr RTOS Demo with CMSIS-NN
+
+This project was used for the [TVMCon 2021 talk on Cortex-M improvements to TVM](https://www.youtube.com/watch?v=6a7o8U-8Op4). It runs a keyword spotting model with the Zephyr RTOS using CMSIS-NN with the Ahead-of-Time (AOT) executor and the stack allocation strategy.
+
+The application starts from [the Zephyr base project](https://docs.zephyrproject.org/latest/application/index.html#application) and makes minimal changes to integrate TVM. To try it out, first refer to the [Zephyr Getting Started](https://docs.zephyrproject.org/latest/getting_started/index.html) page to setup your tooling such as `west` (you can also use the `tlcpack/ci_qemu` image). Then download the [Fixed Virtual Platform (FVP) based on Arm(R) Corstone(TM)-300 software](https://developer.arm.com/tools-and-software/open-source-software/arm-platforms-software/arm-ecosystem-fvps) and set the path for Zephyr to find it:
+
+```
+export ARMFVP_BIN_PATH=/opt/arm/FVP_Corstone_SSE-300/models/Linux64_GCC-6.4/
+```
+
+Download the keyword spotting model to the `model` directory:
+```
+wget \
+    https://github.com/ARM-software/ML-zoo/blob/ee35139af86bdace5e502b09fe8b9da9cb1f06bb/models/keyword_spotting/cnn_small/tflite_int8/cnn_s_quantized.tflite \
+    -O model/cnn_s_quantized.tflite
+``` 
+
+Checkout [CMSIS_5](https://github.com/ARM-software/CMSIS_5.git) (default is `/opt/arm/ethosu/cmsis` to reflect `tlcpack/ci_qemu`):
+```
+git clone "https://github.com/ARM-software/CMSIS_5.git" cmsis
+```
+
+And run the demo using `west`, with the path to CMSIS:
+```
+west build -t run -- -DCMSIS_PATH=/opt/arm/ethosu/cmsis
+```

apps/microtvm/zephyr_cmsisnn/include/crt_config.h

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file include/crt_config.h
+ * \brief CRT configuration for demo app.
+ */
+#ifndef TVM_RUNTIME_CRT_CONFIG_H_
+#define TVM_RUNTIME_CRT_CONFIG_H_
+
+/*! Log level of the CRT runtime */
+#define TVM_CRT_LOG_LEVEL TVM_CRT_LOG_LEVEL_DEBUG
+
+/*! Support low-level debugging in MISRA-C runtime */
+#define TVM_CRT_DEBUG 0
+
+/*! Maximum supported dimension in NDArray */
+#define TVM_CRT_MAX_NDIM 6
+/*! Maximum supported arguments in generated functions */
+#define TVM_CRT_MAX_ARGS 10
+/*! Maximum supported string length in dltype, e.g. "int8", "int16", "float32"
+ */
+#define TVM_CRT_MAX_STRLEN_DLTYPE 10
+/*! Maximum supported string length in function names */
+#define TVM_CRT_MAX_STRLEN_FUNCTION_NAME 120
+/*! Maximum supported string length in parameter names */
+#define TVM_CRT_MAX_STRLEN_PARAM_NAME 80
+
+/*! Maximum number of registered modules. */
+#define TVM_CRT_MAX_REGISTERED_MODULES 2
+
+/*! Size of the global function registry, in bytes. */
+#define TVM_CRT_GLOBAL_FUNC_REGISTRY_SIZE_BYTES 512
+
+/*! Maximum packet size, in bytes, including the length header. */
+#define TVM_CRT_MAX_PACKET_SIZE_BYTES 512
+
+#endif  // TVM_RUNTIME_CRT_CONFIG_H_

apps/microtvm/zephyr_cmsisnn/model/convert_input.py

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pathlib
+import sys
+import numpy as np
+
+
+def create_file(name, prefix, tensor_name, tensor_data, output_path):
+    """
+    This function generates a header file containing the data from the numpy array provided.
+    """
+    file_path = pathlib.Path(f"{output_path}/" + name).resolve()
+    # Create header file with npy_data as a C array
+    raw_path = file_path.with_suffix(".c").resolve()
+    with open(raw_path, "w") as header_file:
+        header_file.write(
+            "#include <stddef.h>\n"
+            "#include <stdint.h>\n"
+            f"const size_t {tensor_name}_len = {tensor_data.size};\n"
+            f"{prefix} float {tensor_name}_storage[] = "
+        )
+        header_file.write("{")
+        for i in np.ndindex(tensor_data.shape):
+            header_file.write(f"{tensor_data[i]}, ")
+        header_file.write("};\n\n")
+
+
+def create_files(input_file, output_dir):
+    """
+    This function generates C files for the input and output arrays required to run inferences
+    """
+    # Create out folder
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Create input header file
+    input_data = np.loadtxt(input_file)
+    create_file("inputs", "const", "input", input_data, output_dir)
+
+    # Create output header file
+    output_data = np.zeros([12], np.float32)
+    create_file(
+        "outputs",
+        "",
+        "output",
+        output_data,
+        output_dir,
+    )
+
+
+if __name__ == "__main__":
+    create_files(sys.argv[1], sys.argv[2])

apps/microtvm/zephyr_cmsisnn/model/convert_labels.py

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pathlib
+import sys
+
+
+def create_labels_header(labels_file, output_path):
+    """
+    This function generates a header file containing the ImageNet labels as an array of strings
+    """
+    labels_path = pathlib.Path(labels_file).resolve()
+    file_path = pathlib.Path(f"{output_path}/labels.c").resolve()
+
+    with open(labels_path) as f:
+        labels = f.readlines()
+
+    with open(file_path, "w") as header_file:
+        header_file.write(f"char* labels[] = {{")
+
+        for _, label in enumerate(labels):
+            header_file.write(f'"{label.rstrip()}",')
+
+        header_file.write("};\n")
+
+
+if __name__ == "__main__":
+    create_labels_header(sys.argv[1], sys.argv[2])