Merge pull request #30 from NVIDIA-ISAAC-ROS/release-dp3.1

Isaac ROS 0.31.0 (DP3.1)

README.md

@@ -22,11 +22,11 @@ This package is powered by [NVIDIA Isaac Transport for ROS (NITROS)](https://dev
 
 The following table summarizes the per-platform performance statistics of sample graphs that use this package, with links included to the full benchmark output. These benchmark configurations are taken from the [Isaac ROS Benchmark](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark#list-of-isaac-ros-benchmarks) collection, based on the [`ros2_benchmark`](https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark) framework.
 
-| Sample Graph                                                                                                                              | Input Size | AGX Orin                                                                                                                                      | Orin NX                                                                                                                                       | Orin Nano 8GB                                                                                                                                       | x86_64 w/ RTX 3060 Ti                                                                                                                                   |
-| ----------------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [Rectify Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_rectify_node.py)              | 1080p      | [875 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-agx_orin.json)<br>1.8 ms   | [479 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-orin_nx.json)<br>3.8 ms    | [333 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-orin_nano_8gb.json)<br>4.0 ms    | [1540 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-x86_64_rtx_3060Ti.json)<br>0.37 ms  |
-| [Stereo Disparity Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_disparity_node.py)   | 1080p      | [151 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-agx_orin.json)<br>9.0 ms | [73.7 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-orin_nx.json)<br>14 ms  | [51.6 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-orin_nano_8gb.json)<br>22 ms  | [451 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-x86_64_rtx_3060Ti.json)<br>2.9 ms  |
-| [Stereo Disparity Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_disparity_graph.py) | 1080p      | [141 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-agx_orin.json)<br>10 ms | [72.0 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-orin_nx.json)<br>16 ms | [48.7 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-orin_nano_8gb.json)<br>24 ms | [425 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-x86_64_rtx_3060Ti.json)<br>3.6 ms |
+| Sample Graph                                                                                                                              | Input Size | AGX Orin                                                                                                                                      | Orin NX                                                                                                                                       | Orin Nano 8GB                                                                                                                                   | x86_64 w/ RTX 4060 Ti                                                                                                                            |
+| ----------------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [Rectify Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_rectify_node.py)              | 1080p      | [509 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-agx_orin.json)<br>5.6 ms   | [385 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-orin_nx.json)<br>5.1 ms    | [283 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-orin_nano.json)<br>8.5 ms    | [819 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_rectify_node-nuc_4060ti.json)<br>4.4 ms    |
+| [Stereo Disparity Node](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_disparity_node.py)   | 1080p      | [158 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-agx_orin.json)<br>10 ms  | [76.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-orin_nx.json)<br>17 ms  | [53.5 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-orin_nano.json)<br>23 ms  | [397 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_node-nuc_4060ti.json)<br>5.8 ms  |
+| [Stereo Disparity Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/scripts//isaac_ros_disparity_graph.py) | 1080p      | [162 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-agx_orin.json)<br>14 ms | [75.1 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-orin_nx.json)<br>20 ms | [50.6 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-orin_nano.json)<br>28 ms | [387 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_disparity_graph-nuc_4060ti.json)<br>8.3 ms |
 
 ## Table of Contents
 
@@ -60,7 +60,7 @@ The following table summarizes the per-platform performance statistics of sample
 
 ## Latest Update
 
-Update 2023-04-05: Source available GXF extensions
+Update 2023-05-25: Improved stereo rectification.
 
 ## Supported Platforms
 
@@ -248,6 +248,7 @@ admin@workstation:/workspaces/isaac_ros-dev$  ros2 launch realsense2_camera rs_l
 
 | Date       | Changes                                                                                                                                                    |
 | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 2023-05-25 | Improved stereo rectification                                                                                                                              |
 | 2023-04-05 | Source available GXF extensions                                                                                                                            |
 | 2022-10-19 | Updated OSS licensing                                                                                                                                      |
 | 2022-08-31 | Image flip support and update to be compatible with JetPack 5.0.2                                                                                          |

isaac_ros_image_pipeline/package.xml

@@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>isaac_ros_image_pipeline</name>
-  <version>0.30.0</version>
+  <version>0.31.0</version>
   <description>Core image processing</description>
 
   <maintainer email="hemals@nvidia.com">Hemal Shah</maintainer>

isaac_ros_image_proc/CMakeLists.txt

@@ -75,6 +75,10 @@ install(TARGETS gxf_image_flip DESTINATION share/${PROJECT_NAME}/gxf/lib/image_p
 add_subdirectory(gxf/tensorops)
 install(TARGETS gxf_tensorops DESTINATION share/${PROJECT_NAME}/gxf/lib/image_proc)
 
+# RectifyParamsGenerator
+add_subdirectory(gxf/rectify_params_generator)
+install(TARGETS gxf_rectify_params_generator DESTINATION share/${PROJECT_NAME}/gxf/lib/image_proc)
+
 ### End extensions
 
 if(BUILD_TESTING)
@@ -88,12 +92,14 @@ if(BUILD_TESTING)
   add_launch_test(test/isaac_ros_image_format_converter_grayscale_test.py)
   add_launch_test(test/isaac_ros_image_format_converter_nv24_test.py)
   add_launch_test(test/isaac_ros_rectify_test.py)
+  add_launch_test(test/isaac_ros_rectify_timestamp_match_test.py)
+  add_launch_test(test/isaac_ros_rectify_oss_comparison_test.py)
+  add_launch_test(test/isaac_ros_rectify_stereo_epipolar_test.py)
   add_launch_test(test/isaac_ros_resize_test.py)
   add_launch_test(test/isaac_ros_resize_invalid_test.py)
   add_launch_test(test/isaac_ros_image_flip_180_test.py)
   add_launch_test(test/isaac_ros_image_flip_hori_test.py)
   add_launch_test(test/isaac_ros_image_flip_vert_test.py)
-
 endif()
 
 ament_auto_package(INSTALL_TO_SHARE launch config)

isaac_ros_image_proc/config/image_format_converter_substitution.yaml

@@ -22,7 +22,7 @@ body:
   groups:
     - entities:
         - components:
-          - type: nvidia::cvcore::tensor_ops::ConvertColorFormat
+          - type: nvidia::isaac::tensor_ops::ConvertColorFormat
             parameters:
               output_type: "BGR_U8"
               receiver: data_receiver
@@ -34,7 +34,7 @@ body:
             overriding_parameters: []
     - entities:
         - components:
-          - type: nvidia::cvcore::tensor_ops::StreamConvertColorFormat
+          - type: nvidia::isaac::tensor_ops::StreamConvertColorFormat
             parameters:
               output_type: "NV24"
               receiver: data_receiver

isaac_ros_image_proc/config/namespace_injector_rule_image_format_converter.yaml

@@ -20,7 +20,7 @@ name: Image Format Converter Namespace Injector Rule
 operation: namespace_injector
 body:
   components:
-    - type: nvidia::cvcore::tensor_ops::ConvertColorFormat
+    - type: nvidia::isaac::tensor_ops::ConvertColorFormat
       path_parameter_keys: [input_adapter, output_adapter]
-    - type: nvidia::cvcore::tensor_ops::StreamConvertColorFormat
+    - type: nvidia::isaac::tensor_ops::StreamConvertColorFormat
       path_parameter_keys: [input_adapter, output_adapter, stream]

isaac_ros_image_proc/config/namespace_injector_rule_rectify.yaml

@@ -20,5 +20,5 @@ name: Image Rectify Namespace Injector Rule
 operation: namespace_injector
 body:
   components:
-    - type: nvidia::cvcore::tensor_ops::StreamUndistort
+    - type: nvidia::isaac::tensor_ops::StreamUndistort
       path_parameter_keys: [input_adapter, output_adapter, stream]

isaac_ros_image_proc/config/namespace_injector_rule_resize.yaml

@@ -20,5 +20,5 @@ name: Image Resize Namespace Injector Rule
 operation: namespace_injector
 body:
   components:
-    - type: nvidia::cvcore::tensor_ops::Resize
+    - type: nvidia::isaac::tensor_ops::Resize
       path_parameter_keys: [input_adapter, output_adapter]

isaac_ros_image_proc/config/nitros_image_flip_node.yaml

@@ -1,6 +1,6 @@
 %YAML 1.2
 # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -53,7 +53,7 @@ components:
     backends: "CUDA"
     mode: "BOTH"
 ---
-name: vault
+name: sink
 components:
 - name: signal
   type: nvidia::gxf::DoubleBufferReceiver
@@ -64,24 +64,27 @@ components:
   parameters:
     receiver: signal
     min_size: 1
-- name: vault
-  type: nvidia::gxf::Vault
+- name: sink
+  type: nvidia::isaac_ros::MessageRelay
   parameters:
     source: signal
-    max_waiting_count: 1
-    drop_waiting: false
 ---
 components:
 - type: nvidia::gxf::Connection
   parameters:
     source: image_flip/data_transmitter
-    target: vault/signal
+    target: sink/signal
 ---
 name: utils
 components:
 - name: clock
   type: nvidia::gxf::RealtimeClock
-- type: nvidia::gxf::GreedyScheduler
+- type: nvidia::gxf::MultiThreadScheduler
   parameters:
     clock: clock
     stop_on_deadlock: false
+    check_recession_period_ms: 1
+    worker_thread_number: 2
+- type: nvidia::gxf::JobStatistics
+  parameters:
+    clock: clock

isaac_ros_image_proc/config/nitros_image_format_converter_node.yaml

@@ -1,6 +1,6 @@
 %YAML 1.2
 # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -43,7 +43,7 @@ components:
     block_size: 14400000
     num_blocks: 40
 - name: converter
-  type: nvidia::cvcore::tensor_ops::ConvertColorFormat
+  type: nvidia::isaac::tensor_ops::ConvertColorFormat
   parameters:
     output_type: "RGB_U8"
     receiver: data_receiver
@@ -53,7 +53,7 @@ components:
     output_adapter: resource/adapter
     output_name: "image"
 ---
-name: vault
+name: sink
 components:
 - name: signal
   type: nvidia::gxf::DoubleBufferReceiver
@@ -64,21 +64,19 @@ components:
   parameters:
     receiver: signal
     min_size: 1
-- name: vault
-  type: nvidia::gxf::Vault
+- name: sink
+  type: nvidia::isaac_ros::MessageRelay
   parameters:
     source: signal
-    max_waiting_count: 1
-    drop_waiting: false
 ---
 name: resource
 components:
 - name: adapter
-  type: nvidia::cvcore::tensor_ops::ImageAdapter
+  type: nvidia::isaac::tensor_ops::ImageAdapter
   parameters:
     message_type: "VideoBuffer"
 - name: stream
-  type: nvidia::cvcore::tensor_ops::TensorStream
+  type: nvidia::isaac::tensor_ops::TensorStream
   parameters:
     backend_type: "VPI"
     engine_type: "GPU"
@@ -87,7 +85,7 @@ components:
 - type: nvidia::gxf::Connection
   parameters:
     source: imageConverter/data_transmitter
-    target: vault/signal
+    target: sink/signal
 ---
 name: utils
 components:
@@ -97,3 +95,7 @@ components:
   parameters:
     clock: clock
     stop_on_deadlock: false
+    check_recession_period_us: 100
+- type: nvidia::gxf::JobStatistics
+  parameters:
+    clock: clock

isaac_ros_image_proc/config/nitros_rectify_node.yaml

@@ -1,6 +1,6 @@
 %YAML 1.2
 # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES
-# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -73,7 +73,7 @@ components:
     transmitter: data_transmitter
     min_size: 1
 - name: input_camera
-  type: nvidia::cvcore::tensor_ops::CameraModel
+  type: nvidia::isaac::tensor_ops::CameraModel
   parameters:
     distortion_type: "FisheyeEquidistant"
     distortion_coefficients: [0, 0, 0, 0,
@@ -82,7 +82,7 @@ components:
     principle_point: [0, 0]
     skew_value: 0.0
 - name: output_camera
-  type: nvidia::cvcore::tensor_ops::CameraModel
+  type: nvidia::isaac::tensor_ops::CameraModel
   parameters:
     distortion_type: "FisheyeEquidistant"
     distortion_coefficients: [0, 0, 0, 0,
@@ -91,7 +91,7 @@ components:
     principle_point: [0, 0]
     skew_value: 0.0
 - name: reference_frame
-  type: nvidia::cvcore::tensor_ops::Frame3D
+  type: nvidia::isaac::tensor_ops::Frame3D
   parameters:
     rotation: [1, 0, 0, 0, 1, 0, 0, 0, 1]
     translation: [0, 0, 0]
@@ -102,7 +102,7 @@ components:
     block_size: 7372800
     num_blocks: 40
 - name: undistort_algo
-  type: nvidia::cvcore::tensor_ops::StreamUndistort
+  type: nvidia::isaac::tensor_ops::StreamUndistort
   parameters:
     input_camera_model: input_camera
     reference_frame: reference_frame
@@ -126,7 +126,7 @@ components:
 - name: data_receiver
   type: nvidia::gxf::DoubleBufferReceiver
   parameters:
-    capacity: 1
+    capacity: 2
     policy: 0
 - type: nvidia::gxf::MessageAvailableSchedulingTerm
   parameters:
@@ -157,7 +157,7 @@ components:
     image_tx: image_transmitter
     camera_model_tx: camera_model_transmitter
 ---
-name: image_vault
+name: image_sink
 components:
 - name: signal
   type: nvidia::gxf::DoubleBufferReceiver
@@ -168,14 +168,12 @@ components:
   parameters:
     receiver: signal
     min_size: 1
-- name: vault
-  type: nvidia::gxf::Vault
+- name: sink
+  type: nvidia::isaac_ros::MessageRelay
   parameters:
     source: signal
-    max_waiting_count: 1
-    drop_waiting: false
 ---
-name: camerainfo_vault
+name: camerainfo_sink
 components:
 - name: signal
   type: nvidia::gxf::DoubleBufferReceiver
@@ -186,22 +184,20 @@ components:
   parameters:
     receiver: signal
     min_size: 1
-- name: vault
-  type: nvidia::gxf::Vault
+- name: sink
+  type: nvidia::isaac_ros::MessageRelay
   parameters:
     source: signal
-    max_waiting_count: 1
-    drop_waiting: false
 ---
 name: resource
 components:
 - name: stream
-  type: nvidia::cvcore::tensor_ops::TensorStream
+  type: nvidia::isaac::tensor_ops::TensorStream
   parameters:
     backend_type: "VPI"
     engine_type: "GPU"
 - name: adapter
-  type: nvidia::cvcore::tensor_ops::ImageAdapter
+  type: nvidia::isaac::tensor_ops::ImageAdapter
   parameters:
     message_type: "VideoBuffer"
 ---
@@ -217,17 +213,22 @@ components:
 - type: nvidia::gxf::Connection
   parameters:
     source: splitter/image_transmitter
-    target: image_vault/signal
+    target: image_sink/signal
 - type: nvidia::gxf::Connection
   parameters:
     source: splitter/camera_model_transmitter
-    target: camerainfo_vault/signal
+    target: camerainfo_sink/signal
 ---
 name: utils
 components:
 - name: clock
   type: nvidia::gxf::RealtimeClock
-- type: nvidia::gxf::GreedyScheduler
+- type: nvidia::gxf::MultiThreadScheduler
   parameters:
     clock: clock
     stop_on_deadlock: false
+    check_recession_period_ms: 1
+    worker_thread_number: 2
+- type: nvidia::gxf::JobStatistics
+  parameters:
+    clock: clock