Merge branch 'master' into dev

Fixed glued pipeline elements. 
https://github.com/maladzenkau/image2rtsp/issues/8

CITATION.cff

@@ -0,0 +1,12 @@
+cff-version: 1.2.0
+preferred-citation:
+  type: software
+  message: If you use this software, please cite it as below.
+  authors:
+  - family-names: Maladzenkau
+    given-names: Dzmitry
+  title: "image2rtsp"
+  version: 1.0
+  date-released: 2023-27-11
+  license: BSD 3-Clause
+  url: "https://github.com/45kmh/image2rtsp"

CMakeLists.txt

@@ -11,6 +11,9 @@ if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 14)
 endif()
 
+# Suppress all warnings
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
+
 if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_compile_options(-Wall -Wextra -Wpedantic)
 endif()
@@ -19,11 +22,15 @@ endif()
 find_package(ament_cmake REQUIRED)
 find_package(rclcpp REQUIRED)
 find_package(sensor_msgs REQUIRED)
+find_package(PkgConfig REQUIRED)
+find_package(OpenCV REQUIRED)
 pkg_check_modules(GST REQUIRED 
   gstreamer-1.0
   gstreamer-app-1.0
   gstreamer-rtsp-server-1.0
 )
+include_directories(${OpenCV_INCLUDE_DIRS})
+
 file(GLOB SOURCES src/video.cpp)
 
 add_executable(image2rtsp src/image2rtsp.cpp ${SOURCES})
@@ -33,7 +40,8 @@ include_directories(
   ${GST_INCLUDE_DIRS}
   )
 target_link_libraries(image2rtsp 
-  ${GST_LIBRARIES} 
+  ${GST_LIBRARIES}
+  ${OpenCV_LIBS}
   )
 
 install(TARGETS

README.md

@@ -2,11 +2,17 @@
 This project is a migration from ROS1 to ROS2. The original code was developed by [CircusMonkey](https://github.com/CircusMonkey/ros_rtsp/tree/master). I would like to express my gratitude for his contribution.
 
 ## image2rtsp
-This project enables the conversion of a selected ROS2 topic of type `sensor_msgs::msg::Image` into an `RTSP` stream, with an anticipated delay of approximately 30-50ms. The generated stream can be utilized for various purposes such as remote control, object detection tasks, monitoring, and more. Please note that the migration process is ongoing, and therefore, the complete functionality of the original package is not yet available.
+This project enables the conversion of a selected ROS2 topic of type `sensor_msgs::msg::Image` or `sensor_msgs::msg::CompressedImage` into an `RTSP` stream, with an anticipated delay of approximately 0,3-0,4s. It also supports usb camera as a direct source. The generated stream can be utilized for various purposes such as remote control, object detection tasks, monitoring and more.
 
-The development is being carried out on Ubuntu 20.04 with ROS2 Foxy.
+Currently supported and tested `sensor_msgs::msg::Image` formats: "**rgb8**", "**rgba8**", "**rgb16**", "**rgba16**", "**bgr8**", "**bgra8**", "**bgr16**", "**bgra16**", "**mono8**", "**mono16**", "**yuv422_yuy2**". 
+
+Supported and tested `sensor_msgs::msg::CompressedImage` formats: "**rgb8; jpeg compressed bgr8**". Other formats may work as well with some color scheme deviations. Please open an issue in this case and attach your **ros2 bag**, so i can fix it. If you need some specific unsupported format, create an issue and i will try to add it as soon as possible, but normally it takes pretty long, so dont hesitate to create a PR.
+
+The development is being carried out on Ubuntu 22.04 with ROS2 Humble. Tested with Intel RealSense d435i.
+
+You are reading now the README for a **default** ROS2 package. If you want to use this package written as a ROS2 component, checkout `ros2_component` branch. 
 ## Dependencies
-- ROS2 Foxy
+- ROS2 Humble
 
 - gstreamer libs:
 ```bash
@@ -19,28 +25,54 @@ sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer-plugins-good1
       mkdir -p ros2_ws/src
       cd ros2_ws/src/
       ```
-  - Clone the package:
+  - Clone the package and then navigate into the directory `image2rtsp`:
       ```bashrc
-      git clone https://github.com/45kmh/image2rtsp.git
+      git clone https://github.com/maladzenkau/image2rtsp.git --single-branch
       ```
   - Adjust  `parameters.yaml` according to your needs:
       ```bashrc
       gedit ~/ros2_ws/src/image2rtsp/config/parameters.yaml
       ```
 # Example ROS2 Image topic stream
-    topic: "/color/image_raw"  # The ROS2 topic to subscribe to
-    mountpoint: "/back"        # Choose the mountpoint for the rtsp stream. 
-                               # This will be able to be accessed from rtsp://<server_ip>/portAndMountpoint
-    caps: "video/x-raw,framerate=10/1,width=640,height=480"
-                               # Set the caps to be applied after getting the ROS2 Image and before the x265 encoder.
-    bitrate: "500"
-    port: "8554"
-    local_only: True           # True = rtsp://127.0.0.1:port (The stream is accessible only from the local machine)
-                               # False = rtsp://0.0.0.0:portAndMountpoint (The stream is accessible from the outside) 
-                               # For example, to access the stream running on the machine with IP = 192.168.20.20,
-                               # use rtsp://192.186.20.20:portAndMountpoint
-                               # True = rtsp://127.0.0.1:portAndMountpoint (The stream is accessible only from the local machine)
-  - Save your configuration and navigate to `image2rtsp ` colcon root, source and build the package:
+    # If the source is a ros2 topic (default case)
+      compressed:       False
+      topic:            "color/image_raw"
+      default_pipeline: |
+                        ( appsrc name=imagesrc do-timestamp=true min-latency=0 
+                          max-latency=0 max-bytes=1000 is-live=true !
+                          videoconvert !
+                          videoscale !
+                          video/x-raw, framerate=30/1, width=640, height=480 !
+                          x264enc tune=zerolatency bitrate=500 key-int-max=30 !
+                          video/x-h264, profile=baseline !
+                          rtph264pay name=pay0 pt=96 )
+
+      # Notice: The framerate setting does not affect the RTSP stream — it entirely depends on the ros2 topic frequency. 
+      # It is included in the pipeline and code for structural reasons. You can likely remove it from the pipeline without impacting the package's behavior.
+
+
+      # If camera serves as a source
+      camera:           False      
+      camera_pipeline:  |
+                        ( v4l2src device=/dev/video0 !
+                          videoconvert !
+                          videoscale !
+                          video/x-raw, framerate=30/1, width=640, height=480 !
+                          x264enc tune=zerolatency bitrate=500 key-int-max=30 !
+                          video/x-h264, profile=baseline !
+                          rtph264pay name=pay0 pt=96 )
+
+      # Notice: Here the framerate might be set to the camera framerate, otherwise "503 Service Unavailable" error will appear.
+
+      # RTSP setup
+      mountpoint:       "/back"
+      port:             "8554"
+      local_only:       True     # True = rtsp://127.0.0.1:portAndMountpoint (The stream is accessible only from the local machine)
+                                 # False = rtsp://0.0.0.0:portAndMountpoint (The stream is accessible from the outside) 
+                                 # For example, to access the stream running on the machine with IP = 192.168.20.20,
+                                 # use rtsp://192.186.20.20:portAndMountpoint
+
+  - Save your configuration and navigate to `ros2_ws` colcon root, source and build the package:
       ```bashrc
       cd ~/ros2_ws/
       colcon build --packages-select image2rtsp
@@ -64,13 +96,6 @@ cd ~/ros2_ws/
 source install/setup.bash
 ros2 launch image2rtsp rtsp.launch.py 
 ```
-## Limitations
-- As was previously mentioned, this package allows the conversion of only one topic into an RTSP Stream.
-- Some machines may have a significantly bigger delay of about 150-200ms. The reason is currently unknown.
-- Its possible to use this package with Humble if ROS2 parameters are hardcoded as variables. 
-## To Do
-- Add a branch with the package rewritten as a ROS2 component
-- Port the package to Nvidia Jetson Orin. Will the use of the Nvidia encoder make the delay smaller?
-- Port packages to ROS2 Humble.
-- Complete the functionality according to the functionality of the original ROS package.
+## Note
 
+- The YAML configuration allows you to fully customize the pipeline according to your needs. This package does not provide any built-in acceleration. As its stability has not been validated across a wide range of Linux systems using advanced hardware or software techniques, support for such configurations is left to the user. There are no plans to update the package to support GPU/CPU acceleration. Please do not open issues related to software/hardware acceleration if they are directly related to the GStreamer pipeline itself. 

config/parameters.yaml

@@ -1,12 +1,39 @@
 /image2rtsp:
    ros__parameters:
-      topic: "/color/image_raw"
-      mountpoint: "/back"
-      bitrate: "500"
-      caps: "video/x-raw,framerate=10/1,width=640,height=480"
-      port: "8554"
-      local_only: True     # True = rtsp://127.0.0.1:port (The stream is accessible only from the local machine)
-                           # False = rtsp://0.0.0.0:portAndMountpoint (The stream is accessible from the outside) 
-                           # For example, to access the stream running on the machine with IP = 192.168.20.20,
-                           # use rtsp://192.186.20.20:portAndMountpoint
-                           # True = rtsp://127.0.0.1:portAndMountpoint (The stream is accessible only from the local machine)
+
+      # If the source is a ros2 topic (default case)
+      compressed:       False
+      topic:            "color/image_raw"
+      default_pipeline: |
+                        ( appsrc name=imagesrc do-timestamp=true min-latency=0 max-latency=0 max-bytes=1000 is-live=true !
+                          videoconvert !
+                          videoscale !
+                          video/x-raw, framerate=30/1, width=640, height=480 !
+                          x264enc tune=zerolatency bitrate=500 key-int-max=30 !
+                          video/x-h264, profile=baseline !
+                          rtph264pay name=pay0 pt=96 )
+
+      # Notice: The framerate setting does not affect the RTSP stream — it entirely depends on the ros2 topic frequency. 
+      # It is included in the pipeline and code for structural reasons. You can likely remove it from the pipeline without impacting the package's behavior.
+
+
+      # If camera serves as a source
+      camera:           False      
+      camera_pipeline:  |
+                        ( v4l2src device=/dev/video0 !
+                          videoconvert !
+                          videoscale !
+                          video/x-raw, framerate=30/1, width=640, height=480 !
+                          x264enc tune=zerolatency bitrate=500 key-int-max=30 !
+                          video/x-h264, profile=baseline !
+                          rtph264pay name=pay0 pt=96 )
+
+      # Notice: Here the framerate might be set to the camera framerate, otherwise "503 Service Unavailable" error will appear.
+
+      # RTSP setup
+      mountpoint:       "/back"
+      port:             "8554"
+      local_only:       True     # True = rtsp://127.0.0.1:portAndMountpoint (The stream is accessible only from the local machine)
+                                 # False = rtsp://0.0.0.0:portAndMountpoint (The stream is accessible from the outside) 
+                                 # For example, to access the stream running on the machine with IP = 192.168.20.20,
+                                 # use rtsp://192.186.20.20:portAndMountpoint

include/image2rtsp.hpp

@@ -6,6 +6,8 @@
 #include <rclcpp/rclcpp.hpp>
 #include <string>
 #include "sensor_msgs/msg/image.hpp"
+#include "sensor_msgs/msg/compressed_image.hpp"
+#include <opencv2/opencv.hpp>
 
 using namespace std;
 
@@ -17,21 +19,25 @@ public:
 private:
     string topic;
     string mountpoint;
-    string bitrate;
-    string caps;
     string port;
     string pipeline;
-    string pipeline_head;
-    string pipeline_tail;
+    string default_pipeline;
+    string camera_pipeline;
+    uint framerate;
     bool local_only;
+    bool camera;
+    bool compressed;
     GstAppSrc *appsrc;
 
     void video_mainloop_start();
     void rtsp_server_add_url(const char *url, const char *sPipeline, GstElement **appsrc);
     void topic_callback(const sensor_msgs::msg::Image::SharedPtr msg);
+    void compressed_topic_callback(const sensor_msgs::msg::CompressedImage::SharedPtr msg);
+    uint extract_framerate(const std::string& pipeline, uint default_framerate);
     GstRTSPServer *rtsp_server_create(const string &port, const bool local_only);
     GstCaps *gst_caps_new_from_image(const sensor_msgs::msg::Image::SharedPtr &msg);
     rclcpp::Subscription<sensor_msgs::msg::Image>::SharedPtr subscription_;
+    rclcpp::Subscription<sensor_msgs::msg::CompressedImage>::SharedPtr subscription_compressed_;
 };
 
 static void media_configure(GstRTSPMediaFactory *factory, GstRTSPMedia *media, GstElement **appsrc);

include/image_encodings.h

@@ -98,7 +98,7 @@ namespace sensor_msgs
     // Miscellaneous
     // This is the UYVY version of YUV422 codec http://www.fourcc.org/yuv.php#UYVY
     // with an 8-bit depth
-    const std::string YUV422="yuv422";
+    const std::string YUV422="yuv422_yuy2";
 
     // Prefixes for abstract image encodings
     const std::string ABSTRACT_ENCODING_PREFIXES[] = {

launch/rtsp.launch.py

@@ -1,10 +1,11 @@
 from launch import LaunchDescription
 from launch.actions import ExecuteProcess
+from ament_index_python.packages import get_package_share_directory
 import os
 
 def generate_launch_description():
-    home_dir = os.path.expanduser("~")
-    script_path = os.path.join(home_dir, 'ros2_ws/src/image2rtsp/python/rtsp.py')
+    pkg_share = get_package_share_directory('image2rtsp')
+    script_path = os.path.join(pkg_share, '../../../../src/image2rtsp/python/rtsp.py')
 
     return LaunchDescription(
         [

package.xml

@@ -11,6 +11,7 @@
 
   <depend>rclcpp</depend>
   <depend>sensor_msgs</depend>
+  <depend>open_CV</depend>
 
   <test_depend>ament_lint_auto</test_depend>
   <test_depend>ament_lint_common</test_depend>

src/image2rtsp.cpp

@@ -9,53 +9,108 @@ using std::placeholders::_1;
 
 Image2rtsp::Image2rtsp() : Node("image2rtsp"){
     // Declare and get the parameters
-    this->declare_parameter("topic");
-    this->declare_parameter("mountpoint");
-    this->declare_parameter("bitrate");
-    this->declare_parameter("caps");
-    this->declare_parameter("port");
-    this->declare_parameter("local_only");
+    this->declare_parameter("topic",        "/color/image_raw");
+    this->declare_parameter("mountpoint",   "/back");
+    this->declare_parameter("port",         "8554");
+    this->declare_parameter("local_only",   true);
+    this->declare_parameter("camera",       false);
+    this->declare_parameter("compressed",   false);
 
-    topic = this->get_parameter("topic").as_string();
-    mountpoint = this->get_parameter("mountpoint").as_string();
-    bitrate = this->get_parameter("bitrate").as_string();
-    caps = this->get_parameter("caps").as_string();
-    port = this->get_parameter("port").as_string();
-    local_only = this->get_parameter("local_only").as_bool();
+    this->declare_parameter("default_pipeline",   R"(
+                                                    ( appsrc name=imagesrc do-timestamp=true min-latency=0 max-latency=0 max-bytes=1000 is-live=true !
+                                                    videoconvert !
+                                                    videoscale !
+                                                    video/x-raw, framerate=30/1, width=640, height=480 !
+                                                    x264enc tune=zerolatency bitrate=500 key-int-max=30 !
+                                                    video/x-h264, profile=baseline !
+                                                    rtph264pay name=pay0 pt=96 )
+                                                    )");
 
-    // Check if the parameter is set, since no default value is provided
-    if (!this->has_parameter("topic") || !this->has_parameter("mountpoint") || !this->has_parameter("bitrate") || !this->has_parameter("caps") || !this->has_parameter("port")){
-        rclcpp::shutdown(); // Shutdown the node if there are some issues with launch file
-        return;
-    }
+    this->declare_parameter("camera_pipeline",    R"(
+                                                    ( v4l2src device=/dev/video0 !
+                                                    videoconvert !
+                                                    videoscale !
+                                                    video/x-raw, framerate=30/1, width=640, height=480 !
+                                                    x264enc tune=zerolatency bitrate=500 key-int-max=30 !
+                                                    video/x-h264, profile=baseline !
+                                                    rtph264pay name=pay0 pt=96 )
+                                                    )");
+
+    topic               = this->get_parameter("topic").as_string();
+    mountpoint          = this->get_parameter("mountpoint").as_string();
+    port                = this->get_parameter("port").as_string();
+    local_only          = this->get_parameter("local_only").as_bool();
+    camera              = this->get_parameter("camera").as_bool();
+    compressed          = this->get_parameter("compressed").as_bool();
+    default_pipeline    = this->get_parameter("default_pipeline").as_string();
+    camera_pipeline     = this->get_parameter("camera_pipeline").as_string();
 
     // Start the subscription
-    subscription_ = this->create_subscription<sensor_msgs::msg::Image>(topic, 10, std::bind(&Image2rtsp::topic_callback, this, _1));
+    if (camera == false){
+        if (compressed == false){
+            subscription_ = this->create_subscription<sensor_msgs::msg::Image>(topic, 10, std::bind(&Image2rtsp::topic_callback, this, _1));
+            RCLCPP_INFO(this->get_logger(), "Subscribing to sensor_msgs::msg::Image");
+        }
+        else {
+            subscription_compressed_ = this->create_subscription<sensor_msgs::msg::CompressedImage>(topic, 10, std::bind(&Image2rtsp::compressed_topic_callback, this, _1));
+            RCLCPP_INFO(this->get_logger(), "Subscribing to sensor_msgs::msg::CompressedImage");
+        }
+    }
+    else {
+        RCLCPP_INFO(this->get_logger(), "Trying to access camera device");
+    }
 
     // Start the RTSP server
     video_mainloop_start();
     rtsp_server = rtsp_server_create(port, local_only);
     appsrc = NULL;
-    // Setup the pipeline
-    pipeline_head = "( appsrc name=imagesrc do-timestamp=true min-latency=0 max-latency=0 max-bytes=1000 is-live=true ! videoconvert ! videoscale ! ";
-    pipeline_tail = "key-int-max=30 ! video/x-h264, profile=baseline ! rtph264pay name=pay0 pt=96 )";
-    pipeline = pipeline_head + caps + " ! x264enc tune=zerolatency bitrate=" + bitrate + pipeline_tail;
-    rtsp_server_add_url(mountpoint.c_str(), pipeline.c_str(), (GstElement **)&(appsrc));
+
+    pipeline = camera ? camera_pipeline : default_pipeline;
+    framerate = extract_framerate(pipeline, 30);
+    rtsp_server_add_url(mountpoint.c_str(), pipeline.c_str(), camera ? nullptr : (GstElement **)&appsrc);
+
     RCLCPP_INFO(this->get_logger(), "Stream available at rtsp://%s:%s%s", gst_rtsp_server_get_address(rtsp_server), port.c_str(), mountpoint.c_str());
 }
 
+uint Image2rtsp::extract_framerate(const std::string& pipeline, uint default_framerate = 30) {
+    std::string search_str = "framerate=";
+    size_t pos = pipeline.find(search_str);
+    if (pos == std::string::npos) {
+        RCLCPP_WARN(this->get_logger(), "Framerate not found in pipeline, using default: %d", default_framerate);
+        return default_framerate;
+    }
+
+    pos += search_str.length();
+
+    size_t end_pos = pipeline.find_first_of("/,", pos);
+    if (end_pos == std::string::npos) {
+        RCLCPP_WARN(this->get_logger(), "Invalid framerate format in pipeline, using default: %d", default_framerate);
+        return default_framerate;
+    }
+
+    std::string framerate_str = pipeline.substr(pos, end_pos - pos);
+
+    framerate_str.erase(0, framerate_str.find_first_not_of(" \t"));
+    framerate_str.erase(framerate_str.find_last_not_of(" \t") + 1);
+    
+    try {
+        uint framerate = std::stoi(framerate_str);
+        if (framerate <= 0) {
+            RCLCPP_WARN(this->get_logger(), "Invalid framerate value %d, using default: %d", framerate, default_framerate);
+            return default_framerate;
+        }
+        RCLCPP_INFO(this->get_logger(), "Using set framerate %d", framerate);
+        return framerate;
+    } catch (const std::exception& e) {
+        RCLCPP_WARN(this->get_logger(), "Failed to parse framerate '%s', using default: %d", framerate_str.c_str(), default_framerate);
+        return default_framerate;
+    }
+}
+
 int main(int argc, char *argv[]){
     rclcpp::init(argc, argv);
-    try{
-        auto node = std::make_shared<Image2rtsp>();
-        if (rclcpp::ok()){
-            rclcpp::spin(node);
-        }
-    }catch (const std::exception &e){
-        RCLCPP_INFO(rclcpp::get_logger("image2rtsp"), "One or more required parameters are not set. Shutting down the node.");
-        RCLCPP_INFO(rclcpp::get_logger("image2rtsp"), "Tip: Start the node only using launch file. Also rebuild after making changes in the YAML file");
-        RCLCPP_ERROR(rclcpp::get_logger("image2rtsp"), "Parameter is not set: %s", e.what());
-    }
+    auto node = std::make_shared<Image2rtsp>();
+    rclcpp::spin(node);
     rclcpp::shutdown();
     return 0;
 }

src/video.cpp

@@ -114,6 +114,7 @@ GstCaps *Image2rtsp::gst_caps_new_from_image(const sensor_msgs::msg::Image::Shar
         {sensor_msgs::image_encodings::BGRA16, "BGRA16"},
         {sensor_msgs::image_encodings::MONO8, "GRAY8"},
         {sensor_msgs::image_encodings::MONO16, "GRAY16_LE"},
+        {sensor_msgs::image_encodings::YUV422, "YUY2"},
     };
 
     if (msg->is_bigendian){
@@ -131,7 +132,7 @@ GstCaps *Image2rtsp::gst_caps_new_from_image(const sensor_msgs::msg::Image::Shar
                                "format", G_TYPE_STRING, format->second.c_str(),
                                "width", G_TYPE_INT, msg->width,
                                "height", G_TYPE_INT, msg->height,
-                               "framerate", GST_TYPE_FRACTION, 10, 1,
+                               "framerate", GST_TYPE_FRACTION, framerate, 1,
                                nullptr);
 }
 
@@ -166,4 +167,44 @@ void Image2rtsp::topic_callback(const sensor_msgs::msg::Image::SharedPtr msg){
         GST_BUFFER_FLAG_SET(buf, GST_BUFFER_FLAG_LIVE);
         gst_app_src_push_buffer(appsrc, buf);
     }
+}
+
+void Image2rtsp::compressed_topic_callback(const sensor_msgs::msg::CompressedImage::SharedPtr msg){
+    if (appsrc == NULL) return;
+    // Decompress the image
+    cv::Mat img = cv::imdecode(cv::Mat(msg->data), cv::IMREAD_UNCHANGED);
+    if (img.empty()) {
+        RCLCPP_ERROR(this->get_logger(), "Failed to decompress image");
+        return;
+    }
+
+    // Determine the GStreamer caps
+    std::string gst_format;
+    switch (img.type()) {
+        case CV_8UC3: gst_format = "BGR"; break;    // BGR images
+        case CV_8UC4: gst_format = "RGBA"; break;   // RGBA images
+        case CV_8UC1: gst_format = "GRAY8"; break;  // Grayscale images
+        default:
+            RCLCPP_ERROR(this->get_logger(), "Unsupported image type");
+            return;
+    }
+
+    GstCaps *caps = gst_caps_new_simple("video/x-raw",
+                                        "format", G_TYPE_STRING, gst_format.c_str(),
+                                        "width", G_TYPE_INT, img.cols,
+                                        "height", G_TYPE_INT, img.rows,
+                                        "framerate", GST_TYPE_FRACTION, framerate, 1,
+                                        nullptr);
+
+    // Set caps on appsrc
+    gst_app_src_set_caps(appsrc, caps);
+    gst_caps_unref(caps);
+
+    // Create a GstBuffer and fill it with the image data
+    GstBuffer *buf = gst_buffer_new_allocate(nullptr, img.total() * img.elemSize(), nullptr);
+    gst_buffer_fill(buf, 0, img.data, img.total() * img.elemSize());
+    GST_BUFFER_FLAG_SET(buf, GST_BUFFER_FLAG_LIVE);
+
+    // Push the buffer to GStreamer
+    gst_app_src_push_buffer(appsrc, buf);
 }