Robot basic Vision Functionalities

During our analysis of the basic functionalities of the hacked RoboMaster S1 (hereafter referred to as the 'robot'), we stumbled upon a vision module in the SDK with code examples for detecting Robots and Markers.

Robot detection

The robot has a built-in model for recognising other robots, this was included in the SDK vision module.
It works as an object detection machine learning algorithm, drawing a bounding box over detected robot instances.
The following example code can be run after you connected to a network to which the robot is connected as well with the proper specifications.

Source

import cv2
import robomaster
from robomaster import robot
from robomaster import vision


class RobotInfo:

    def __init__(self, x, y, w, h):
        self._x = x
        self._y = y
        self._w = w
        self._h = h

    @property
    def pt1(self):
        return int((self._x - self._w / 2) * 1280), int((self._y - self._h / 2) * 720)

    @property
    def pt2(self):
        return int((self._x + self._w / 2) * 1280), int((self._y + self._h / 2) * 720)

    @property
    def center(self):
        return int(self._x * 1280), int(self._y * 720)


robots = []


def on_detect_person(person_info):
    number = len(person_info)
    robots.clear()
    for i in range(0, number):
        x, y, w, h = person_info[i]
        robots.append(RobotInfo(x, y, w, h))
        print("robot: x:{0}, y:{1}, w:{2}, h:{3}".format(x, y, w, h))


if __name__ == '__main__':
    ep_robot = robot.Robot()
    ep_robot.initialize(conn_type="sta")

    ep_vision = ep_robot.vision
    ep_camera = ep_robot.camera

    ep_camera.start_video_stream(display=False)
    result = ep_vision.sub_detect_info(name="robot", callback=on_detect_person)

    for i in range(0, 500):
        img = ep_camera.read_cv2_image(strategy="newest", timeout=0.5)
        for j in range(0, len(robots)):
            cv2.rectangle(img, robots[j].pt1, robots[j].pt2, (255, 255, 255))
        cv2.imshow("robots", img)
        cv2.waitKey(1)
    cv2.destroyAllWindows()
    result = ep_vision.unsub_detect_info(name="robot")
    cv2.destroyAllWindows()
    ep_camera.stop_video_stream()
    ep_robot.close()

Vision Markers

The RoboMaster company also includes Vision Markers in there product line. Since these markers were made for the robots, there naturally is a built-in model for recognition. This model is just like the Robot detection described previously. The example code for this is noted down below and can be run if the robot is connect to a router like described here.

Source

import cv2
import robomaster
from robomaster import robot
from robomaster import vision


class MarkerInfo:

    def __init__(self, x, y, w, h, info):
        self._x = x
        self._y = y
        self._w = w
        self._h = h
        self._info = info

    @property
    def pt1(self):
        return int((self._x - self._w / 2) * 1280), int((self._y - self._h / 2) * 720)

    @property
    def pt2(self):
        return int((self._x + self._w / 2) * 1280), int((self._y + self._h / 2) * 720)

    @property
    def center(self):
        return int(self._x * 1280), int(self._y * 720)

    @property
    def text(self):
        return self._info


markers = []


def on_detect_marker(marker_info):
    number = len(marker_info)
    markers.clear()
    for i in range(0, number):
        x, y, w, h, info = marker_info[i]
        markers.append(MarkerInfo(x, y, w, h, info))
        print("marker:{0} x:{1}, y:{2}, w:{3}, h:{4}".format(info, x, y, w, h))


if __name__ == '__main__':
    ep_robot = robot.Robot()
    ep_robot.initialize(conn_type="sta")

    ep_vision = ep_robot.vision
    ep_camera = ep_robot.camera

    ep_camera.start_video_stream(display=False)
    result = ep_vision.sub_detect_info(name="marker", callback=on_detect_marker)

    for i in range(0, 500):
        img = ep_camera.read_cv2_image(strategy="newest", timeout=0.5)
        for j in range(0, len(markers)):
            cv2.rectangle(img, markers[j].pt1, markers[j].pt2, (255, 255, 255))
            cv2.putText(img, markers[j].text, markers[j].center, cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 3)
        cv2.imshow("Markers", img)
        cv2.waitKey(1)
    cv2.destroyAllWindows()

    result = ep_vision.unsub_detect_info(name="marker")
    cv2.destroyAllWindows()
    ep_camera.stop_video_stream()
    ep_robot.close()

Sources:

1. D. (n.d.). RoboMaster-SDK. GitHub. Retrieved October 22, 2021, from https://github.com/dji-sdk/RoboMaster-SDK
2. Introduction — RoboMaster Developer Guide documentation. (n.d.). RoboMaster Developer Guide. Retrieved October 15, 2021, from https://robomaster-dev.readthedocs.io/en/latest/introduction.html

Related issues

Issues: #8 & #31