ROS 交叉编译指南

构建 ROS 项目交叉编译环境

本文介绍了在 X86_64 架构下交叉编译 ARM64 项目的几种方法：

  1. 使用 qemu + chroot
  2. 使用 qemu + docker
  3. 使用 toolchain
  4. 使用 docker + toolchain

在本教程中，我们将重点介绍使用 docker + toolchain 进行交叉编译。这种方法有助于简化交叉编译环境的管理。如果用于 CI/CD，建议使用本机 + toolchain 进行交叉编译，以避免 docker 可能带来的不稳定性。

制作交叉编译所需的根文件系统

首先，我将使用 docker 拉取 ARM64 版本的 Ubuntu 18.04 镜像并构建容器，以配置交叉编译环境。这样做的好处是便于管理，当然也可以选择使用 chroot + git 进行管理。

Dockerfile.arm64_ubuntu1804_ros

创建一个 Dockerfile，用于生成 ARM64 架构的 Ubuntu 18.04 镜像，并配置交叉编译环境：

# 使用基础的 Ubuntu 镜像和架构参数
ARG UBUNTU="18.04"
ARG ARCH="arm64"
FROM ubuntu:${UBUNTU}

# 拷贝本机的 qemu-aarch64-static
COPY qemu-aarch64-static /usr/bin/

# 替换 apt 源为阿里云镜像
RUN sed -i 's|http://archive.ubuntu.com/ubuntu/|http://mirrors.ustc.edu.cn/ubuntu/|g' /etc/apt/sources.list \
    && sed -i 's|http://ports.ubuntu.com/ubuntu-ports/|http://mirrors.ustc.edu.cn/ubuntu-ports/|g' /etc/apt/sources.list

# 更新软件源并安装必要的软件包
RUN apt-get update && apt-get install -y \
    wget \
    && rm -rf /var/lib/apt/lists/*

构建镜像

构建 Docker 镜像：

# 构建用于 ARM64 架构的 Docker 镜像
docker buildx build --platform linux/arm64 -t arm-ubuntu18.04-ros --load -f ./Dockerfile.arm64_ubuntu1804_ros .

利用镜像生成容器

运行以下命令生成并配置容器：

# 生成容器
docker run -it --name rootfs arm-ubuntu18.04-ros /bin/bash

# 配置环境
# 进入容器后再执行 ROS 安装脚本
wget http://fishros.com/install -O fishros && . fishros

# 完成编译测试(编译速度慢)，退出容器
exit

导出和上传容器

将容器打包成镜像，上传到 DockerHub，以便下次使用：

# 1. 将容器打包成镜像
docker commit rootfs my-image:latest

# 2. 登录 DockerHub
docker login

# 3. 标记镜像
docker tag my-image:latest my-username/my-repo:latest

# 4. 上传镜像到 DockerHub
docker push my-username/my-repo:latest

# 5. 将容器导出为 rootfs.tar 文件，用作交叉容器的根文件系统
docker export rootfs > rootfs.tar

确保将 rootfs、my-image、my-username 和 my-repo 替换为实际使用的名称和仓库。

制作交叉编译镜像

启动镜像：

docker run -it \
--name cross-build \
--shm-size=10g \
--ulimit memlock=-1 \
--cpus=12 \
--platform linux/amd64 \
-v /home/ubuntu/Documents/cross-build-docker/work:/app/work \
--user root \
ubuntu:18.04

在 cross-build 容器中，配置交叉编译环境，将 rootfs.tar 添加到挂载目录并执行 update_paths_and_links.sh 脚本：

#!/bin/bash

# 交叉编译根文件系统放置位置
# 检查是否传入参数
if [ $# -eq 0 ]; then
    echo "没有提供根文件系统路径，将使用默认路径"
    exit 1
else
    ROOTFS_DIR="$1"
    echo "使用传入的路径作为根文件系统路径：$ROOTFS_DIR"
fi

# tar -xvf rootfs.tar -C "$ROOTFS_DIR"

# 定义目标目录
# 若有第三方库，在此处添加相关路径
target_dirs_cmake=(
    "$ROOTFS_DIR/usr/lib/aarch64-linux-gnu/cmake"
    "$ROOTFS_DIR/opt/ros/melodic/share"
    "$ROOTFS_DIR/usr/lib/cmake"
)

target_dirs_pkgconfig=(
    "$ROOTFS_DIR/opt/ros/melodic/lib/pkgconfig"
    "$ROOTFS_DIR/usr/share/pkgconfig"
)

# 替换路径函数
replace_paths() {
    local dir="$1"
    local old_path="$2"
    local new_path="$3"

    # 使用 find 命令查找目标目录下的所有 .cmake 或 .pc 文件，并使用 sed 替换路径
    find "$dir" -type f \( -name "*.cmake" -o -name "*.pc" \) -exec sed -i "s#$old_path#$new_path#g" {} +
}

# 修改 .cmake 文件中的路径
for dir in "${target_dirs_cmake[@]}"; do
    replace_paths "$dir" "/usr/lib" "$ROOTFS_DIR/usr/lib"
    replace_paths "$dir" "/usr/include" "$ROOTFS_DIR/usr/include"
    replace_paths "$dir" "/opt/ros" "$ROOTFS_DIR/opt/ros"
done

# 修改 .pc 文件中的路径
for dir in "${target_dirs_pkgconfig[@]}"; do
    replace_paths "$dir" "/usr/lib" "$ROOTFS_DIR/usr/lib"
    replace_paths "$dir" "/usr/include" "$ROOTFS_DIR/usr/include"
    replace_paths "$dir" "/opt/ros" "$ROOTFS_DIR/opt/ros"
done

# 修改 VTKConfig.cmake 文件中的路径，需要根据所用的 VTK 版本设置
vtk_file_path="$ROOTFS_DIR/usr/lib/cmake/vtk-6.3/VTKConfig.cmake"
if [ -f "$vtk_file_path" ]; then
    sed -i "s|set(VTK_INSTALL_PREFIX \"/usr\")|set(VTK_INSTALL_PREFIX \"$ROOTFS_DIR/usr\")|" "$vtk_file_path"
    echo "VTKConfig.cmake file modified."
else
    echo "VTKConfig.cmake file not found."
fi

# 遍历根文件系统中的符号链接并修改路径
find "$ROOTFS_DIR" -type l | while read link; do
    target=$(readlink "$link")  # 获取符号链接的目标
    if [[ $target == /* ]]; then
        # 如果目标是绝对路径，则在其前面添加根文件系统路径
        target="$ROOTFS_DIR$target"
        # 使用修改后的目标路径创建软链接
        ln -sf "$target" "$link"
        echo "Modified symlink: $link --> $target"
    fi
done

echo "All modifications done!"

在容器中配置 ROS 环境并进行编译测试：

# 需要catkin_make,在容器中配置ros环境,执行 ROS 安装脚本
wget http://fishros.com/install -O fishros && . fishros

# 完成交叉编译测试(编译速度快)，退出容器
exit

将容器打包成镜像，并上传到 DockerHub 以便下次使用：

# 将容器打包成镜像
docker commit cross-build your_dockerhub_username/cross-build-ros
# 上传 dockerhub ，便于下次使用
docker push your_dockerhub_username/cross-build-ros

交叉编译镜像使用

要使用交叉编译镜像，还需要编写一个 toolchain.cmake 文件，该文件告诉 CMake 如何在交叉编译环境中查找工具和库。下面是一个示例 toolchain.cmake 文件：

set(CMAKE_SYSTEM_NAME Linux) 
set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)
set(CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR})
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(CMAKE_PREFIX_PATH  ${CMAKE_FIND_ROOT_PATH} ${CMAKE_SYSROOT}/opt/ros/melodic)
set(CMAKE_LIBRARY_PATH /usr/lib/aarch64-linux-gnu ${CMAKE_SYSROOT}/usr/local/lib ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/ ${CMAKE_SYSROOT}/usr/lib)
set(CMAKE_INCLUDE_PATH ${CMAKE_SYSROOT}/usr/local/include ${CMAKE_SYSROOT}/usr/include/aarch64-linux-gnu ${CMAKE_SYSROOT}/usr/include)

set(CMAKE_PREFIX_PATH 
  ${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/cmake/Qt5
  ${CMAKE_SYSROOT}/usr/local/pcl-1.10.0
  ${CMAKE_SYSROOT}/usr/local/opencv-4.2.0
  ${CMAKE_SYSROOT}/usr/local/g2o-20201223/lib/cmake/g2o
  ${CMAKE_SYSROOT}/opt/ros/melodic
  ${CMAKE_PREFIX_PATH}  # 原来的值（如果有）
)

set(CMAKE_LIBRARY_PATH 
  ${CMAKE_SYSROOT}/usr/local/pcl-1.10.0/lib
  ${CMAKE_SYSROOT}/usr/local/opencv-4.2.0/lib
  ${CMAKE_SYSROOT}/usr/local/g2o-20201223/lib
  ${CMAKE_LIBRARY_PATH}
)

set(CMAKE_INCLUDE_PATH 
  ${CMAKE_SYSROOT}/usr/local/pcl-1.10.0/include
  ${CMAKE_SYSROOT}/usr/local/opencv-4.2.0/include
  ${CMAKE_SYSROOT}/usr/local/g2o-20201223/include
  ${CMAKE_INCLUDE_PATH}
)

# 注意：以下路径均基于交叉编译目标平台的 rootfs
# QT5 环境变量
export CMAKE_PREFIX_PATH=${CMAKE_SYSROOT}/usr/lib/aarch64-linux-gnu/cmake/Qt5:$CMAKE_PREFIX_PATH

# PCL 环境变量
export CMAKE_PREFIX_PATH=${CMAKE_SYSROOT}/usr/local/pcl-1.10.0:$CMAKE_PREFIX_PATH
export LD_LIBRARY_PATH=${CMAKE_SYSROOT}/usr/local/pcl-1.10.0/lib:$LD_LIBRARY_PATH
export PKG_CONFIG_PATH=${CMAKE_SYSROOT}/usr/local/pcl-1.10.0/lib/pkgconfig:$PKG_CONFIG_PATH

# OpenCV 环境变量
export CMAKE_PREFIX_PATH=${CMAKE_SYSROOT}/usr/local/opencv-4.2.0:$CMAKE_PREFIX_PATH
export LD_LIBRARY_PATH=${CMAKE_SYSROOT}/usr/local/opencv-4.2.0/lib:$LD_LIBRARY_PATH
export PKG_CONFIG_PATH=${CMAKE_SYSROOT}/usr/local/opencv-4.2.0/lib/pkgconfig:$PKG_CONFIG_PATH

# g2o 环境变量
export CMAKE_PREFIX_PATH=${CMAKE_SYSROOT}/usr/local/g2o-20201223/lib/cmake/g2o:$CMAKE_PREFIX_PATH
export LD_LIBRARY_PATH=${CMAKE_SYSROOT}/usr/local/g2o-20201223/lib:$LD_LIBRARY_PATH
export PKG_CONFIG_PATH=${CMAKE_SYSROOT}/usr/local/g2o-20201223/lib/pkgconfig:$PKG_CONFIG_PATH

# 包含路径（头文件路径）
export CPLUS_INCLUDE_PATH=${CMAKE_SYSROOT}/usr/local/pcl-1.10.0/include:${CMAKE_SYSROOT}/usr/local/opencv-4.2.0/include:${CMAKE_SYSROOT}/usr/local/g2o-20201223/include:$CPLUS_INCLUDE_PATH
export C_INCLUDE_PATH=${CMAKE_SYSROOT}/usr/local/pcl-1.10.0/include:${CMAKE_SYSROOT}/usr/local/opencv-4.2.0/include:${CMAKE_SYSROOT}/usr/local/g2o-20201223/include:$C_INCLUDE_PATH

下面是一个示例 xbuild.sh 脚本，用于在交叉编译环境中构建 ROS 项目。

#!/bin/bash

# 切换到ROS工作空间目录
cd ./catkin_ws

# 记录脚本开始时间
start_time=$(date +"%Y-%m-%d %H:%M:%S")
echo "脚本开始时间：$start_time"

set -e

echo_fatal(){
  echo -e "\E[1;31m""$@\033[0m"
  exit 1
}

echo_err(){
  echo -e "\E[1;31m""$@\033[0m"
}

echo_ok(){
  echo -e "\E[1;32m""$@\033[0m"
}

source /opt/ros/melodic/setup.bash
CMAKE_SYSROOT="/cross-arm/crosssysroot_nano"
LIB_DIR="aarch64-linux-gnu"
# toolchain.cmake 所在目录
toolchina="/app/work"
CMAKE_ARGS="-DARM64=True"
ROS_VERSION="melodic"

echo ${ROS_VERSION}
export PKG_CONFIG_PATH=${CMAKE_SYSROOT}/opt/ros/${ROS_VERSION}/lib/pkgconfig/:${CMAKE_SYSROOT}/usr/lib/pkgconfig/:${CMAKE_SYSROOT}/usr/share/pkgconfig:${CMAKE_SYSROOT}/usr/lib/${LIB_DIR}/pkgconfig/

export LD_LIBRARY_PATH=${CMAKE_SYSROOT}/opt/ros/${ROS_VERSION}/lib:${CMAKE_SYSROOT}/usr/lib/${LIB_DIR}:${CMAKE_SYSROOT}/lib/${LIB_DIR}:${CMAKE_SYSROOT}/usr/${LIB_DIR}/lib:${CMAKE_SYSROOT}/usr/lib:${CMAKE_SYSROOT}/lib/:/usr/${LIB_DIR}/lib:/usr/lib/${LIB_DIR}:/lib/${LIB_DIR}

echo_ok "Start to build ..."
catkin_make install -DCMAKE_TOOLCHAIN_FILE=${toolchina}/toolchain.cmake ${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release -DCATKIN_ENABLE_TESTING=false -DCATKIN_WHITELIST_PACKAGES="" -j8

echo_ok "Start to reset path..."
sed -i "s#$(pwd)#/home/ubuntu/catkin_ws#g" `find install/ -name "*.py"`
sed -i "s#$(pwd)#/home/ubuntu/catkin_ws#g" `find install/ -name "*.sh"`
sed -i "s#$(pwd)#/home/ubuntu/catkin_ws#g" `find install/ -name "*.pc"`
sed -i "s#$(pwd)#/home/ubuntu/catkin_ws#g" `find install/ -name "*.cmake"`
sed -i "s#${CMAKE_SYSROOT}##g" `find install/ -name "*.py"`
sed -i "s#${CMAKE_SYSROOT}##g" `find install/ -name "*.cmake"`

echo_ok "Start to delete old files..."
rm -rf deploy
rm -rf zip
rm -rf *.zip
rm -rf *.deb
mkdir -p deploy/DEBIAN
mkdir -p deploy/home/ubuntu/catkin_ws
cp -r $(ls -A -1 | grep -v 'src\|build\|devel\|deploy\|bak\|.vscode\|.catkin_workspace\|pack.sh') deploy/home/ubuntu/catkin_ws

chmod 775 ../catkin_ws -R
chown ubuntu ../catkin_ws -R
chgrp ubuntu ../catkin_ws -R

echo_ok "Start to make packages..."
touch deploy/DEBIAN/control
echo "Package: test" >> deploy/DEBIAN/control
echo "Version: 001" >> deploy/DEBIAN/control
echo "Architecture: arm64" >> deploy/DEBIAN/control
echo "Maintainer: " >> deploy/DEBIAN/control
echo "Installed-Size: 110000" >> deploy/DEBIAN/control
echo "Description: ." >> deploy/DEBIAN/control

dpkg -b deploy ./
MD5_DEB=$(md5sum test*.deb | awk -F ' ' '{print $1}')

chmod 775 ../catkin_ws -R
chown ubuntu ../catkin_ws -R
chgrp ubuntu ../catkin_ws -R

# 记录脚本结束时间
end_time=$(date +"%Y-%m-%d %H:%M:%S")
echo "脚本结束时间：$end_time"

# 计算脚本运行时间
start_timestamp=$(date -d "$start_time" +%s)
end_timestamp=$(date -d "$end_time" +%s)
runtime=$((end_timestamp - start_timestamp))

echo "脚本运行时间：$runtime 秒"

任务完成后，将生成一个deb包，可用于将交叉编译的ROS程序安装到对应的ARM64平台上。