From d1fccca77106b461c7ac6b036e32103102db0197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 13 May 2020 08:51:17 -0400 Subject: [PATCH 1/3] Split Windows Dockerfile into two. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One for Dashing (and Eloquent) and one for Foxy. Signed-off-by: Steven! Ragnarök --- ...Dockerfile.msvc2019 => Dockerfile.dashing} | 0 windows_docker_resources/Dockerfile.foxy | 153 ++++++++++++++++++ 2 files changed, 153 insertions(+) rename windows_docker_resources/{Dockerfile.msvc2019 => Dockerfile.dashing} (100%) create mode 100644 windows_docker_resources/Dockerfile.foxy diff --git a/windows_docker_resources/Dockerfile.msvc2019 b/windows_docker_resources/Dockerfile.dashing similarity index 100% rename from windows_docker_resources/Dockerfile.msvc2019 rename to windows_docker_resources/Dockerfile.dashing diff --git a/windows_docker_resources/Dockerfile.foxy b/windows_docker_resources/Dockerfile.foxy new file mode 100644 index 000000000..65ad26d32 --- /dev/null +++ b/windows_docker_resources/Dockerfile.foxy @@ -0,0 +1,153 @@ +# escape=` +# This Dockerfile needs to be built from the parent directory (ros2/ci) so the build context +# includes the python scripts + +# To find this value run in powershell: +# $(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion').ReleaseId +ARG WINDOWS_RELEASE_ID=1909 + +# In order to ensure the image is correctly compatible with the host system a +# more precise version than the release id is required. To find this value run +# the following in powershell. +# $(Get-ItemProperty -Path 'HKLM:SOFTWARE\Microsoft\Windows NT\CurrentVersion\Update\TargetingInfo\Installed\Server.OS.amd64' -Name Version).Version" +# Fall back to the Release ID +ARG WINDOWS_RELEASE_VERSION=$WINDOWS_RELEASE_ID + +# Indicates that the windows image will be used as the base image. Must be same or older than host. +# --isolation=hyperv is needed for both build/run if the image id is older than the host id +# Use --isolation=process if you need to build in a mounted volume +FROM mcr.microsoft.com/windows:$WINDOWS_RELEASE_VERSION + +# These are versioned files, so they shouldn't invalidate caches. Renaming to fixed names. +# Regularly updated installers are next to their associated code. +ADD https://github.com/ADLINK-IST/opensplice/releases/download/OSPL_V6_9_190925OSS_RELEASE/PXXX-VortexOpenSplice-6.9.190925OSS-HDE-x86_64.win-vs2019-installer.zip C:\TEMP\OpenSplice.zip + +# OpenSSL +ADD https://slproweb.com/download/Win64OpenSSL-1_0_2u.exe C:\TEMP\Win64OpenSSL.exe + +# OpenCV +ADD https://github.com/ros2/ros2/releases/download/opencv-archives/opencv-3.4.6-vc16.VS2019.zip C:\TEMP\opencv.zip + +# Python 3.7 +ADD https://www.python.org/ftp/python/3.7.6/python-3.7.6-amd64.exe C:\TEMP\python-37.exe + +# Custom choco packages +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/asio.1.12.1.nupkg C:\TEMP\asio.nupkg +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/bullet.2.89.0.nupkg C:\TEMP\bullet.nupkg +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/cunit.2.1.3.nupkg C:\TEMP\cunit.nupkg +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/eigen.3.3.4.nupkg C:\TEMP\eigen.nupkg +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/log4cxx.0.10.0.nupkg C:\TEMP\log4cxx.nupkg +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/tinyxml-usestl.2.6.2.nupkg C:\TEMP\tinyxml.nupkg +ADD https://github.com/ros2/choco-packages/releases/download/2020-02-24/tinyxml2.6.0.0.nupkg C:\TEMP\tinyxml2.nupkg + +# xmllint files +ADD https://www.zlatkovic.com/pub/libxml/64bit/libxml2-2.9.3-win32-x86_64.7z C:\TEMP\libxml2.7z +ADD https://www.zlatkovic.com/pub/libxml/64bit/zlib-1.2.8-win32-x86_64.7z C:\TEMP\zlib.7z +ADD https://www.zlatkovic.com/pub/libxml/64bit/iconv-1.14-win32-x86_64.7z C:\TEMP\iconv.7z + +# Restore the default Windows shell for correct batch processing. +SHELL ["cmd", "/S", "/C"] + +RUN C:\TEMP\python-37.exe /quiet ` + TargetDir=C:\Python37 ` + PrependPath=1 ` + Include_debug=1 ` + Include_symbols=1 + +# Install Chocolatey by powershell script +RUN powershell -noexit "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" + +# choco installs +RUN choco install -y cmake curl git vcredist2013 vcredist140 cppcheck patch +RUN choco install -y -s C:\TEMP asio cunit eigen tinyxml-usestl tinyxml2 log4cxx bullet + +RUN C:\TEMP\Win64OpenSSL.exe /VERYSILENT + +# For extracting .7z files +ADD https://www.7-zip.org/a/7z1900-x64.exe C:\TEMP\ +RUN C:\TEMP\7z1900-x64.exe /S /D="C:\Program Files\7-zip" +RUN setx PATH "%PATH%;C:\Program Files\7-zip" + +# Install xmllint +# 7z.exe command line usage https://sevenzip.osdn.jp/chm/cmdline/commands/extract.htm +RUN 7z.exe x C:\TEMP\libxml2.7z -aoa -oC:\xmllint +RUN 7z.exe x C:\TEMP\zlib.7z -aoa -oC:\xmllint +RUN 7z.exe x C:\TEMP\iconv.7z -aoa -oC:\xmllint +RUN 7z.exe x C:\TEMP\opencv.zip -aoa -oC:\ + +RUN 7z.exe x C:\TEMP\OpenSplice.zip -aoa -oC:\opensplice + +# Environment setup +ENV OPENSSL_CONF C:\OpenSSL-Win64\bin\openssl.cfg +ENV OpenCV_DIR C:\opencv +ENV OSPL_HOME C:\opensplice\HDE\x86_64.win64 +# You can't use ENV to append to the PATH https://stackoverflow.com/questions/42092932/appending-to-path-in-a-windows-docker-container +RUN setx PATH "%PATH%;C:\Program Files\Git\cmd;C:\Program Files\CMake\bin;C:\OpenSSL-Win64\bin\;C:\xmllint\bin;"C:\opencv\x64\vc16\bin" + +RUN powershell -Command Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 +RUN mkdir C:\ws +WORKDIR C:\ci + +# Invalidate once a day to cause online installers to check for updates +RUN echo "@todays_date" + +# Qt5 online installer +ADD http://download.qt.io/official_releases/online_installers/qt-unified-windows-x86-online.exe C:\TEMP\ + +# Install Qt5 with automated install script, no msvc2019 version exists but 2017 is compatible +# Updater/silentUpdate options did not work for me. Install scripts finds and installs the most recent LTS version +COPY qt-installer.qs C:\TEMP\ + +# Installing Qt5 requires an account. This file contains a username and secret account token +RUN mkdir C:\Users\ContainerAdministrator\AppData\Roaming\Qt +COPY qtaccount\qtaccount.ini C:\Users\ContainerAdministrator\AppData\Roaming\Qt\qtaccount.ini + +RUN C:\TEMP\qt-unified-windows-x86-online.exe --script C:\TEMP\qt-installer.qs MsvcVersion=2019 ErrorLogname="%ERROR_FILENAME%" +RUN IF EXIST "%ERROR_FILENAME%" EXIT 1 + +RUN choco upgrade -y chocolatey +RUN choco upgrade -y all + +# The rest of the python packages are installed through run_ros2_batch.py +RUN python -m pip install -U pip setuptools pydot PyQt5 + +COPY rticonnextdds-license\rti_license.dat C:\connext\ +ENV RTI_LICENSE_FILE C:\connext\rti_license.dat + +COPY rticonnextdds-src\openssl-1.0.2n-target-x64Win64VS2017.zip C:\TEMP\connext\ +RUN 7z.exe x C:\TEMP\connext\openssl-1.0.2n-target-x64Win64VS2017.zip -aoa -oC:\connext\ +ENV RTI_OPENSSL_BIN C:\connext\openssl-1.0.2n\x64Win64VS2017\release\bin +ENV RTI_OPENSSL_LIB C:\connext\openssl-1.0.2n\x64Win64VS2017\release\lib + +COPY rticonnextdds-src\rti_connext_dds-5.3.1-pro-host-x64Win64.exe.??? C:\TEMP\connext\ +RUN copy /b C:\TEMP\connext\rti_connext_dds-5.3.1-pro-host-x64Win64.exe.??? C:\TEMP\connext\rti_connext_dds-5.3.1-pro-host-x64Win64.exe +RUN C:\TEMP\connext\rti_connext_dds-5.3.1-pro-host-x64Win64.exe ` + --mode unattended ` + --unattendedmodeui minimalWithDialogs ` + --prefix "%ProgramFiles%" + +COPY rticonnextdds-src\openssl-1.0.2n-5.3.1-host-x64Win64.rtipkg C:\TEMP\connext\ +RUN ""%ProgramFiles%\rti_connext_dds-5.3.1\bin\rtipkginstall.bat" C:\TEMP\connext\openssl-1.0.2n-5.3.1-host-x64Win64.rtipkg" +COPY rticonnextdds-src\rti_connext_dds-5.3.1-pro-target-x64Win64VS2017.rtipkg.??? C:\TEMP\connext\ +RUN copy /b C:\TEMP\connext\rti_connext_dds-5.3.1-pro-target-x64Win64VS2017.rtipkg.??? C:\TEMP\connext\rti_connext_dds-5.3.1-pro-target-x64Win64VS2017.rtipkg +RUN ""%ProgramFiles%\rti_connext_dds-5.3.1\bin\rtipkginstall.bat" C:\TEMP\connext\rti_connext_dds-5.3.1-pro-target-x64Win64VS2017.rtipkg" +COPY rticonnextdds-src\rti_security_plugins-5.3.1-host-x64Win64.rtipkg C:\TEMP\connext\ +RUN ""%ProgramFiles%\rti_connext_dds-5.3.1\bin\rtipkginstall.bat" C:\TEMP\connext\rti_security_plugins-5.3.1-host-x64Win64.rtipkg" +COPY rticonnextdds-src\rti_security_plugins-5.3.1-target-x64Win64VS2017.rtipkg C:\TEMP\connext\ +RUN ""%ProgramFiles%\rti_connext_dds-5.3.1\bin\rtipkginstall.bat" C:\TEMP\connext\rti_security_plugins-5.3.1-target-x64Win64VS2017.rtipkg" + +# Visual Studio Build Tools and .Net SDK` +ADD https://aka.ms/vs/16/release/vs_BuildTools.exe C:\TEMP\ + +# 3010 is an acceptable exit code (install was successful but restart required), but it will confuse docker. +# This installer invalidates docker image caches pretty regularly, so it's late in the order. See documentation for installer at: +# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019 +RUN C:\TEMP\vs_BuildTools.exe --quiet --wait --norestart --includeRecommended ` + --add Microsoft.Net.Component.4.8.SDK ` + --add Microsoft.VisualStudio.Workload.VCTools ` + --add Microsoft.Component.MSBuild ` + --add Microsoft.VisualStudio.Component.VC.CLI.Support ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0 + +CMD ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvarsall.bat", "x86_amd64", "&&", ` + "python", "run_ros2_batch.py", "%CI_ARGS%"] From 032862591c20537e583e1d79811475e205507284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 13 May 2020 09:32:03 -0400 Subject: [PATCH 2/3] Use Windows Dockerfile based on ROS distro. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than using separate Dockerfiles for separate Visual Studio versions, do so based on the CI_ROS_DISTRO parameter. Signed-off-by: Steven! Ragnarök --- job_templates/ci_job.xml.em | 5 +++-- job_templates/packaging_job.xml.em | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/job_templates/ci_job.xml.em b/job_templates/ci_job.xml.em index f54c84657..c75e9521c 100644 --- a/job_templates/ci_job.xml.em +++ b/job_templates/ci_job.xml.em @@ -103,6 +103,7 @@ colcon_branch: ${build.buildVariableResolver.resolve('CI_COLCON_BRANCH')},
use_whitespace: ${build.buildVariableResolver.resolve('CI_USE_WHITESPACE_IN_PATHS')},
isolated: ${build.buildVariableResolver.resolve('CI_ISOLATED')},
ubuntu_distro: ${build.buildVariableResolver.resolve('CI_UBUNTU_DISTRO')},
+ros_distro: ${build.buildVariableResolver.resolve('CI_ROS_DISTRO')},
colcon_mixin_url: ${build.buildVariableResolver.resolve('CI_COLCON_MIXIN_URL')},
cmake_build_type: ${build.buildVariableResolver.resolve('CI_CMAKE_BUILD_TYPE')},
build_args: ${build.buildVariableResolver.resolve('CI_BUILD_ARGS')},
@@ -357,8 +358,8 @@ setlocal enableDelayedExpansion rmdir /S /Q ws workspace "work space" echo "# BEGIN SECTION: Build DockerFile" -set CONTAINER_NAME=ros2_windows_ci_msvc%CI_VISUAL_STUDIO_VERSION% -set DOCKERFILE=windows_docker_resources\Dockerfile.msvc%CI_VISUAL_STUDIO_VERSION% +set CONTAINER_NAME=ros2_windows_ci_%CI_ROS_DISTRO% +set DOCKERFILE=windows_docker_resources\Dockerfile.%CI_ROS_DISTRO% rem "Change dockerfile once per day to invalidate docker caches" powershell "(Get-Content ${Env:DOCKERFILE}).replace('@@todays_date', $(Get-Date).ToLongDateString()) | Set-Content ${Env:DOCKERFILE}" diff --git a/job_templates/packaging_job.xml.em b/job_templates/packaging_job.xml.em index be3fd597a..eb3397efe 100644 --- a/job_templates/packaging_job.xml.em +++ b/job_templates/packaging_job.xml.em @@ -110,6 +110,7 @@ All packages listed here have to be available from either the primary or supplem @[if 'linux' in os_name]@ ubuntu_distro: ${build.buildVariableResolver.resolve('CI_UBUNTU_DISTRO')},
@[end if]@ +ros_distro: ${build.buildVariableResolver.resolve('CI_ROS_DISTRO')},
branch: ${build.buildVariableResolver.resolve('CI_BRANCH_TO_TEST')},
ci_branch: ${build.buildVariableResolver.resolve('CI_SCRIPTS_BRANCH')},
repos_url: ${build.buildVariableResolver.resolve('CI_ROS2_REPOS_URL')},
@@ -343,8 +344,8 @@ setlocal enableDelayedExpansion rmdir /S /Q ws workspace echo "# BEGIN SECTION: Build DockerFile" -set CONTAINER_NAME=ros2_windows_ci_msvc%CI_VISUAL_STUDIO_VERSION% -set DOCKERFILE=windows_docker_resources\Dockerfile.msvc%CI_VISUAL_STUDIO_VERSION% +set CONTAINER_NAME=ros2_windows_ci_%CI_ROS_DISTRO% +set DOCKERFILE=windows_docker_resources\Dockerfile.%CI_ROS_DISTRO%% rem "Change dockerfile once per day to invalidate docker caches" powershell "(Get-Content ${Env:DOCKERFILE}).replace('@@todays_date', $(Get-Date).ToLongDateString()) | Set-Content ${Env:DOCKERFILE}" From 8826aa1e0d2dc5cd7ac6dc5255f2db573dc618b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 13 May 2020 09:38:45 -0400 Subject: [PATCH 3/3] When building Eloquent on Windows use the Dashing dockerfile. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashing and Eloquent share supported Python and OpenSSL versions and so can share a Dockerfile. Signed-off-by: Steven! Ragnarök --- job_templates/ci_job.xml.em | 4 ++++ job_templates/packaging_job.xml.em | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/job_templates/ci_job.xml.em b/job_templates/ci_job.xml.em index c75e9521c..e60e459da 100644 --- a/job_templates/ci_job.xml.em +++ b/job_templates/ci_job.xml.em @@ -358,6 +358,10 @@ setlocal enableDelayedExpansion rmdir /S /Q ws workspace "work space" echo "# BEGIN SECTION: Build DockerFile" +@# Eloquent uses the Dashing Dockerfile. +if "!CI_ROS_DISTRO!" == "eloquent" ( + set "CI_ROS_DISTRO=dashing" +) set CONTAINER_NAME=ros2_windows_ci_%CI_ROS_DISTRO% set DOCKERFILE=windows_docker_resources\Dockerfile.%CI_ROS_DISTRO% diff --git a/job_templates/packaging_job.xml.em b/job_templates/packaging_job.xml.em index eb3397efe..fc2d4cdf5 100644 --- a/job_templates/packaging_job.xml.em +++ b/job_templates/packaging_job.xml.em @@ -344,6 +344,10 @@ setlocal enableDelayedExpansion rmdir /S /Q ws workspace echo "# BEGIN SECTION: Build DockerFile" +@# Eloquent uses the Dashing Dockerfile. +if "!CI_ROS_DISTRO!" == "eloquent" ( + set "CI_ROS_DISTRO=dashing" +) set CONTAINER_NAME=ros2_windows_ci_%CI_ROS_DISTRO% set DOCKERFILE=windows_docker_resources\Dockerfile.%CI_ROS_DISTRO%%