From 511c2dad66f8aa8fc54806ac4b4eae68805a1e15 Mon Sep 17 00:00:00 2001
From: Felix Weiglhofer <weiglhofer@fias.uni-frankfurt.de>
Date: Mon, 6 Nov 2023 13:27:07 +0000
Subject: [PATCH] Move base containers into different repository.

---
 .../cbm_online => .ci/online}/Dockerfile      |  9 +++---
 .../cbm_online => .ci/online}/DockerfileBuild |  4 +--
 .../online}/make_build_image.sh               |  2 +-
 .../online}/scripts/install.sh                |  2 +-
 .../cbm_online => .ci/online}/test_run.sh     |  0
 .../cbm_online => .ci/online}/test_run_gpu.sh |  0
 .gitlab-ci.yml                                |  8 ++---
 algo/containers/online_dev/Dockerfile         | 10 ------
 algo/containers/online_dev/build_and_push.sh  | 11 -------
 .../online_dev/scripts/install_fairsoft.sh    | 32 -------------------
 .../online_dev/scripts/install_packages.sh    | 29 -----------------
 .../online_dev/scripts/install_rocm.sh        | 22 -------------
 algo/containers/online_runtime/Dockerfile     | 12 -------
 .../online_runtime/build_and_push.sh          | 11 -------
 .../scripts/install_hip_runtime.sh            | 25 ---------------
 .../scripts/install_packages.sh               | 20 ------------
 docs/online/container/HowTo.md                | 26 +++++++++------
 17 files changed, 30 insertions(+), 193 deletions(-)
 rename {algo/containers/cbm_online => .ci/online}/Dockerfile (84%)
 rename {algo/containers/cbm_online => .ci/online}/DockerfileBuild (86%)
 rename {algo/containers/cbm_online => .ci/online}/make_build_image.sh (96%)
 rename {algo/containers/cbm_online => .ci/online}/scripts/install.sh (97%)
 rename {algo/containers/cbm_online => .ci/online}/test_run.sh (100%)
 rename {algo/containers/cbm_online => .ci/online}/test_run_gpu.sh (100%)
 delete mode 100644 algo/containers/online_dev/Dockerfile
 delete mode 100755 algo/containers/online_dev/build_and_push.sh
 delete mode 100755 algo/containers/online_dev/scripts/install_fairsoft.sh
 delete mode 100755 algo/containers/online_dev/scripts/install_packages.sh
 delete mode 100755 algo/containers/online_dev/scripts/install_rocm.sh
 delete mode 100644 algo/containers/online_runtime/Dockerfile
 delete mode 100755 algo/containers/online_runtime/build_and_push.sh
 delete mode 100755 algo/containers/online_runtime/scripts/install_hip_runtime.sh
 delete mode 100755 algo/containers/online_runtime/scripts/install_packages.sh

diff --git a/algo/containers/cbm_online/Dockerfile b/.ci/online/Dockerfile
similarity index 84%
rename from algo/containers/cbm_online/Dockerfile
rename to .ci/online/Dockerfile
index 5085fa041d..420b660229 100644
--- a/algo/containers/cbm_online/Dockerfile
+++ b/.ci/online/Dockerfile
@@ -1,13 +1,14 @@
-ARG REGISTRY=hub.cbm.gsi.de/computing/cbmroot
+ARG TAG=12-fair202211.1-rocm5.4
+ARG REGISTRY=hub.cbm.gsi.de/computing/images/online
 
-FROM ${REGISTRY}/online_dev:latest as build
+FROM ${REGISTRY}/dev:${TAG} as build
   ARG SOURCE_DIR=/opt/cbm/src
   WORKDIR ${SOURCE_DIR}
   # Kaniko doesn't support RUN --mount, so we have to copy the source code instead
   COPY . .
-  RUN algo/containers/cbm_online/scripts/install.sh
+  RUN .ci/online/scripts/install.sh
 
-FROM ${REGISTRY}/online_runtime:latest
+FROM ${REGISTRY}/runtime:${TAG}
   ARG CBMROOT_PATH=/opt/cbm/cbmroot
   ARG FAIRSOFT_PATH=/opt/cbm/fairsoft
   COPY --from=build ${CBMROOT_PATH}/bin ${CBMROOT_PATH}/bin
diff --git a/algo/containers/cbm_online/DockerfileBuild b/.ci/online/DockerfileBuild
similarity index 86%
rename from algo/containers/cbm_online/DockerfileBuild
rename to .ci/online/DockerfileBuild
index 2efec38e0f..373353f733 100644
--- a/algo/containers/cbm_online/DockerfileBuild
+++ b/.ci/online/DockerfileBuild
@@ -1,11 +1,11 @@
 FROM gcr.io/kaniko-project/executor:v1.16.0-debug
-    ARG USER="username"
+    ARG USERNAME="username"
     ARG PASSWORD="your_password"
     ARG REGISTRY="hub.cbm.gsi.de"
     ARG IMAGE="${REGISTRY}/computing/cbmroot/cbm_online"
     ARG TAG="${USER}-debug"
     RUN mkdir -p /kaniko/.docker
-    RUN echo "{\"auths\":{\"${REGISTRY}\":{\"username\":\"${USER}\",\"password\":\"${PASSWORD}\"}}}" > /kaniko/.docker/config.json
+    RUN echo "{\"auths\":{\"${REGISTRY}\":{\"username\":\"${USERNAME}\",\"password\":\"${PASSWORD}\"}}}" > /kaniko/.docker/config.json
     # TODO: Mount CbmRoot source instead of copying it into the container (-> same with authentification...)
     COPY . /workspace
     ENV IMAGE=${IMAGE}
diff --git a/algo/containers/cbm_online/make_build_image.sh b/.ci/online/make_build_image.sh
similarity index 96%
rename from algo/containers/cbm_online/make_build_image.sh
rename to .ci/online/make_build_image.sh
index 0ace824b02..063b0a8107 100755
--- a/algo/containers/cbm_online/make_build_image.sh
+++ b/.ci/online/make_build_image.sh
@@ -36,7 +36,7 @@ registry="hub.cbm.gsi.de/computing/cbmroot"
 image="cbm_online"
 
 build_args=" \
-    --build-arg="USER=$user" \
+    --build-arg="USERNAME=$user" \
     --build-arg="PASSWORD=$password" \
     --build-arg="TAG=$tag" \
 "
diff --git a/algo/containers/cbm_online/scripts/install.sh b/.ci/online/scripts/install.sh
similarity index 97%
rename from algo/containers/cbm_online/scripts/install.sh
rename to .ci/online/scripts/install.sh
index 549dc1a9e9..67bb9096e9 100755
--- a/algo/containers/cbm_online/scripts/install.sh
+++ b/.ci/online/scripts/install.sh
@@ -6,7 +6,7 @@ srcdir=cbmroot
 build_type=RelWithDebInfo
 install_prefix=/opt/cbm/cbmroot
 fairsoft_path=/opt/cbm/fairsoft
-enable_hip=OFF
+enable_hip=ON
 rocm_root=/opt/rocm
 hip_archs="gfx906;gfx908" # MI50;MI100
 n_jobs=16
diff --git a/algo/containers/cbm_online/test_run.sh b/.ci/online/test_run.sh
similarity index 100%
rename from algo/containers/cbm_online/test_run.sh
rename to .ci/online/test_run.sh
diff --git a/algo/containers/cbm_online/test_run_gpu.sh b/.ci/online/test_run_gpu.sh
similarity index 100%
rename from algo/containers/cbm_online/test_run_gpu.sh
rename to .ci/online/test_run_gpu.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index daba192922..0ce6acde46 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -536,12 +536,12 @@ BuildOnlineContainerMR:
   tags:
     - docker
   image:
-    name: gcr.io/kaniko-project/executor:v1.16.0-debug
+    name: $CI_REGISTRY/computing/images/base/kaniko/executor:v1.16.0-debug
     entrypoint: [""]
   variables:
     IMAGE_NAME: $CI_REGISTRY_IMAGE/cbm_online
     IMAGE_TAG: mr$CI_MERGE_REQUEST_IID
-    DOCKERFILE: $CI_PROJECT_DIR/algo/containers/cbm_online/Dockerfile
+    DOCKERFILE: $CI_PROJECT_DIR/.ci/online/Dockerfile
   script:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
@@ -575,12 +575,12 @@ BuildOnlineContainerBranch:
   tags:
     - docker
   image:
-    name: gcr.io/kaniko-project/executor:v1.16.0-debug
+    name: $CI_REGISTRY/computing/images/base/kaniko/executor:v1.16.0-debug
     entrypoint: [""]
   variables:
     IMAGE_NAME: $CI_REGISTRY_IMAGE/cbm_online
     IMAGE_TAG: $CI_COMMIT_BRANCH
-    DOCKERFILE: $CI_PROJECT_DIR/algo/containers/cbm_online/Dockerfile
+    DOCKERFILE: $CI_PROJECT_DIR/.ci/online/Dockerfile
   script:
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
diff --git a/algo/containers/online_dev/Dockerfile b/algo/containers/online_dev/Dockerfile
deleted file mode 100644
index 298903937f..0000000000
--- a/algo/containers/online_dev/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-FROM ubuntu:22.04 as rocm
-  RUN --mount=source=scripts,target=s s/install_rocm.sh
-
-FROM debian:12-slim
-  RUN --mount=source=scripts,target=s s/install_packages.sh
-
-  ARG FAIRSOFT_VERSION=nov22p1
-  RUN --mount=source=scripts,target=s s/install_fairsoft.sh ${FAIRSOFT_VERSION}
-
-  COPY --from=rocm /opt/rocm /opt/rocm
diff --git a/algo/containers/online_dev/build_and_push.sh b/algo/containers/online_dev/build_and_push.sh
deleted file mode 100755
index 28317a7d6b..0000000000
--- a/algo/containers/online_dev/build_and_push.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-#TODO: make ROCm & fairsoft version a parameter
-
-set -e
-
-registry="hub.cbm.gsi.de/computing/cbmroot"
-image="online_dev"
-tag="latest"
-
-docker build -t $registry/$image:$tag --push .
diff --git a/algo/containers/online_dev/scripts/install_fairsoft.sh b/algo/containers/online_dev/scripts/install_fairsoft.sh
deleted file mode 100755
index 1ee11d8a5c..0000000000
--- a/algo/containers/online_dev/scripts/install_fairsoft.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-set -e
-
-version=$1
-
-if [ -z "$version" ]; then
-    echo "Error: No version specified"
-    exit 1
-fi
-
-upstream="https://github.com/FairRootGroup/FairSoft.git"
-cxx_std="17"
-build_type="RelWithDebInfo"
-packages="boost root fmt"
-install_dir="/opt/cbm/fairsoft"
-
-git clone -b $version $upstream FairSoft
-cd FairSoft
-
-cmake_defs=" \
-    -DCMAKE_CXX_STANDARD=$cxx_std \
-    -DCMAKE_BUILD_TYPE=$build_type \
-    -DCMAKE_INSTALL_PREFIX=$install_dir \
-"
-
-cmake -S . -B build $cmake_defs
-cmake --build build -j$(nproc) --target $packages
-
-# Cleanup
-cd ..
-rm -rf FairSoft
diff --git a/algo/containers/online_dev/scripts/install_packages.sh b/algo/containers/online_dev/scripts/install_packages.sh
deleted file mode 100755
index 734ab7ba2b..0000000000
--- a/algo/containers/online_dev/scripts/install_packages.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#! /bin/bash
-
-set -e
-
-# System dependencies required by FairSoft
-FAIRSOFT_DEPS="autoconf automake binutils \
-	bison build-essential bzip2 ca-certificates coreutils \
-	curl debianutils file findutils flex g++ gcc gfortran git gzip \
-	hostname libbz2-dev libcurl4-openssl-dev libgsl-dev libicu-dev \
-	libfftw3-dev \
-	libgl1-mesa-dev libglu1-mesa-dev libgrpc++-dev \
-	libncurses-dev libreadline-dev libsqlite3-dev libssl-dev libtool \
-	libx11-dev libxerces-c-dev libxext-dev libxft-dev \
-	libxml2-dev libxmu-dev libxpm-dev libyaml-cpp-dev lsb-release make patch cmake \
-	python3-dev protobuf-compiler-grpc rsync sed subversion tar unzip wget xutils-dev xz-utils"
-
-# Additional system dependencies required by CBMROOT
-CBMROOT_DEPS="libtbb-dev sqlite3 libsqlite3-dev libgsl-dev"
-
-# Additional packages (mostly for debugging)
-ADDITIONAL_DEPS="gdb-minimal valgrind vim-tiny"
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get update
-apt-get -y upgrade
-apt-get -y install --no-install-recommends $FAIRSOFT_DEPS $CBMROOT_DEPS $ADDITIONAL_DEPS
-
-apt-get -y clean
-rm -rf /var/lib/apt/lists/*
diff --git a/algo/containers/online_dev/scripts/install_rocm.sh b/algo/containers/online_dev/scripts/install_rocm.sh
deleted file mode 100755
index 7e0aa8afdc..0000000000
--- a/algo/containers/online_dev/scripts/install_rocm.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-set -e
-
-ROCM_VERSION="5.4"
-ROCM_PACKAGES="rocm-hip-runtime-dev5.4 hipcub-dev5.4" # Need to add version suffix, why??
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-apt-get -y upgrade
-
-apt-get -y install --no-install-recommends ca-certificates curl libnuma-dev gnupg
-curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
-printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list
-# printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list
-
-apt-get update
-apt-get -y install --no-install-recommends $ROCM_PACKAGES
-
-apt-get -y clean
-rm -rf /var/lib/apt/lists/*
diff --git a/algo/containers/online_runtime/Dockerfile b/algo/containers/online_runtime/Dockerfile
deleted file mode 100644
index d7f24999d6..0000000000
--- a/algo/containers/online_runtime/Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-ARG RUNTIME_BASE_IMAGE=debian:12-slim
-ARG DEV_IMAGE=hub.cbm.gsi.de/computing/cbmroot/online_dev:latest
-
-FROM ubuntu:22.04 as rocm_runtime
-  RUN --mount=source=scripts,target=s s/install_hip_runtime.sh
-
-FROM ${DEV_IMAGE} as online_dev
-FROM ${RUNTIME_BASE_IMAGE} as online_runtime
-  RUN --mount=source=scripts,target=s s/install_packages.sh
-
-  COPY --from=rocm_runtime /opt/rocm /opt/rocm
-  COPY --from=online_dev /opt/cbm/fairsoft/lib/libboost* /opt/cbm/fairsoft/lib/
diff --git a/algo/containers/online_runtime/build_and_push.sh b/algo/containers/online_runtime/build_and_push.sh
deleted file mode 100755
index 7c05888e00..0000000000
--- a/algo/containers/online_runtime/build_and_push.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-#TODO: make rocm version a parameter
-
-set -e
-
-registry="hub.cbm.gsi.de/computing/cbmroot"
-image="online_runtime"
-tag="latest"
-
-docker build -f Dockerfile -t $registry/$image:$tag --push .
diff --git a/algo/containers/online_runtime/scripts/install_hip_runtime.sh b/algo/containers/online_runtime/scripts/install_hip_runtime.sh
deleted file mode 100755
index 6887db1ad1..0000000000
--- a/algo/containers/online_runtime/scripts/install_hip_runtime.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -e
-
-ROCM_VERSION="5.4"
-ROCM_PACKAGES="rocm-hip-runtime5.4" # Need to add version suffix, why??
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-apt-get -y upgrade
-
-apt-get -y install --no-install-recommends ca-certificates curl libnuma-dev gnupg
-curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
-printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list
-# printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list
-
-apt-get update
-apt-get -y install --no-install-recommends $ROCM_PACKAGES
-
-apt-get -y clean
-rm -rf /var/lib/apt/lists/*
-
-# Remove LLVM to save space
-rm -rf /opt/rocm/llvm
diff --git a/algo/containers/online_runtime/scripts/install_packages.sh b/algo/containers/online_runtime/scripts/install_packages.sh
deleted file mode 100755
index 2b6284ed62..0000000000
--- a/algo/containers/online_runtime/scripts/install_packages.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#! /bin/bash
-
-set -e
-
-# Runtime dependencies of CbmRoot (online only)
-CBMROOT_DEPS="libstdc++6 libgomp1 libc6 libgcc-s1 libtbb12"
-
-# Runtime dependencies of ROCm
-ROCM_DEPS="libnuma1 libdrm2 libdrm-amdgpu1"
-
-# Additional packages (mostly for debugging)
-ADDITIONAL_DEPS="gdb-minimal valgrind vim-tiny"
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get update
-apt-get -y upgrade
-apt-get -y install --no-install-recommends $CBMROOT_DEPS $ROCM_DEPS $ADDITIONAL_DEPS
-
-apt-get -y clean
-rm -rf /var/lib/apt/lists/*
diff --git a/docs/online/container/HowTo.md b/docs/online/container/HowTo.md
index c4089c65bd..ffce7a8ec2 100644
--- a/docs/online/container/HowTo.md
+++ b/docs/online/container/HowTo.md
@@ -4,6 +4,11 @@ This document provides an overview over the containers currently used for CBM on
 
 ## Build container
 
+### Base images
+
+Base images are hosted at [computing/images/base](https://git.cbm.gsi.de/computing/images/base). These are mirrored from places like docker-hub and google container registry to keep building images independent of external sources as well as possible. Included are OS images like Debian and Ubuntu, and tools like Kaniko and Crane.
+Mirrored images are updated once per week.
+
 ### Login
 
 Login via: `docker login hub.cbm.gsi.de`
@@ -11,25 +16,28 @@ Login via: `docker login hub.cbm.gsi.de`
 ### Build Images
 
 Currently 3 images are build, each depending on the previous one(s):
-1. `online_dev`: Contains all files to compile the CbmRoot online-code. This includes:
+1. `online/dev`: Contains all files to compile the CbmRoot online-code. This includes:
     - Subset of FairSoft packages (boost, root, fmt)
     - ROCm GPU compiler + hip development packages
     - Various system libraries (+ header)
-2. `online_runtime`: Minimal container that only contains libraries required to run
+2. `online/runtime`: Minimal container that only contains libraries required to run
   online reconstruction
 3. `cbm_online`: Runnable container. Consists of `online_runtime` + cbm online binaries
 
-Each image is accompanied by a `build_and_push.sh` script, that builds the image
-and pushes it to the container registry.
+The first two are hosted outside of CbmRoot in https://git.cbm.gsi.de/computing/images/online. And are rebuilt once per week, to keep system packages up-to-date.
 
-To rebuild and publish the base images:
+Building new images locally should also be done via Kaniko. To ensure reproducibility with the CI:
 ```
-cd online_dev
-./build_and_push.sh
-cd ../online_runtime
-./build_and_push.sh
+mv secrets.json.in secrets.json
+# Enter your gitlab username and password in secrets.json
+docker build -t cbm-builder -f Dockerfile.build . # Create the Kaniko build container
+docker run -it --rm -v $PWD:/workspace cbm-builder # Enter the build container
+# Run kaniko with the respective Dockerfile to build the dev or runtime image.
 ```
 
+**Your password is stored in plain text in the container. Make sure to delete it after you're done.**
+
+
 Building the online code and creating a new image, happens inside a container via kaniko.
 To build new cbm_online-images locally, you first have to create a new build-image:
 ```
-- 
GitLab