Commit bc81d801 authored by Evgeny Lavrik's avatar Evgeny Lavrik
Browse files

cbmdock

parent 61827052
# Kaniko is the recommended method for running Docker-builds as of May 2019.
# See https://docs.gitlab.com/ee/ci/docker/using_kaniko.html for GitLab's documentation.
stages:
- fairroot
- cbmroot
- cbmroot-hub
fairroot:
stage: fairroot
image:
# We recommend using the CERN version of the Kaniko image: gitlab-registry.cern.ch/ci-tools/docker-image-builder
name: gitlab-registry.cern.ch/ci-tools/docker-image-builder
entrypoint: [""]
script:
# Prepare Kaniko configuration file
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
# Build and push the image from the Dockerfile at the root of the project.
# To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
# See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
- /kaniko/executor --cache=true --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/FairRoot.Dockerfile --destination $CI_REGISTRY_IMAGE/$CI_JOB_NAME:latest
retry: 2
only:
changes:
- FairRoot.Dockerfile
cbmroot:
stage: cbmroot
needs:
- job: fairroot
artifacts: false
image:
# We recommend using the CERN version of the Kaniko image: gitlab-registry.cern.ch/ci-tools/docker-image-builder
name: gitlab-registry.cern.ch/ci-tools/docker-image-builder
entrypoint: [""]
script:
# Prepare Kaniko configuration file
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
# Build and push the image from the Dockerfile at the root of the project.
# To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
# See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
- /kaniko/executor --cache=true --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/CbmRoot.Dockerfile --destination $CI_REGISTRY_IMAGE/$CI_JOB_NAME:latest
retry: 2
only:
changes:
- CbmRoot.Dockerfile
cbmroot-hub:
stage: cbmroot-hub
# needs:
# - job: cbmroot
# artifacts: false
image:
# We recommend using the CERN version of the Kaniko image: gitlab-registry.cern.ch/ci-tools/docker-image-builder
name: gitlab-registry.cern.ch/ci-tools/docker-image-builder
entrypoint: [""]
script:
# Prepare Kaniko configuration file
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
# Build and push the image from the Dockerfile at the root of the project.
# To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
# See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
- /kaniko/executor --cache=true --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Jupyterhub.Dockerfile --destination $CI_REGISTRY_IMAGE:latest
retry: 2
only:
changes:
- Jupyterhub.Dockerfile
\ No newline at end of file
ARG BASE_IMAGE=gitlab-registry.cern.ch/elavrik/cbmroot-hub/fairroot:latest
FROM $BASE_IMAGE
#
# CbmRoot
#
WORKDIR /tmp/cbmroot-build
RUN git clone https://git.cbm.gsi.de/CbmSoft/CbmRoot.git CbmRoot && \
cd CbmRoot && git checkout f66ec1b244a2dd33468d8cd0f6a30bf7016c3685 && cd .. && \
mkdir -vp CbmRoot_build && cmake -S CbmRoot -B CbmRoot_build -DCMAKE_INSTALL_PREFIX=/opt/cbmroot && \
make -C CbmRoot_build -j2 install && \
cd / && rm -rf /tmp/cbmroot*
ENV \
ROOTSYS="/opt/fairsoft_jun19p2" \
ROOT_LIBRARY_DIR="/opt/fairsoft_jun19p2/lib/root" \
ROOT_LIBRARIES="-L/opt/fairsoft_jun19p2/lib/root -lGui -lCore -lImt -lRIO -lNet -lHist -lGraf -lGraf3d -lGpad -lROOTVecOps -lTree -lTreePlayer -lRint -lPostscript -lMatrix -lPhysics -lMathCore -lThread -lMultiProc -lROOTDataFrame -pthread -Wl,-rpath,/opt/fairsoft_jun19p2/lib/root -lm -ldl -rdynamic" \
ROOT_INCLUDE_DIR="/opt/fairsoft_jun19p2/include/root6" \
ROOT_INCLUDE_PATH="/opt/cbmroot/include:/opt/fairroot_v18.2.1/include:/opt/fairsoft_jun19p2/include/TGeant3:/opt/cbmroot/include" \
VMCWORKDIR="/opt/cbmroot/share/cbmroot" \
FAIRLIBDIR="/opt/cbmroot/lib" \
PYTHONPATH="/opt/cbmroot/lib:/opt/fairsoft_jun19p2/lib:/opt/fairsoft_jun19p2/lib/root:/opt/fairsoft_jun19p2/lib/Geant4:/opt/fairsoft_jun19p2/lib/g4py" \
LD_LIBRARY_PATH="/opt/cbmroot/lib:/opt/fairroot_v18.2.1/lib:/opt/fairsoft_jun19p2/lib:/opt/fairsoft_jun19p2/lib/root:$LD_LIBRARY_PATH" \
PATH="/opt/cbmroot/bin:/opt/fairroot_v18.2.1/bin:/opt/fairsoft_jun19p2/bin:/opt/cmake-3.16.4/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"
WORKDIR /opt/cbmroot
# Jupyterhub interactive environment with CbmRoot
### Getting docker
Docker is a software suite to run software in a platform independent way akin to virtual machines.
Learn how to download and install it on your system (admin rights required): https://www.docker.com/get-started
### CbmRoot docker environment
There are following containers in this package build upon each other in the following order:
* fairsoft
* fairroot
* cbmroot
* jupyterlab
To run any of these containers, you need to initialize them first, e.g. run the following commands:
```bash
docker-compose up -d cbmroot
```
The pre-build container images will then be downloaded. The containers will run and keep any files you create inside it until you won't remove them from your system with:
```bash
docker-compose down --rmi=all -v
```
If you want more persistency, keep your files in the `code` directory and they will never be removed by docker.
The `code` directory is mounted in all containers in the `/opt/code` directory and also in `/home/cbmdock/code` in cbmroot-dev container.
The software is installed in `/opt` within containers, e.g.:
* `/opt/fairsoft`
* `/opt/fairroot`
* `/opt/cbmroot`
To enter the container environment, run
```bash
docker-compose exec cbmroot bash
```
### Running interactive ROOT analysis in JupyterLab
Run the following command in your terminal
## Set up
Pull image from registry:
* CERN:
```bash
docker login -u ${USER} gitlab-registry.cern.ch
docker pull gitlab-registry.cern.ch/elavrik/cbmroot-hub
docker-compose up -d jupyterhub
```
* Docker hub:
You can navigate to http://localhost:8000 in your browser.
Authorize yourself with user `user` and password `test`.
### CbmRoot development environment
If you are a CbmRoot power-user or a developer you might want to check out the `cbmroot-dev` container, also known as `cbmdock`.
When creating the container for the first time, run:
```bash
docker pull elavrik/cbmroot-hub
export UID
docker-compose up cbmroot-dev
```
## Run
and wait until the development version of CbmRoot will be compiled.
You can change `GIT_REPO` and `GIT_TAG` variables in the `docker-compose.yml` to compile the code from destination and git tag (or commit) of your choise, respectively.
The files will be shared with the container and your computer (located in `code/cbmroot`), so you can edit them conveniently with your favorite code editor.
With docker cli
`docker run -it -p 8000:8000 elavrik/cbmroot-hub`
You either can enter the container as usual or ssh into it by issuing the command:
```bash
ssh -Y cbmdock@localhost -p 2222
```
With docker-compose
`docker-compose up -d cbmroot-hub`
### Troubleshooting
Navigate with your browser to `https://localhost:8000`
use default login `user` and password `test` to start the session.
If you have no visualization you need to install the x11 on your host machine.
In macOS it is XQuartz. Run it, go to preferences -> security and enable connections from network clients.
ARG BASE_IMAGE=elavrik/cbmroot:latest
FROM $BASE_IMAGE
ARG GIT_REPO=https://git.cbm.gsi.de/computing/cbmroot.git
ARG GIT_TAG=6bd453ba
ARG USER_ID=1000
#
# CbmRoot Development version
#
RUN apt-get install -y sudo xauth openssh-server && \
sed -i "s/^.*PasswordAuthentication.*$/PasswordAuthentication no/" /etc/ssh/sshd_config && \
sed -i "s/^.*X11Forwarding.*$/X11Forwarding yes/" /etc/ssh/sshd_config && \
sed -i "s/^.*X11UseLocalhost.*$/X11UseLocalhost no/" /etc/ssh/sshd_config && \
mkdir -p /var/run/sshd && mkdir -p /run/sshd && \
useradd cbmdock -m -u ${USER_ID} -U -p '*' -s /bin/bash && \
mkdir /home/cbmdock/.ssh && chmod 700 /home/cbmdock/.ssh && \
echo '. /opt/code/cbmroot/build/config.sh' >> /home/cbmdock/.bashrc && \
chown cbmdock:cbmdock -R /home/cbmdock && \
echo 'cbmdock ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/cbmdock && \
chmod 0440 /etc/sudoers.d/cbmdock
WORKDIR /tmp
ADD entrypoint.sh entrypoint.sh
ENV \
ROOTSYS="/opt/fairsoft" \
ROOT_LIBRARY_DIR="/opt/fairsoft/lib/root" \
ROOT_LIBRARIES="-L/opt/fairsoft/lib/root -lGui -lCore -lImt -lRIO -lNet -lHist -lGraf -lGraf3d -lGpad -lROOTVecOps -lTree -lTreePlayer -lRint -lPostscript -lMatrix -lPhysics -lMathCore -lThread -lMultiProc -lROOTDataFrame -pthread -Wl,-rpath,/opt/fairsoft/lib/root -lm -ldl -rdynamic" \
ROOT_INCLUDE_DIR="/opt/fairsoft/include/root6" \
ROOT_INCLUDE_PATH="/opt/code/cbmroot/include:/opt/fairroot/include:/opt/fairsoft/include/TGeant3" \
VMCWORKDIR="/opt/code/cbmroot" \
FAIRLIBDIR="/opt/code/cbmroot/build/lib" \
PYTHONPATH="/opt/code/cbmroot/build/lib:/opt/fairsoft/lib:/opt/fairsoft/lib/root:/opt/fairsoft/lib/Geant4:/opt/fairsoft/lib/g4py" \
LD_LIBRARY_PATH="/opt/code/cbmroot/build/lib:/opt/fairroot/lib:/opt/fairsoft/lib:/opt/fairsoft/lib/root:$LD_LIBRARY_PATH" \
PATH="/opt/code/cbmroot/bin:/opt/fairroot/bin:/opt/fairsoft/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" \
GIT_REPO=${GIT_REPO} \
GIT_TAG=${GIT_TAG}
WORKDIR /opt/code/cbmroot
#!/bin/bash
sudo /usr/sbin/sshd
cd /opt/code
git clone ${GIT_REPO} cbmroot ; \
cd cbmroot && git checkout ${GIT_TAG} && mkdir -vp build && cd build && \
cmake .. -DCMAKE_INSTALL_PREFIX=/opt/cbmroot && \
make -j`nproc`
tail -f /dev/null
\ No newline at end of file
ARG BASE_IMAGE=elavrik/fairroot:latest
FROM $BASE_IMAGE
ARG GIT_TAG=6ef969bc
#
# CbmRoot
#
WORKDIR /tmp/cbmroot-build
RUN git clone https://git.cbm.gsi.de/computing/cbmroot.git CbmRoot && \
cd CbmRoot && git checkout ${GIT_TAG} && cd .. && \
mkdir -vp CbmRoot_build && cmake -S CbmRoot -B CbmRoot_build -DCMAKE_INSTALL_PREFIX=/opt/cbmroot && \
make -C CbmRoot_build -j`nproc` install && \
cd / && rm -rf /tmp/cbmroot*
ENV \
ROOTSYS="/opt/fairsoft" \
ROOT_LIBRARY_DIR="/opt/fairsoft/lib/root" \
ROOT_LIBRARIES="-L/opt/fairsoft/lib/root -lGui -lCore -lImt -lRIO -lNet -lHist -lGraf -lGraf3d -lGpad -lROOTVecOps -lTree -lTreePlayer -lRint -lPostscript -lMatrix -lPhysics -lMathCore -lThread -lMultiProc -lROOTDataFrame -pthread -Wl,-rpath,/opt/fairsoft/lib/root -lm -ldl -rdynamic" \
ROOT_INCLUDE_DIR="/opt/fairsoft/include/root6" \
ROOT_INCLUDE_PATH="/opt/cbmroot/include:/opt/fairroot/include:/opt/fairsoft/include/TGeant3:/opt/cbmroot/include" \
VMCWORKDIR="/opt/cbmroot/share/cbmroot" \
FAIRLIBDIR="/opt/cbmroot/lib" \
PYTHONPATH="/opt/cbmroot/lib:/opt/fairsoft/lib:/opt/fairsoft/lib/root:/opt/fairsoft/lib/Geant4:/opt/fairsoft/lib/g4py" \
LD_LIBRARY_PATH="/opt/cbmroot/lib:/opt/fairroot/lib:/opt/fairsoft/lib:/opt/fairsoft/lib/root:$LD_LIBRARY_PATH" \
PATH="/opt/cbmroot/bin:/opt/fairroot/bin:/opt/fairsoft/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"
WORKDIR /opt/cbmroot
version: '3.2'
version: '3.8'
x-function: &dev-suite
volumes:
- /tmp/.X11-unix:/tmp/.X11-unix:ro
- ${HOME}:/tmp/hostHome:ro
tty: true
environment:
# - DISPLAY
- DISPLAY=host.docker.internal:0
- XAUTHORITY=/tmp/hostHome/.Xauthority
ports:
- "2222:22"
cap_add:
- ALL
volumes:
workspace:
driver: local
driver_opts:
type: none
device: ${PWD}/code
o: bind
services:
### fairsoft ##############################################
fairsoft:
image: elavrik/fairsoft:jun19p2
build:
context: ./fairsoft
args:
GIT_TAG: jun19p2
volumes:
- workspace:/opt/code
tty: true
### fairroot ##############################################
fairroot:
image: elavrik/fairroot:latest
build:
context: ./fairroot
args:
- GIT_TAG=v18.4.0
volumes:
- workspace:/opt/code
tty: true
### Workspace Utilities ##################################
cbmroot-hub:
### cbmroot ##############################################
cbmroot:
image: elavrik/cbmroot:latest
build:
context: ./
image: elavrik/cbmroot-hub
context: ./cbmroot
args:
- GIT_TAG=6ef969bc
volumes:
- type: bind
source: ${PWD}/persistent
target: /home/user/persistent
- workspace:/opt/code
tty: true
command: /bin/bash
### cbmroot-dev ##############################################
cbmroot-dev:
<<: *dev-suite
build:
context: ./cbmroot-dev
args:
- GIT_REPO=https://git.cbm.gsi.de/computing/cbmroot.git
- GIT_TAG=6ef969bc
- USER_ID=${UID:?run "export UID" first}
volumes:
- workspace:/opt/code
- workspace:/home/cbmdock/code
- ~/.ssh/id_rsa.pub:/home/cbmdock/.ssh/authorized_keys
- /tmp/.X11-unix:/tmp/.X11-unix:ro
- ${HOME}:/tmp/hostHome:ro
hostname: cbmdock
tty: true
working_dir: /home/cbmdock
user: cbmdock
command: sh -c "/tmp/entrypoint.sh"
### jupyterhub ##############################################
jupyterhub:
image: elavrik/cbmroot-jupyterhub:latest
build:
context: ./jupyterhub
volumes:
- workspace:/opt/code
- ./jupyterhub/notebooks:/home/user/notebooks
ports:
- "22:22"
- "8000:8000"
tty: true
environment:
- TEST=TEST
ARG BASE_IMAGE=elavrik/fairsoft:latest
FROM $BASE_IMAGE
ARG GIT_TAG=v18.4.0
#
# FairRoot
#
WORKDIR /tmp/fairroot-build
RUN git clone https://github.com/FairRootGroup/FairRoot.git -b ${GIT_TAG} FairRoot && \
mkdir -vp FairRoot_build && cmake -S FairRoot -B FairRoot_build -DCMAKE_INSTALL_PREFIX=/opt/fairroot && \
make -C FairRoot_build -j`nproc` install && \
rm -r FairRoot_build && rm -rf /tmp/fair*
ENV FAIRROOTPATH=/opt/fairroot
compiler=gcc
debug=no
optimize=no
geant4_download_install_data_automatic=yes
geant4_install_data_from_dir=no
geant4mt=no
build_root6=yes
build_python=yes
install_sim=yes
build_MQOnly=no
SIMPATH_INSTALL=/opt/fairsoft_jun19p2
NCPU=2
ARG BASE_IMAGE=ubuntu:18.04
FROM $BASE_IMAGE
ARG GIT_TAG=jun19p2
#
# Install dependencies
#
......@@ -18,45 +21,27 @@ RUN apt-get update && \
libsqlite3-0 libsqlite3-dev && \
apt-get clean
#
# Steal newer CMake
#
WORKDIR /opt
RUN wget https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-Linux-x86_64.sh -O cmake.sh && chmod +x cmake.sh && \
mkdir /opt/cmake-3.16.4 && ./cmake.sh --skip-license --prefix=/opt/cmake-3.16.4 && \
rm cmake.sh
ENV PATH="/opt/cmake-3.16.4/bin:${PATH}"
#
# FairSoft
#
WORKDIR /tmp/fairsoft-build
ADD ./fairsoft.conf ./fairsoft.conf
RUN git clone https://github.com/FairRootGroup/FairSoft.git -b jun19p2 FairSoft && cd FairSoft && \
RUN git clone https://github.com/FairRootGroup/FairSoft.git -b ${GIT_TAG} FairSoft && cd FairSoft && \
./configure.sh ../fairsoft.conf && ./make_clean.sh all && rm -rf /tmp/fair*
ENV SIMPATH="/opt/fairsoft_jun19p2"\
G4INCLDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4INCL1.0"\
G4LEVELGAMMADATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/PhotonEvaporation5.3"\
G4RADIOACTIVEDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/RadioactiveDecay5.3"\
G4PIIDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4PII1.3"\
G4SAIDXSDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4SAIDDATA2.0"\
G4ABLADATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4ABLA3.1"\
G4REALSURFACEDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/RealSurface2.1.1"\
G4NEUTRONHPDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4NDL4.5"\
G4PARTICLEXSDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4PARTICLEXS1.1"\
G4ENSDFSTATEDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4ENSDFSTATE2.2"\
G4LEDATA="/opt/fairsoft_jun19p2/share/Geant4-10.5.1/data/G4EMLOW7.7"
ENV SIMPATH="/opt/fairsoft"\
G4INCLDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4INCL1.0"\
G4LEVELGAMMADATA="/opt/fairsoft/share/Geant4-10.5.1/data/PhotonEvaporation5.3"\
G4RADIOACTIVEDATA="/opt/fairsoft/share/Geant4-10.5.1/data/RadioactiveDecay5.3"\
G4PIIDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4PII1.3"\
G4SAIDXSDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4SAIDDATA2.0"\
G4ABLADATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4ABLA3.1"\
G4REALSURFACEDATA="/opt/fairsoft/share/Geant4-10.5.1/data/RealSurface2.1.1"\
G4NEUTRONHPDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4NDL4.5"\
G4PARTICLEXSDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4PARTICLEXS1.1"\
G4ENSDFSTATEDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4ENSDFSTATE2.2"\
G4LEDATA="/opt/fairsoft/share/Geant4-10.5.1/data/G4EMLOW7.7"
ENV PATH="${SIMPATH}/bin:${PATH}"
#
# FairRoot
#
WORKDIR /tmp/fairroot-build
RUN git clone https://github.com/FairRootGroup/FairRoot.git -b v18.2.1 FairRoot && \
mkdir -vp FairRoot_build && cmake -S FairRoot -B FairRoot_build -DCMAKE_INSTALL_PREFIX=/opt/fairroot_v18.2.1 && \
make -C FairRoot_build -j2 install && \
rm -r FairRoot_build && rm -rf /tmp/fair*
ENV FAIRROOTPATH=/opt/fairroot_v18.2.1
WORKDIR /opt/fairsoft
compiler=gcc
debug=no
optimize=no
geant4_download_install_data_automatic=yes
geant4_install_data_from_dir=no
geant4mt=no
build_root6=yes
build_python=yes
install_sim=yes
build_MQOnly=no
SIMPATH_INSTALL=/opt/fairsoft
ARG BASE_IMAGE=gitlab-registry.cern.ch/elavrik/cbmroot-hub/cbmroot:latest
ARG BASE_IMAGE=elavrik/cbmroot:latest
FROM $BASE_IMAGE
# Default user
......@@ -18,7 +18,7 @@ RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
python2 -m pip install metakernel && python2 -m pip install ipykernel && python2 -m ipykernel install && \
python3 -m pip install metakernel && python3 -m pip install ipykernel && python3 -m ipykernel install && \
python3 -m pip install --upgrade pip setuptools notebook jupyterhub && \
jupyter kernelspec install /opt/fairsoft_jun19p2/share/root/etc/notebook/kernels/root
jupyter kernelspec install /opt/fairsoft/share/root/etc/notebook/kernels/root
WORKDIR /srv/jupyterhub
ADD ./jupyterhub_config.py ./jupyterhub_config.py
......
......@@ -998,4 +998,4 @@ for var in os.environ:
#c.CryptKeeper.keys = []
## The number of threads to allocate for encryption
#c.CryptKeeper.n_threads = 40
#c.CryptKeeper.n_threads = 40
\ No newline at end of file
.ipynb_checkpoints
*.root
\ No newline at end of file
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment