github.com/apptainer/singularity@v3.1.1+incompatible/examples/legacy/2.2/contrib/centos7-ompi_cuda.def (about) 1 # Copyright (c) 2017, Paulo Souza. All rights reserved. 2 # 3 # This definition file needs a 2GB image file. 4 # 5 # What is included: 6 # - Infiniband drivers 7 # - CUDA 7.5 runtime 8 # - CUDA-aware OpenMPI 9 # - Linux perf tool 10 # - Dstat tool 11 # 12 # We do recommend a CentOS 7.3 host to build this image. 13 # 14 # How to build the image: 15 # sudo singularity create -s 2000 centos7-ompi_cuda.img 16 # sudo singularity bootstrap centos7-ompi_cuda.img centos7-ompi_cuda.def 17 # 18 # After image creation you do not need sudo to use the image. 19 # 20 21 BootStrap: yum 22 OSVersion: 7 23 MirrorURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/$basearch/ 24 Include: yum 25 26 # If you want the updates (available at the bootstrap date) to be installed 27 # inside the container during the bootstrap instead of the General Availability 28 # point release (7.x) then uncomment the following line 29 #UpdateURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/updates/$basearch/ 30 # runscript: 31 32 %runscript 33 export LD_LIBRARY_PATH=/liboverride:$LD_LIBRARY_PATH:/libfallback 34 export LD_RUN_PATH=/liboverride:$LD_RUN_PATH:/libfallback 35 eval "$@" 36 # post: 37 38 %post 39 export ctnlicense=/LICENSE 40 mkdir -p /app /liboverride /libfallback /prj /localscratch /localflash /opt $ctnlicense 41 alias install_pkg="yum -y install" 42 # Reference: https://www.liquidweb.com/kb/enable-epel-repository/ 43 install_pkg epel-release 44 yum -y update 45 alias clean_pkg='echo "clean not needed."' 46 # dev-tools: 47 yum -y install gcc 48 yum -y install gcc-c++ 49 yum -y install gcc-gfortran 50 yum -y install python-devel 51 install_pkg make 52 install_pkg cmake 53 install_pkg autoconf 54 # basic-tools: 55 install_pkg vim 56 install_pkg curl hostname 57 install_pkg wget 58 install_pkg unzip 59 install_pkg tar 60 install_pkg gzip 61 install_pkg bc 62 install_pkg less 63 install_pkg util-linux 64 install_pkg strace 65 install_pkg which perl-Digest-SHA man 66 # perf-tools: 67 yum install -y flex flex-devel 68 yum install -y bison bison-devel 69 yum install -y elfutils-libelf-devel elfutils-libelf 70 yum install -y systemtap-sdt-devel 71 yum install -y audit-libs audit-libs-devel audit-libs-python 72 yum install -y openssl-devel openssl openssl-libs 73 yum install -y slang slang-devel slang-slsh 74 yum install -y perl perl-ExtUtils-Embed 75 yum install -y xz-devel 76 yum install -y numactl numactl-devel numactl-libs 77 yum install -y cpuid 78 yum install -y elfutils elfutils-devel elfutils-libelf elfutils-libelf-devel elfutils-libs 79 yum install -y libunwind libunwind-devel 80 yum install -y binutils binutils-devel 81 yum install -y e2fsprogs 82 #install_pkg dstat 83 #install_pkg perf 84 echo '2ca34f40067b0c914dbe474bd4ce44dbaec068d684d870c93953480062ce0e42ae32d70392ed572b1cf683ef06683f73c9d98e883405fb75ccbf614a732ce009 -' > dstat.zip.sha512 85 curl -sSL "https://github.com/dagwieers/dstat/archive/0.7.3.zip" | tee dstat.zip | sha512sum -c dstat.zip.sha512 86 unzip dstat.zip && rm -d dstat.zip* 87 cat dstat-*/COPYING > $ctnlicense/dstat.COPYING 88 cd dstat-* 89 make install 90 cd - 91 rm -rf dstat-* 92 echo '5aee3e0b77ff9895ee7e214552c14124f90defa72f15811db81a89e7e09c66ee643a3d1c90a97358b049c863ebe8a811355d758729ca6012bc52e822e54cc044 -' > perf.zip.sha512 93 curl -sSL "https://github.com/torvalds/linux/archive/v4.10.zip" | tee perf.zip | sha512sum -c perf.zip.sha512 94 unzip perf.zip && rm -f perf.zip* 95 cat linux-*/COPYING > $ctnlicense/perf.COPYING 96 cd linux-*/tools/perf 97 make 98 mv perf /usr/bin/ 99 cd - 100 rm -rf linux-* 101 echo '600a0161d7eca085594191b133cb25ec6bb6db41d0857cfac7f6b3bdf45647961f329c4cd423fd3dacb34fd09183b762c0b70bcd7b06a878666c8a241c3a7e53 -' > pmu-tools.zip.sha512 102 curl -sSL "https://github.com/andikleen/pmu-tools/archive/r105.zip" | tee pmu-tools.zip | sha512sum -c pmu-tools.zip.sha512 103 unzip pmu-tools.zip -d /app && rm -rf pmu-tools.zip* 104 cat /app/pmu-tools*/COPYING > $ctnlicense/pmu-tools.COPYING 105 echo '739dc803dd90b34e7b0a817eeb32d9a737a6809e8f2ad6bcfca16505d4934c1ac62b6fec1e90f59cbf6aabf8d38852c980115c9c47758c9fb3b0539dc2f6abb2 -' > likwid.tgz.sha512 106 curl -sSL "http://ftp.fau.de/pub/likwid/likwid-stable.tar.gz" |tee likwid.tgz|sha512sum -c likwid.tgz.sha512 107 tar -zxvf likwid.tgz && rm -f likwid.tgz* 108 cd likwid-* 109 cat COPYING > $ctnlicense/likwid.COPYING 110 make 111 make install 112 cd - 113 rm -rf likwid-* 114 # perf reference: http://www.tecmint.com/perf-performance-monitoring-and-analysis-tool-for-linux/ 115 # Additionally, keep in mind that some perf commands may be restricted to 116 # `root by default, which can be disabled (until the system is rebooted) by doing: 117 # echo 0 > /proc/sys/kernel/perf_event_paranoid 118 # If you need to disable paranoid mode permanently, update the following setting in /etc/sysctl.conf file. 119 # kernel.perf_event_paranoid = 0 120 # /proc/sys/kernel/kptr_restrict 121 # sudo sysctl -w kernel.perf_event_paranoid=0 122 # infiniband: 123 # Infiniband + RDMA Reference: 124 # https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Networking_Guide/sec-InfiniBand_and_RDMA_related_software_packages.html 125 yum -y install rdma 126 yum -y install libibverbs-devel libsysfs-devel # required by custom openmpi 127 yum -y install infinipath-psm libcxgb3 libcxgb4 libipathverbs libmthca libmlx4 libmlx5 libnes libocrdma 128 yum -y install librdmacm librdmacm-utils ibacm 129 yum -y install libibverbs-utils 130 yum -y install infiniband-diags ibutils 131 yum -y install perftest qperf 132 #required by OpenMPI 133 yum install -y numactl numactl-devel numactl-libs 134 yum install -y environment-modules hwloc hwloc-libs hwloc-devel libfabric libpsm2 libfabric-devel 135 # cuda75: 136 echo '40315500010e078d7cfb71534ac8d0a52c5da9dfbe24ecb6fd7c7d1dcef60ae0db309a51925e395c13af390327f5a9f4682516d9d4994beaa3684b69d459b96b -' > cuda-repo.rpm.sha512 137 curl -sSL "http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-repo-rhel7-7.5-18.x86_64.rpm" |\ 138 tee cuda-repo.rpm | sha512sum -c cuda-repo.rpm.sha512 139 rpm -i cuda-repo.rpm && rm -f cuda-repo.rpm* 140 yum -y update 141 export cuda_ver=7-5 142 export CUDA_VERSION=7.5.18 143 install_pkg cuda-minimal-build-$cuda_ver 144 install_pkg cuda-command-line-tools-$cuda_ver 145 ln -s /usr/local/cuda-*/ /usr/local/cuda 146 export CUDA_PATH=/usr/local/cuda 147 export CUDA_HOME=$CUDA_PATH 148 export CUDA_INC=$CUDA_PATH/include 149 export CUDADIR=$CUDA_PATH 150 export CUDA_ROOT=$CUDA_PATH 151 env | grep "^CUDA.*="|sed -e "s/^/export /" >> /environment 152 echo 'export PATH=$CUDA_PATH/bin:$PATH' >> /environment 153 echo 'export LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH' >> /environment 154 export LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH 155 cat /usr/local/cuda/doc/EULA.txt > $ctnlicense/CUDA.EULA.txt 156 # cuda-aware-mpi: 157 echo '970e48d19b544a401511a1fe2c6ff9655593607ef5db8cfcb3a913e34d2362d58fe537c356642bd98d2f2355daf3556d7ca37e1b90c14ceabbaae73840716cdb -' > openmpi.tgz.sha512 158 curl -sSL "https://www.open-mpi.org/software/ompi/v2.0/downloads/openmpi-2.0.2.tar.gz" | \ 159 tee openmpi.tgz | sha512sum -c openmpi.tgz.sha512 160 tar -zxvf openmpi.tgz && rm -f openmpi.tgz* 161 cd openmpi-* 162 cat LICENSE > $ctnlicense/OpenMPI.LICENSE 163 ./configure --prefix=/app/openmpi --with-verbs --with-cuda \ 164 CFLAGS=-I$CUDA_PATH/include CPPFLAGS=-I$CUDA_PATH/include \ 165 CXXFLAGS=-I$CUDA_PATH/include LDFLAGS=-L$CUDA_PATH/lib64 166 make 167 make install 168 cd - 169 rm -rf openmpi-* 170 export PATH=/app/openmpi/bin:$PATH 171 export LD_LIBRARY_PATH=/app/openmpi/lib:$LD_LIBRARY_PATH 172 export LD_RUN_PATH=/app/openmpi/lib:$LD_RUN_PATH 173 echo 'export PATH=/app/openmpi/bin:$PATH' >> /environment 174 echo 'export LD_LIBRARY_PATH=/app/openmpi/lib:$LD_LIBRARY_PATH' >> /environment 175 echo 'export LD_RUN_PATH=/app/openmpi/lib:$LD_RUN_PATH' >> /environment 176 echo '74561a004160e064b44848044c9595b337337b60d8f23c87608cbbceee52e8737f4770c2d8fc003f9e0478ea13e6451e29fd13796b45803ccd56f765e6fe058c -' > osu-micro-benchmarks.tgz.sha512 177 curl -sSL "http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz" | \ 178 tee osu-micro-benchmarks.tgz | sha512sum -c osu-micro-benchmarks.tgz.sha512 179 tar -zxvf osu-micro-benchmarks.tgz && rm -f osu-micro-benchmarks.tgz* 180 cd osu-micro-benchmarks-* 181 cat COPYRIGHT > $ctnlicense/OSU.COPYRIGHT 182 ./configure --prefix=/app/openmpi CC=/app/openmpi/bin/mpicc CXX=/app/openmpi/bin/mpicxx 183 make 184 make install 185 cd - 186 rm -rf osu-micro-benchmarks-* 187 ln -s /app/openmpi/libexec/osu-micro-benchmarks/mpi/*/osu_* /app/openmpi/bin/