github.com/apptainer/singularity@v3.1.1+incompatible/examples/legacy/2.2/contrib/centos7-ompi_cuda.def (about)

     1  # Copyright (c) 2017, Paulo Souza. All rights reserved.
     2  #
     3  # This definition file needs a 2GB image file.
     4  #
     5  # What is included:
     6  # - Infiniband drivers
     7  # - CUDA 7.5 runtime
     8  # - CUDA-aware OpenMPI
     9  # - Linux perf tool
    10  # - Dstat tool
    11  #
    12  # We do recommend a CentOS 7.3 host to build this image.
    13  #
    14  # How to build the image:
    15  # sudo singularity create -s 2000 centos7-ompi_cuda.img
    16  # sudo singularity bootstrap centos7-ompi_cuda.img centos7-ompi_cuda.def
    17  #
    18  # After image creation you do not need sudo to use the image.
    19  #
    20  
    21  BootStrap: yum
    22  OSVersion: 7
    23  MirrorURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/$basearch/
    24  Include: yum
    25  
    26  # If you want the updates (available at the bootstrap date) to be installed
    27  # inside the container during the bootstrap instead of the General Availability
    28  # point release (7.x) then uncomment the following line
    29  #UpdateURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/updates/$basearch/
    30  # runscript:
    31  
    32  %runscript
    33      export LD_LIBRARY_PATH=/liboverride:$LD_LIBRARY_PATH:/libfallback
    34      export LD_RUN_PATH=/liboverride:$LD_RUN_PATH:/libfallback
    35      eval "$@"
    36  # post:
    37  
    38  %post
    39      export ctnlicense=/LICENSE
    40      mkdir -p /app /liboverride /libfallback /prj /localscratch /localflash /opt $ctnlicense
    41      alias install_pkg="yum -y install"
    42      # Reference: https://www.liquidweb.com/kb/enable-epel-repository/
    43      install_pkg epel-release
    44      yum -y update
    45      alias clean_pkg='echo "clean not needed."'
    46  # dev-tools:
    47      yum -y install gcc
    48      yum -y install gcc-c++
    49      yum -y install gcc-gfortran
    50      yum -y install python-devel
    51      install_pkg make
    52      install_pkg cmake
    53      install_pkg autoconf
    54  # basic-tools:
    55      install_pkg vim
    56      install_pkg curl hostname 
    57      install_pkg wget
    58      install_pkg unzip
    59      install_pkg tar
    60      install_pkg gzip
    61      install_pkg bc
    62      install_pkg less
    63      install_pkg util-linux
    64      install_pkg strace
    65      install_pkg which perl-Digest-SHA man
    66  # perf-tools:
    67      yum install -y flex flex-devel
    68      yum install -y bison bison-devel
    69      yum install -y elfutils-libelf-devel elfutils-libelf
    70      yum install -y systemtap-sdt-devel
    71      yum install -y audit-libs audit-libs-devel audit-libs-python
    72      yum install -y openssl-devel openssl openssl-libs
    73      yum install -y slang slang-devel slang-slsh
    74      yum install -y perl perl-ExtUtils-Embed
    75      yum install -y xz-devel
    76      yum install -y numactl numactl-devel numactl-libs
    77      yum install -y cpuid
    78      yum install -y elfutils elfutils-devel elfutils-libelf elfutils-libelf-devel elfutils-libs
    79      yum install -y libunwind libunwind-devel
    80      yum install -y binutils binutils-devel
    81      yum install -y e2fsprogs
    82      #install_pkg dstat    
    83      #install_pkg perf
    84      echo '2ca34f40067b0c914dbe474bd4ce44dbaec068d684d870c93953480062ce0e42ae32d70392ed572b1cf683ef06683f73c9d98e883405fb75ccbf614a732ce009  -' > dstat.zip.sha512
    85      curl -sSL "https://github.com/dagwieers/dstat/archive/0.7.3.zip" | tee dstat.zip | sha512sum -c dstat.zip.sha512
    86      unzip dstat.zip && rm -d dstat.zip*
    87      cat dstat-*/COPYING > $ctnlicense/dstat.COPYING
    88      cd dstat-*
    89      make install
    90      cd - 
    91      rm -rf dstat-*
    92      echo '5aee3e0b77ff9895ee7e214552c14124f90defa72f15811db81a89e7e09c66ee643a3d1c90a97358b049c863ebe8a811355d758729ca6012bc52e822e54cc044  -' > perf.zip.sha512
    93      curl -sSL "https://github.com/torvalds/linux/archive/v4.10.zip" | tee perf.zip | sha512sum -c perf.zip.sha512
    94      unzip perf.zip && rm -f perf.zip*
    95      cat linux-*/COPYING > $ctnlicense/perf.COPYING
    96      cd linux-*/tools/perf
    97      make
    98      mv perf /usr/bin/
    99      cd -
   100      rm -rf linux-*
   101      echo '600a0161d7eca085594191b133cb25ec6bb6db41d0857cfac7f6b3bdf45647961f329c4cd423fd3dacb34fd09183b762c0b70bcd7b06a878666c8a241c3a7e53  -' > pmu-tools.zip.sha512
   102      curl -sSL "https://github.com/andikleen/pmu-tools/archive/r105.zip" | tee pmu-tools.zip | sha512sum -c pmu-tools.zip.sha512
   103      unzip pmu-tools.zip -d /app && rm -rf pmu-tools.zip*
   104      cat /app/pmu-tools*/COPYING > $ctnlicense/pmu-tools.COPYING
   105      echo '739dc803dd90b34e7b0a817eeb32d9a737a6809e8f2ad6bcfca16505d4934c1ac62b6fec1e90f59cbf6aabf8d38852c980115c9c47758c9fb3b0539dc2f6abb2  -' > likwid.tgz.sha512
   106      curl -sSL "http://ftp.fau.de/pub/likwid/likwid-stable.tar.gz" |tee likwid.tgz|sha512sum -c likwid.tgz.sha512
   107      tar -zxvf likwid.tgz && rm -f likwid.tgz*
   108      cd likwid-*
   109      cat COPYING > $ctnlicense/likwid.COPYING
   110      make
   111      make install
   112      cd -
   113      rm -rf likwid-*
   114      # perf reference: http://www.tecmint.com/perf-performance-monitoring-and-analysis-tool-for-linux/
   115      # Additionally, keep in mind that some perf commands may be restricted to
   116      # `root by default, which can be disabled (until the system is rebooted) by doing:
   117      # echo 0 > /proc/sys/kernel/perf_event_paranoid
   118      # If you need to disable paranoid mode permanently, update the following setting in /etc/sysctl.conf file.
   119      # kernel.perf_event_paranoid = 0
   120      # /proc/sys/kernel/kptr_restrict
   121      # sudo sysctl -w kernel.perf_event_paranoid=0
   122  # infiniband:
   123      # Infiniband + RDMA Reference:
   124      # https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Networking_Guide/sec-InfiniBand_and_RDMA_related_software_packages.html    
   125      yum -y install rdma
   126      yum -y install libibverbs-devel libsysfs-devel # required by custom openmpi
   127      yum -y install infinipath-psm libcxgb3 libcxgb4 libipathverbs libmthca libmlx4 libmlx5 libnes libocrdma
   128      yum -y install librdmacm librdmacm-utils ibacm
   129      yum -y install libibverbs-utils
   130      yum -y install infiniband-diags ibutils
   131      yum -y install perftest qperf
   132      #required by OpenMPI
   133      yum install -y numactl numactl-devel numactl-libs
   134      yum install -y environment-modules hwloc hwloc-libs hwloc-devel libfabric libpsm2 libfabric-devel
   135  # cuda75:
   136      echo '40315500010e078d7cfb71534ac8d0a52c5da9dfbe24ecb6fd7c7d1dcef60ae0db309a51925e395c13af390327f5a9f4682516d9d4994beaa3684b69d459b96b  -' > cuda-repo.rpm.sha512
   137      curl -sSL "http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-repo-rhel7-7.5-18.x86_64.rpm" |\
   138          tee cuda-repo.rpm | sha512sum -c  cuda-repo.rpm.sha512
   139      rpm -i  cuda-repo.rpm && rm -f  cuda-repo.rpm*
   140      yum -y update
   141      export cuda_ver=7-5
   142      export CUDA_VERSION=7.5.18
   143      install_pkg cuda-minimal-build-$cuda_ver
   144      install_pkg cuda-command-line-tools-$cuda_ver
   145      ln -s /usr/local/cuda-*/ /usr/local/cuda
   146      export CUDA_PATH=/usr/local/cuda
   147      export CUDA_HOME=$CUDA_PATH
   148      export CUDA_INC=$CUDA_PATH/include
   149      export CUDADIR=$CUDA_PATH
   150      export CUDA_ROOT=$CUDA_PATH
   151      env | grep "^CUDA.*="|sed -e "s/^/export /" >> /environment
   152      echo 'export PATH=$CUDA_PATH/bin:$PATH' >> /environment
   153      echo 'export LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH' >> /environment
   154      export LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH
   155      cat /usr/local/cuda/doc/EULA.txt > $ctnlicense/CUDA.EULA.txt
   156  # cuda-aware-mpi:
   157      echo '970e48d19b544a401511a1fe2c6ff9655593607ef5db8cfcb3a913e34d2362d58fe537c356642bd98d2f2355daf3556d7ca37e1b90c14ceabbaae73840716cdb  -' > openmpi.tgz.sha512    
   158      curl -sSL "https://www.open-mpi.org/software/ompi/v2.0/downloads/openmpi-2.0.2.tar.gz" | \
   159        tee openmpi.tgz | sha512sum -c openmpi.tgz.sha512
   160      tar -zxvf openmpi.tgz && rm -f openmpi.tgz*
   161      cd openmpi-*
   162      cat LICENSE > $ctnlicense/OpenMPI.LICENSE
   163      ./configure --prefix=/app/openmpi --with-verbs --with-cuda \
   164          CFLAGS=-I$CUDA_PATH/include CPPFLAGS=-I$CUDA_PATH/include \
   165          CXXFLAGS=-I$CUDA_PATH/include LDFLAGS=-L$CUDA_PATH/lib64
   166      make
   167      make install
   168      cd -
   169      rm -rf openmpi-*
   170      export PATH=/app/openmpi/bin:$PATH
   171      export LD_LIBRARY_PATH=/app/openmpi/lib:$LD_LIBRARY_PATH
   172      export LD_RUN_PATH=/app/openmpi/lib:$LD_RUN_PATH
   173      echo 'export PATH=/app/openmpi/bin:$PATH' >> /environment
   174      echo 'export LD_LIBRARY_PATH=/app/openmpi/lib:$LD_LIBRARY_PATH' >> /environment
   175      echo 'export LD_RUN_PATH=/app/openmpi/lib:$LD_RUN_PATH' >> /environment
   176      echo '74561a004160e064b44848044c9595b337337b60d8f23c87608cbbceee52e8737f4770c2d8fc003f9e0478ea13e6451e29fd13796b45803ccd56f765e6fe058c  -' > osu-micro-benchmarks.tgz.sha512
   177      curl -sSL "http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz" | \
   178          tee osu-micro-benchmarks.tgz | sha512sum -c osu-micro-benchmarks.tgz.sha512
   179      tar -zxvf osu-micro-benchmarks.tgz && rm -f osu-micro-benchmarks.tgz*
   180      cd osu-micro-benchmarks-*
   181      cat COPYRIGHT > $ctnlicense/OSU.COPYRIGHT
   182      ./configure --prefix=/app/openmpi CC=/app/openmpi/bin/mpicc CXX=/app/openmpi/bin/mpicxx
   183      make
   184      make install
   185      cd -
   186      rm -rf osu-micro-benchmarks-*
   187      ln -s /app/openmpi/libexec/osu-micro-benchmarks/mpi/*/osu_* /app/openmpi/bin/