S3

generate a unique postfix

export BUCKET_NAME=pcluster-$(uuidgen --random | cut -d'-' -f1)
echo "Your bucket name will be ${BUCKET_NAME}"
aws s3 mb s3://${BUCKET_NAME}

Spack Env

cat << EOF >> spack.yaml
spack:
  view: false
  concretization: separately

  config:
    install_tree: /fsx/spack/opt/spack
    build_jobs: 4

  packages:
    all:
      providers:
        blas:
        - openblas
        mpi:
        - mpich
      variants: +mpi
    mpich:
      variants: ~wrapperrpath netmod=ofi
      version:
        - 3.2.1
    binutils:
      variants: +gold+headers+libiberty~nls
      version:
        - 2.33.1
    cmake:
      version:
        - 3.17.3
    openfoam:
      variants: +paraview
      version:
        - 'develop'
    paraview:
      variants: +osmesa+python3
      version:
        - 5.8.1
    mesa:
      variants: swr=avx,avx2
      version:
        - 18.3.6
    llvm:
      version:
        - 6.0.1
    hwloc:
      version:
        - 1.11.11

  definitions:
  - hackathon:
    - openfoam
    - gromacs
  - arch:
    - '%gcc@7.3.1 arch=linux-amzn2-skylake_avx512'
  mirrors: { "mirror": "s3://spack-mirrors/amzn2-e4s" }
EOF
aws s3 cp spack.yaml s3://${BUCKET_NAME}/spack.yaml

Post Install Script

The next step is to create a post-installation script to be executed by ParallelCluster when provisioning the instances.

wget --no-check-certificate -qO- https://gromacs-on-pcluster.workshop.aws/setup/s3/_index.sys.files/post-install.sh \
      |sed "s/BUCKET_NAME/${BUCKET_NAME}/" > post-install.sh
chmod +x post-install.sh

# upload to your bucket
aws s3 cp post-install.sh s3://${BUCKET_NAME}/post-install.sh

# delete local copies
rm -rf post-install.sh

Before continuing, check the content of your bucket using the AWS CLI with the command aws s3 ls s3://${BUCKET_NAME} or the AWS console.

Next, define the configuration of the ML cluster by creating the AWS ParallelCluster configuration file.