scripts/zpios-survey.sh

   1 #!/bin/bash
   2 #
   3 # Wrapper script for easily running a survey of zpios based tests
   4 #
   5
   6 basedir="$(dirname $0)"
   7
   8 SCRIPT_COMMON=common.sh
   9 if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
  10 . "${basedir}/${SCRIPT_COMMON}"
  11 else
  12 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
  13 fi
  14
  15 PROG=zpios-survey.sh
  16
  17 usage() {
  18 cat << EOF
  19 USAGE:
  20 $0 [hvp] <-c config> <-t test>
  21
  22 DESCRIPTION:
  23         Helper script for easy zpios survey benchmarking.
  24
  25 OPTIONS:
  26         -h      Show this message
  27         -v      Verbose
  28         -p      Enable profiling
  29         -c      Zpool configuration
  30         -t      Zpios test
  31         -l      Zpios survey log
  32
  33 EOF
  34 }
  35
  36 print_header() {
  37 tee -a ${ZPIOS_SURVEY_LOG} << EOF
  38
  39 ================================================================
  40 Test: $1
  41 EOF
  42 }
  43
  44 # Baseline performance for an out of the box config with no manual tuning.
  45 # Ideally, we want everything to be automatically tuned for your system and
  46 # for this to perform reasonably well.
  47 zpios_survey_base() {
  48         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+baseline"
  49         print_header ${TEST_NAME}
  50
  51         ${ZFS_SH} ${VERBOSE_FLAG} | \
  52                 tee -a ${ZPIOS_SURVEY_LOG}
  53         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} | \
  54                 tee -a ${ZPIOS_SURVEY_LOG}
  55         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
  56                 tee -a ${ZPIOS_SURVEY_LOG}
  57 }
  58
  59 # Disable ZFS's prefetching.  For some reason still not clear to me
  60 # current prefetching policy is quite bad for a random workload.
  61 # Allowing the algorithm to detect a random workload and not do
  62 # anything may be the way to address this issue.
  63 zpios_survey_prefetch() {
  64         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+prefetch"
  65         print_header ${TEST_NAME}
  66
  67         ${ZFS_SH} ${VERBOSE_FLAG}               \
  68                 tee -a ${ZPIOS_SURVEY_LOG}
  69         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
  70                 -o "--noprefetch" |                                    \
  71                 tee -a ${ZPIOS_SURVEY_LOG}
  72         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
  73                 tee -a ${ZPIOS_SURVEY_LOG}
  74 }
  75
  76 # Simulating a zerocopy IO path should improve performance by freeing up
  77 # lots of CPU which is wasted move data between buffers.
  78 zpios_survey_zerocopy() {
  79         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+zerocopy"
  80         print_header ${TEST_NAME}
  81
  82         ${ZFS_SH} ${VERBOSE_FLAG} | \
  83                 tee -a ${ZPIOS_SURVEY_LOG}
  84         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
  85                 -o "--zerocopy" |                                      \
  86                 tee -a ${ZPIOS_SURVEY_LOG}
  87         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
  88                 tee -a ${ZPIOS_SURVEY_LOG}
  89 }
  90
  91 # Disabling checksumming should show some (if small) improvement
  92 # simply due to freeing up a modest amount of CPU.
  93 zpios_survey_checksum() {
  94         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+checksum"
  95         print_header ${TEST_NAME}
  96
  97         ${ZFS_SH} ${VERBOSE_FLAG} | \
  98                 tee -a ${ZPIOS_SURVEY_LOG}
  99         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
 100                 -s "set checksum=off" |                                \
 101                 tee -a ${ZPIOS_SURVEY_LOG}
 102         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
 103                 tee -a ${ZPIOS_SURVEY_LOG}
 104 }
 105
 106 # Increasing the pending IO depth also seems to improve things likely
 107 # at the expense of latency.  This should be explored more because I'm
 108 # seeing a much bigger impact there that I would have expected.  There
 109 # may be some low hanging fruit to be found here.
 110 zpios_survey_pending() {
 111         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+pending"
 112         print_header ${TEST_NAME}
 113
 114         ${ZFS_SH} ${VERBOSE_FLAG}                  \
 115                 zfs="zfs_vdev_max_pending=1024" | \
 116                 tee -a ${ZPIOS_SURVEY_LOG}
 117         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} | \
 118                 tee -a ${ZPIOS_SURVEY_LOG}
 119         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
 120                 tee -a ${ZPIOS_SURVEY_LOG}
 121 }
 122
 123 # To avoid memory fragmentation issues our slab implementation can be
 124 # based on a virtual address space.  Interestingly, we take a pretty
 125 # substantial performance penalty for this somewhere in the low level
 126 # IO drivers.  If we back the slab with kmem pages we see far better
 127 # read performance numbers at the cost of memory fragmention and general
 128 # system instability due to large allocations.  This may be because of
 129 # an optimization in the low level drivers due to the contigeous kmem
 130 # based memory.  This needs to be explained.  The good news here is that
 131 # with zerocopy interfaces added at the DMU layer we could gaurentee
 132 # kmem based memory for a pool of pages.
 133 #
 134 # 0x100 = KMC_KMEM - Force kmem_* based slab
 135 # 0x200 = KMC_VMEM - Force vmem_* based slab
 136 zpios_survey_kmem() {
 137         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+kmem"
 138         print_header ${TEST_NAME}
 139
 140         ${ZFS_SH} ${VERBOSE_FLAG}             \
 141                 zfs="zio_bulk_flags=0x100" | \
 142                 tee -a ${ZPIOS_SURVEY_LOG}
 143         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} | \
 144                 tee -a ${ZPIOS_SURVEY_LOG}
 145         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
 146                 tee -a ${ZPIOS_SURVEY_LOG}
 147 }
 148
 149 # Apply all possible turning concurrently to get a best case number
 150 zpios_survey_all() {
 151         TEST_NAME="${ZPOOL_CONFIG}+${ZPIOS_TEST}+all"
 152         print_header ${TEST_NAME}
 153
 154         ${ZFS_SH} ${VERBOSE_FLAG}                \
 155                 zfs="zfs_vdev_max_pending=1024" \
 156                 zfs="zio_bulk_flags=0x100" |    \
 157                 tee -a ${ZPIOS_SURVEY_LOG}
 158         ${ZPIOS_SH} ${VERBOSE_FLAG} -c ${ZPOOL_CONFIG} -t ${ZPIOS_TEST} \
 159                 -o "--noprefetch --zerocopy"                           \
 160                 -s "set checksum=off" |                                \
 161                 tee -a ${ZPIOS_SURVEY_LOG}
 162         ${ZFS_SH} -u ${VERBOSE_FLAG} | \
 163                 tee -a ${ZPIOS_SURVEY_LOG}
 164 }
 165
 166
 167 PROFILE=
 168 ZPOOL_NAME=zpios-survey
 169 ZPOOL_CONFIG=zpool-config.sh
 170 ZPIOS_TEST=zpios-test.sh
 171 ZPIOS_SURVEY_LOG=/dev/null
 172
 173 while getopts 'hvpc:t:l:' OPTION; do
 174         case $OPTION in
 175         h)
 176                 usage
 177                 exit 1
 178                 ;;
 179         v)
 180                 VERBOSE=1
 181                 VERBOSE_FLAG="-v"
 182                 ;;
 183         p)
 184                 PROFILE=1
 185                 PROFILE_FLAG="-p"
 186                 ;;
 187         c)
 188                 ZPOOL_CONFIG=${OPTARG}
 189                 ;;
 190         t)
 191                 ZPIOS_TEST=${OPTARG}
 192                 ;;
 193         l)
 194                 ZPIOS_SURVEY_LOG=${OPTARG}
 195                 ;;
 196         ?)
 197                 usage
 198                 exit
 199                 ;;
 200         esac
 201 done
 202
 203 if [ $(id -u) != 0 ]; then
 204         die "Must run as root"
 205 fi
 206
 207 zpios_survey_base
 208 zpios_survey_prefetch
 209 zpios_survey_zerocopy
 210 zpios_survey_checksum
 211 zpios_survey_pending
 212 zpios_survey_kmem
 213 zpios_survey_all
 214
 215 exit 0