#!/bin/bash

echo "Collect System information"

echo "OS: $(cat /etc/system-release)"

echo "Kernel: $(uname -r)"

echo "MD version: $(mdadm --version 2>&1)"

echo "fio version: $(fio --version)"

echo "XFS version: $(mkfs.xfs -V)"

echo "Display NVME information"

lspci -vt | grep P4500

while read line
do
    PCI_BUS=$(echo $line | cut -d ' ' -f 1)
    NVME_IDX=$(ls /sys/bus/pci/drivers/nvme/0000:${PCI_BUS}/nvme)
    NUMA_NODE=$(lspci -vvvs ${PCI_BUS} | grep NUMA) 
    echo ${PCI_BUS} ${NVME_IDX} ${NUMA_NODE}
done < <(lspci | grep Non-)

if [ $(lspci | grep Non- | wc -l) != 33 ]; then
    echo "NVME count is not 32, exit script"
    exit
fi

if [ $(cat /proc/mdstat | grep chunks | wc -l) != 0 ]; then
    echo "Please remove all MD raid before run the script, exit script"
    exit
fi

echo "Create md array"

# note that /dev/nvme0 is a M.2 drive that is extra. So dont use that

yes | sudo mdadm -Cv -l 0 /dev/md0 -n 8 /dev/nvme{1,2,3,4,5,6,7,8}n1
yes | sudo mdadm -Cv -l 0 /dev/md1 -n 8 /dev/nvme{9,10,11,12,13,14,15,16}n1
yes | sudo mdadm -Cv -l 0 /dev/md2 -n 8 /dev/nvme{17,18,19,20,21,22,23,24}n1
yes | sudo mdadm -Cv -l 0 /dev/md3 -n 8 /dev/nvme{25,26,27,28,29,30,31,32}n1

echo "create file system"

sudo mkfs.xfs -f /dev/md0
sudo mkfs.xfs -f /dev/md1
sudo mkfs.xfs -f /dev/md2
sudo mkfs.xfs -f /dev/md3

echo "mount file system"

sudo mkdir /mnt/array0
sudo mkdir /mnt/array1
sudo mkdir /mnt/array2
sudo mkdir /mnt/array3

sudo mount /dev/md0 /mnt/array0
sudo mount /dev/md1 /mnt/array1
sudo mount /dev/md2 /mnt/array2
sudo mount /dev/md3 /mnt/array3

sudo chown clameter /mnt/array{0,1,2,3}

echo "create regular fio configuration and run"

echo "[global]
rw=write
size=4G
blocksize=2M
iodepth=4
fsync_on_close=1
direct=0
numjobs=1

[md0-1]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file1

[md0-2]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file2

[md0-3]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file3

[md0-4]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file4

[md0-5]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file5

[md0-6]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file6


[md1-1]
numa_cpu_nodes=1
numa_mem_policy=bind:1
#filename=/dev/md1
filename=/mnt/array1/file1

[md1-2]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file2

[md1-3]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file3

[md1-4]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file4

[md1-5]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file5

[md1-6]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file6

[md2-1]
numa_cpu_nodes=2
numa_mem_policy=bind:2
#filename=/dev/md2
filename=/mnt/array2/file1

[md2-2]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file2

[md2-3]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file3

[md2-4]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file4

[md2-5]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file5

[md2-6]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file6

[md3-1]
numa_cpu_nodes=3
numa_mem_policy=bind:3
#filename=/dev/md3
filename=/mnt/array3/file1

[md3-2]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file2

[md3-3]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file3

[md3-4]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file4

[md3-5]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file5

[md3-6]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file6" > regular_thread6_file6

echo "Regular, thread=6, file=6, ignore round-1 to avoid get wrong performance" 
fio regular_thread6_file6 | grep WRITE

echo "Regular, thread=6, file=6, round-2"
fio regular_thread6_file6 | grep WRITE

echo "Regular, thread=6, file=6, round-3"
fio regular_thread6_file6 | grep WRITE

echo "Regular, thread=6, file=6, round-4"
fio regular_thread6_file6 | grep WRITE

echo "create direct fio configuration and run"

echo "[global]
rw=write
size=4G
blocksize=2M
iodepth=4
fsync_on_close=1
direct=1
numjobs=1

[md0-1]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file1

[md0-2]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file2

[md0-3]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file3

[md0-4]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file4

[md0-5]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file5

[md0-6]
numa_cpu_nodes=0
numa_mem_policy=bind:0
filename=/mnt/array0/file6


[md1-1]
numa_cpu_nodes=1
numa_mem_policy=bind:1
#filename=/dev/md1
filename=/mnt/array1/file1

[md1-2]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file2

[md1-3]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file3

[md1-4]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file4

[md1-5]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file5

[md1-6]
numa_cpu_nodes=1
numa_mem_policy=bind:1
filename=/mnt/array1/file6

[md2-1]
numa_cpu_nodes=2
numa_mem_policy=bind:2
#filename=/dev/md2
filename=/mnt/array2/file1

[md2-2]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file2

[md2-3]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file3

[md2-4]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file4

[md2-5]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file5

[md2-6]
numa_cpu_nodes=2
numa_mem_policy=bind:2
filename=/mnt/array2/file6

[md3-1]
numa_cpu_nodes=3
numa_mem_policy=bind:3
#filename=/dev/md3
filename=/mnt/array3/file1

[md3-2]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file2

[md3-3]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file3

[md3-4]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file4

[md3-5]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file5

[md3-6]
numa_cpu_nodes=3
numa_mem_policy=bind:3
filename=/mnt/array3/file6" > direct_thread6_file6

echo "Direct, thread=6, file=6, ignore round-1 to avoid get wrong performance"
fio direct_thread6_file6 | grep WRITE

echo "Direct, thread=6, file=6, round-2"
fio direct_thread6_file6 | grep WRITE

echo "Direct, thread=6, file=6, round-3"
fio direct_thread6_file6 | grep WRITE

echo "Direct, thread=6, file=6, round-4"
fio direct_thread6_file6 | grep WRITE

echo "TEST Done"