#!/bin/bash
export PATH="$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
declare -a SLOTS
declare -a OSDS_BY_NVME
declare -a IDS
declare -a LVS
declare -a LUKS
declare -A OSDS_UUID
force=0
randomize=0
usage() {
    MSG="""
    Usage: $basename $0 [-h] [-n </dev/nvmeXnY>] [-f] [-a]

    Argument:
        -n: nvme disk (nvmeXn2). If not set, operates on all nvmeXn2 disks
	-f: force disk reset. Usefull if the number of max attempts has been reached
	-a: After reseting the nvme disk, archive ceph OSD crashes.
	-r: Randomize start between 1-90 seconds.

 """
    printf "$MSG" 
}

# Options
#
while getopts "afhn:r" OPTION
do
        case $OPTION in
                h)
                        usage
			exit
                        ;;
                a)
                        archive=1
                        ;;
                f)
                        force=1
                        ;;
                n)
                        NVME=$OPTARG
                        ;;
                r)
                        randomize=1
                        ;;
        esac
done

archive_osd_crash() {
	OSD=$1
        ceph crash ls-new | grep -w $OSD | awk '{print $1}' | while read i
	do
	       	ceph crash archive $i
		echo "crash $i archived"
	done
}

check_attempts() {
	SN=$1
	# Get the current timestamp
	current_timestamp=$(date +"%s")

	# 30 * 24 hours in seconds
	hours=$((30 * 24 * 60 * 60 ))

	# Calculate the timestamp for x hours ago
	hours_ago=$((current_timestamp - hours))

	count=$(awk -v start_time="$hours_ago" -v sn="$SN" '
            {
                log_time=$1
                if (log_time >= start_time && $3 == sn) {
                    print $0
                }
            }
        ' /var/log/ct-repair-nvme-osd.log | wc -l)

        echo "$count"
}
[ $randomize -eq 1 ] && sleep $((RANDOM % 90))

echo Listing disks to repair.
[ -z "$NVME" ] && DEVS=$(lsblk | awk '{print $1}' | grep n2)
[ ! -z "$NVME" ] && DEVS=${NVME}
[ -z "$DEVS" ] && echo "No disk to repair" && exit 0
echo $DEVS
# Retrieve OSDs For this Host
for DEV in $DEVS
do
    SN=$(nvme list | grep $DEV | tail -n 1 | awk '{print $2}')

    log_count=$(check_attempts $SN)
    echo "Number of auto repairs for $DEV $SN : $log_count"
    # Check if count exceeds 3 and exit the script
    if [ $log_count -ge 1 ] && [ $force -ne 1 ]; then
	echo "Already ran ct-repair-nvme-osd $log_count times for $DEV $SN. Not retrying."
        continue
    fi
    bad_series=("000060232912" "000060232913" "000060232937" "000060233094" "000060233095" "000060233128")
    valid=true
    for prefix in "${bad_series[@]}"; do
      if [[ $SN == $prefix* ]]; then
        valid=false
        break
      fi
    done

    if ! $valid  && [ $force -ne 1 ]; then
        echo $(date +"%s") $DEV $SN - Part of bad series >> /var/log/ct-repair-nvme-osd.log
        continue
    fi

    echo $(date +"%s") $DEV $SN >> /var/log/ct-repair-nvme-osd.log
    IDX=$(echo ${DEV} | grep -o '[[:digit:]]*' |head -1)
    SEARCH=" nvme${IDX}n1 "
    OSDS=$(ceph device ls-by-host $(hostname) | grep "${SEARCH}" | awk -F ${SEARCH} '{print $2}'| awk -F' osd.' '{for(i=2;i<=NF;i++) print $i};'|xargs)
    OSDS_BY_NVME[$IDX]=$OSDS
    for OSD in $OSDS; do
        ID=$(cat /var/lib/ceph/osd/ceph-${OSD}/lockbox.keyring | grep client | awk -F'.' '{print $3}'| tr -d "]")
        OSDS_UUID[$ID]=${OSD}
    done

  IDX=$(echo ${DEV} | grep -o '[[:digit:]]*' |head -1)
  # Physical nvme slot
  SLOTS[$IDX]=$(cat /sys/block/${DEV}/device/address | xargs -i{} lspci -v -s {} | grep "Physical Slot"| awk -F':' '{print $2}'|xargs)
  # VG
  VG=$(pvs --noheadings -o vg_name /dev/${DEV} 2>/dev/null|xargs)
  # LV path and LUKS crypt UUID
  OLDIFS=$IFS
  IFS=$'\n'
  for line in $(lvs --noheadings -o lv_path,lv_uuid,lv_name ${VG}); do
    NAME=$(echo $line| awk '{print $3}'|awk -F 'block-' '{print $2}')
    OSD=${OSDS_UUID[$NAME]}
    IDS[$OSD]=$NAME
    LUKS[$OSD]=$(echo $line| awk '{print $2}')
    LVS[$OSD]=$(echo $line| awk '{print $1}')
  done
  IFS=$OLDIFS
done

echo
for nvme in ${!SLOTS[@]}; do
    echo Nvme${nvme}:
    echo -e "\tSLOT: ${SLOTS[$nvme]}"
    echo -e "\tOSDS:${OSDS_BY_NVME[$nvme]}"
    for OSD in ${OSDS_BY_NVME[$nvme]}; do
        echo osd.${OSD}
        echo -e "\tIDS: ${IDS[$OSD]}"
        echo -e "\tLUKS: ${LUKS[$OSD]}"
        echo -e "\tLVS: ${LVS[$OSD]}"
    done
done
exit

echo
# Iterating on each disk
for nvme in ${!SLOTS[@]}; do
    # Powering off the disk
    echo Powering off nvme${nvme}
    (echo 0 > /sys/bus/pci/slots/${SLOTS[$nvme]}/power)
    sleep 5
    # Cleaning device mapper table
    for OSD in ${OSDS_BY_NVME[$nvme]}; do
        echo Cleaning ${LUKS[$OSD]}
        dmsetup remove ${LUKS[$OSD]}
        echo Cleaning ${LVS[$OSD]}
        dmsetup remove ${LVS[$OSD]}
    done
    sleep 1

    # Powering on teh disk
    echo Powering on nvme${nvme}
    (echo 1 > /sys/bus/pci/slots/${SLOTS[$nvme]}/power)
    sleep 5

    for OSD in ${OSDS_BY_NVME[$nvme]}; do
        echo Opening LUKS ${LUKS[$OSD]}
        (ceph config-key get dm-crypt/osd/${IDS[$OSD]}/luks | /usr/sbin/cryptsetup --key-file - --allow-discards luksOpen ${LVS[$OSD]} ${LUKS[$OSD]})

        echo Enabling osd.${OSD}
        chown -R ceph:ceph /var/lib/ceph/osd/ceph-${OSD}
        ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/mapper/${LUKS[$OSD]} --path /var/lib/ceph/osd/ceph-${OSD} --no-mon-config
        rm -f /var/lib/ceph/osd/ceph-${OSD}/block
        ln -snf /dev/mapper/${LUKS[$OSD]} /var/lib/ceph/osd/ceph-${OSD}/block
        DM=$(ls -l /dev/mapper/${LUKS[$OSD]} | awk -F'->' '{print $2}'| awk -F'/' '{print $2}')
        chown -R ceph:ceph /dev/${DM}
        chown -R ceph:ceph /var/lib/ceph/osd/ceph-${OSD}
        systemctl enable ceph-volume@lvm-${OSD}-${IDS[$OSD]}
        systemctl enable --runtime ceph-osd@${OSD}

        echo Restarting OSD service : ${OSD}
        systemctl start ceph-osd@${OSD}
	sleep 10
    done
    if (lsblk /dev/nvme${nvme}n1 &>/dev/null| grep crypt); then
        OUTPUT="${OUTPUT}\nOSD Repair Failed for NVME ${nvme}"
        OUTPUT="${OUTPUT}\nYou can retry this script !"
     else
        OUTPUT="${OUTPUT}\nOSD Repair for NVME ${nvme} OK :"
        OUTPUT="${OUTPUT}\n $(lsblk /dev/nvme${nvme}n1 --noheading)'"
	# Look for OSD crashes for 60 seconds and archive them.
	if [ $archive -eq 1 ] 
	then
	  for i in {1..6}
	  do
	    for OSD in ${OSDS_BY_NVME[$nvme]}
	    do
  	      archive_osd_crash ${OSD}
            done
	    sleep 10
          done
	fi
    fi
done

printf "$OUTPUT\n"
