Merge pull request #26 from 45Drives/ubm

UBM Tools and Rules
45Drives · Aug 2, 2023 · 6d8e4f1 · 6d8e4f1
2 parents 122e53f + ae49aa5
commit 6d8e4f1
Show file tree

Hide file tree

Showing 26 changed files with 403 additions and 22 deletions.
diff --git a/Makefile b/Makefile
@@ -1,27 +1,36 @@
 all:
 
 install:
-	mkdir -p $(DESTDIR)/opt/45drives/tools
-	mkdir -p $(DESTDIR)/etc/45drives/server_info
-	mkdir -p $(DESTDIR)/usr/bin
-	cp -a src/fakeroot/opt/45drives/tools/* $(DESTDIR)/opt/45drives/tools
+	mkdir -p "$(DESTDIR)/opt/45drives/tools"
+	mkdir -p "$(DESTDIR)/opt/45drives/ubm"
+	mkdir -p "$(DESTDIR)/etc/45drives/server_info"
+	mkdir -p "$(DESTDIR)/usr/bin"
+	mkdir -p "$(DESTDIR)/usr/lib/udev/rules.d"
+	cp -a tools/* "$(DESTDIR)/opt/45drives/tools"
+	install -m 755 -t "$(DESTDIR)/opt/45drives/ubm" ubm/ubm_id_disk
+	install -m 755 -t "$(DESTDIR)/opt/45drives/ubm" ubm/ubm_id_enc
+	install -m 755 -t "$(DESTDIR)/opt/45drives/ubm" ubm/ubm_patch_vdev_id_conf
+	install -m 644 -t "$(DESTDIR)/opt/45drives/ubm" ubm/ubm_funcs.sh
+	install -m 644 -t "$(DESTDIR)/opt/45drives/ubm" ubm/ubm_slot_name_map.txt
+	install -m 644 -t "$(DESTDIR)/usr/lib/udev/rules.d" ubm/67-ubm.rules
 ifdef TOOLS_VERSION
-	echo $(TOOLS_VERSION) > $(DESTDIR)/etc/45drives/server_info/tools_version
+	echo $(TOOLS_VERSION) > "$(DESTDIR)/etc/45drives/server_info/tools_version"
 endif
-	ln -sf /opt/45drives/tools/cephfs-dir-stats $(DESTDIR)/usr/bin/cephfs-dir-stats
-	ln -sf /opt/45drives/tools/dmap $(DESTDIR)/usr/bin/dmap
-	ln -sf /opt/45drives/tools/findosd $(DESTDIR)/usr/bin/findosd
-	ln -sf /opt/45drives/tools/lsdev $(DESTDIR)/usr/bin/lsdev
-	ln -sf /opt/45drives/tools/server_identifier $(DESTDIR)/usr/bin/server_identifier
-	ln -sf /opt/45drives/tools/zcreate $(DESTDIR)/usr/bin/zcreate
+	ln -sf /opt/45drives/tools/cephfs-dir-stats "$(DESTDIR)/usr/bin/cephfs-dir-stats"
+	ln -sf /opt/45drives/tools/dmap "$(DESTDIR)/usr/bin/dmap"
+	ln -sf /opt/45drives/tools/findosd "$(DESTDIR)/usr/bin/findosd"
+	ln -sf /opt/45drives/tools/lsdev "$(DESTDIR)/usr/bin/lsdev"
+	ln -sf /opt/45drives/tools/server_identifier "$(DESTDIR)/usr/bin/server_identifier"
+	ln -sf /opt/45drives/tools/zcreate "$(DESTDIR)/usr/bin/zcreate"
 
 uninstall:
-	rm -rf $(DESTDIR)/etc/45drives/server_info
-	rm -rf $(DESTDIR)/opt/45drives/tools
-	rm -f $(DESTDIR)/usr/bin/cephfs-dir-stats
-	rm -f $(DESTDIR)/usr/bin/dmap
-	rm -f $(DESTDIR)/usr/bin/findosd
-	rm -f $(DESTDIR)/usr/bin/lsdev
-	rm -f $(DESTDIR)/usr/bin/server_identifier
-	rm -f $(DESTDIR)/usr/bin/zcreate
-
+	rm -rf "$(DESTDIR)/etc/45drives/server_info"
+	rm -rf "$(DESTDIR)/opt/45drives/tools"
+	rm -rf "$(DESTDIR)/opt/45drives/ubm"
+	rm -f "$(DESTDIR)/usr/bin/cephfs-dir-stats"
+	rm -f "$(DESTDIR)/usr/bin/dmap"
+	rm -f "$(DESTDIR)/usr/bin/findosd"
+	rm -f "$(DESTDIR)/usr/bin/lsdev"
+	rm -f "$(DESTDIR)/usr/bin/server_identifier"
+	rm -f "$(DESTDIR)/usr/bin/zcreate"
+	rm -f "$(DESTDIR)/usr/lib/udev/rules.d/67-ubm.rules"
diff --git a/packaging/el7/main.spec b/packaging/el7/main.spec
@@ -48,11 +48,14 @@ fi
 
 %files
 %dir /opt/45drives/tools
+%dir /opt/45drives/ubm
 %dir /etc/45drives/server_info
 %defattr(-,root,root,-)
 /etc/45drives/server_info/*
 /opt/45drives/tools/*
+/opt/45drives/ubm/*
 %{_bindir}/*
+/usr/lib/udev/rules.d/*
 
 %changelog
 * Fri Jun 23 2023 Mark Hooper <mhooper@45drives.com> 2.2.3-1
@@ -267,4 +270,4 @@ fi
 - configured autopackaging for 45drives-tools on rocky and ubuntu
 - updated uninstall process to remove all files associated with 45drives-tools
 - updated makefile
-- dmap outputs full version in vdev_id.conf
+- dmap outputs full version in vdev_id.conf
diff --git a/packaging/el8/main.spec b/packaging/el8/main.spec
@@ -48,11 +48,14 @@ fi
 
 %files
 %dir /opt/45drives/tools
+%dir /opt/45drives/ubm
 %dir /etc/45drives/server_info
 %defattr(-,root,root,-)
 /etc/45drives/server_info/*
 /opt/45drives/tools/*
+/opt/45drives/ubm/*
 %{_bindir}/*
+/usr/lib/udev/rules.d/*
 
 %changelog
 * Fri Jun 23 2023 Mark Hooper <mhooper@45drives.com> 2.2.3-1
@@ -276,4 +279,4 @@ fi
 - configured autopackaging for 45drives-tools on rocky and ubuntu
 - updated uninstall process to remove all files associated with 45drives-tools
 - updated makefile
-- dmap outputs full version in vdev_id.conf
+- dmap outputs full version in vdev_id.conf
diff --git a/...fakeroot/opt/45drives/tools/68-vdev.rules → tools/68-vdev.rules b/...fakeroot/opt/45drives/tools/68-vdev.rules → tools/68-vdev.rules
@@ -2,6 +2,9 @@
 # /usr/lib/udev/rules.d/68-vdev.rules
 #
 
+# skip these rules if disk was handled by 67-ubm.rules
+ENV{DEVTYPE}=="disk", ENV{DISK_HANDLED_BY_UBM}=="1", GOTO="68-vdev_end"
+
 ENV{DEVTYPE}=="disk", IMPORT{program}="/usr/lib/udev/vdev_id_45drives -d %k -c /etc/vdev_id.conf"
 
 KERNEL=="*[!0-9]", ENV{SUBSYSTEM}=="block", ENV{ID_VDEV}=="?*", SYMLINK+="$env{ID_VDEV_PATH}"
@@ -11,3 +14,4 @@ KERNEL=="*[0-9]", ENV{SUBSYSTEM}=="block", ENV{DEVTYPE}=="partition", ENV{ID_VDE
 KERNEL=="dm-[0-9]*", ENV{SUBSYSTEM}=="block", ENV{ID_VDEV}=="?*", SYMLINK+="$env{ID_VDEV_PATH}"
 KERNEL=="dm-[0-9]*", ENV{SUBSYSTEM}=="block", ENV{ID_VDEV}=="?*", SYMLINK+="$env{ID_VDEV_ALT_PATH}"
 
+LABEL="68-vdev_end"
diff --git a/...akeroot/opt/45drives/tools/add-lv-tags.sh → tools/add-lv-tags.sh b/...akeroot/opt/45drives/tools/add-lv-tags.sh → tools/add-lv-tags.sh
diff --git a/...eroot/opt/45drives/tools/cephfs-dir-stats → tools/cephfs-dir-stats b/...eroot/opt/45drives/tools/cephfs-dir-stats → tools/cephfs-dir-stats
diff --git a/src/fakeroot/opt/45drives/tools/dmap → tools/dmap b/src/fakeroot/opt/45drives/tools/dmap → tools/dmap
diff --git a/src/fakeroot/opt/45drives/tools/findosd → tools/findosd b/src/fakeroot/opt/45drives/tools/findosd → tools/findosd
diff --git a/...t/opt/45drives/tools/generate-osd-vars.sh → tools/generate-osd-vars.sh b/...t/opt/45drives/tools/generate-osd-vars.sh → tools/generate-osd-vars.sh
diff --git a/src/fakeroot/opt/45drives/tools/loadtest → tools/loadtest b/src/fakeroot/opt/45drives/tools/loadtest → tools/loadtest
diff --git a/src/fakeroot/opt/45drives/tools/lsdev → tools/lsdev b/src/fakeroot/opt/45drives/tools/lsdev → tools/lsdev
diff --git a/...ot/opt/45drives/tools/modify-crush-map.sh → tools/modify-crush-map.sh b/...ot/opt/45drives/tools/modify-crush-map.sh → tools/modify-crush-map.sh
diff --git a/src/fakeroot/opt/45drives/tools/pci.ids → tools/pci.ids b/src/fakeroot/opt/45drives/tools/pci.ids → tools/pci.ids
diff --git a/...root/opt/45drives/tools/server_identifier → tools/server_identifier b/...root/opt/45drives/tools/server_identifier → tools/server_identifier
diff --git a/src/fakeroot/opt/45drives/tools/storcli2 → tools/storcli2 b/src/fakeroot/opt/45drives/tools/storcli2 → tools/storcli2
diff --git a/src/fakeroot/opt/45drives/tools/storcli64 → tools/storcli64 b/src/fakeroot/opt/45drives/tools/storcli64 → tools/storcli64
diff --git a/...akeroot/opt/45drives/tools/table_print.py → tools/table_print.py b/...akeroot/opt/45drives/tools/table_print.py → tools/table_print.py
diff --git a/...eroot/opt/45drives/tools/vdev_id_45drives → tools/vdev_id_45drives b/...eroot/opt/45drives/tools/vdev_id_45drives → tools/vdev_id_45drives
diff --git a/src/fakeroot/opt/45drives/tools/wipedev → tools/wipedev b/src/fakeroot/opt/45drives/tools/wipedev → tools/wipedev
diff --git a/src/fakeroot/opt/45drives/tools/zcreate → tools/zcreate b/src/fakeroot/opt/45drives/tools/zcreate → tools/zcreate
diff --git a/ubm/67-ubm.rules b/ubm/67-ubm.rules
@@ -0,0 +1,30 @@
+#
+# /lib/udev/rules.d/67-ubm.rules
+#
+# udev rule definitions for Universal Backplane Management via Broadcom 9600 series HBA
+#
+# part of 45drives-tools
+#
+# Authors
+# Josh Boudreau <jboudreau@45drives.com>
+
+################################
+# Disk rules
+################################
+
+SUBSYSTEMS=="scsi", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{UBM_SLOT_NAME}!="?*", IMPORT{program}="/opt/45drives/ubm/ubm_id_disk %k"
+SUBSYSTEMS=="scsi", SUBSYSTEM=="block", ENV{DEVTYPE}=="partition", ENV{UBM_SLOT_NAME}!="?*", IMPORT{program}="/opt/45drives/ubm/ubm_id_disk %k"
+
+# DISK_HANDLED_BY_UBM environment variable will make udev skip rules in 68-vdev.rules
+ENV{SUBSYSTEM}=="block", ENV{UBM_SLOT_NAME}=="?*", ENV{DISK_HANDLED_BY_UBM}="1"
+
+KERNEL=="*[!0-9]", ENV{SUBSYSTEM}=="block", ENV{UBM_SLOT_NAME}=="?*", SYMLINK+="disk/by-vdev/$env{UBM_SLOT_NAME}"
+KERNEL=="*[!0-9]", ENV{SUBSYSTEM}=="block", ENV{UBM_SLOT_NUM}=="?*", SYMLINK+="disk/by-slot-number/$env{UBM_SLOT_NUM}"
+KERNEL=="*[0-9]", ENV{SUBSYSTEM}=="block", ENV{DEVTYPE}=="partition", ENV{UBM_SLOT_NAME}=="?*", SYMLINK+="disk/by-vdev/$env{UBM_SLOT_NAME}-part%n"
+KERNEL=="*[0-9]", ENV{SUBSYSTEM}=="block", ENV{DEVTYPE}=="partition", ENV{UBM_SLOT_NUM}=="?*", SYMLINK+="disk/by-slot-number/$env{UBM_SLOT_NUM}-part%n"
+
+KERNEL=="*[!0-9]", ENV{SUBSYSTEM}=="block", ENV{ID_PATH}=="?*", ENV{UBM_SLOT_NAME}=="?*", RUN+="/opt/45drives/ubm/ubm_patch_vdev_id_conf $env{UBM_SLOT_NAME} $env{ID_PATH}"
+
+################################
+# Enclosure (backplane) rules
+################################
diff --git a/ubm/ubm_funcs.sh b/ubm/ubm_funcs.sh
@@ -0,0 +1,201 @@
+# ubm_funcs.sh - source to include ubm helper functions
+#
+# Part of 45drives-tools
+#
+# Authors
+# Josh Boudreau <jboudreau@45drives.com>
+
+# shellcheck shell=bash
+
+[ -z "$BASH" ] && echo "Must use bash" >&2 && exit 1
+
+[ "${BASH_SOURCE[0]}" -ef "$0" ] && echo "This script should only be sourced, not executed." >&2 && exit 1
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")"
+
+CACHE_DIR='/var/cache/45drives/ubm'
+
+SLOT_NAME_MAP_FILE="$SCRIPT_DIR/ubm_slot_name_map.txt"
+
+perror() {
+  local EXIT_CODE=$1
+  shift
+  echo "${BASH_SOURCE[0]}: ${FUNCNAME[1]}:" "$@" "($EXIT_CODE)" >&2
+  return "$EXIT_CODE"
+}
+
+die() {
+  local EXIT_CODE=$1
+  shift
+  echo "${BASH_SOURCE[0]}: ${FUNCNAME[1]}:" "$@" "($EXIT_CODE)" >&2
+  exit "$EXIT_CODE"
+}
+
+# will exit script on error
+get_map_key() {
+  if [[ -z "$UBM_MAP_KEY" ]]; then
+    if [[ ! -r "$CACHE_DIR/map_key" ]]; then
+      mkdir -p "$CACHE_DIR" >/dev/null
+      local LOOKUP_KEY
+      LOOKUP_KEY=$(
+        (
+          set -o pipefail
+          ipmitool fru | awk -F: '
+            BEGIN {
+              found_key = 0
+            }
+            $1 ~ "Product Name" {
+              found_key=1
+              key=toupper($2);
+              sub(/-(TURBO|BASE|ENHANCED).*$/, "", key);
+              sub(/\s+/, "", key);
+              print key;
+            }
+            END {
+              if (!found_key) {
+                print "map key lookup failed" > "/dev/stderr"
+                exit 1
+              }
+              exit 0
+            }
+            '
+        )
+      ) || die $? "Failed to get Product Name from FRU"
+      echo "$LOOKUP_KEY" >"$CACHE_DIR/map_key"
+    fi
+    UBM_MAP_KEY="$(<"$CACHE_DIR/map_key")"
+    export UBM_MAP_KEY
+  fi
+  echo "$UBM_MAP_KEY"
+}
+
+normalize_block_dev_name() {
+  local BLOCK_DEV_NAME=$1
+  BLOCK_DEV_NAME=${BLOCK_DEV_NAME#/dev/}
+  BLOCK_DEV_NAME=${BLOCK_DEV_NAME#/sys/block/}
+  # if its a partition:
+  local PARENT_DEV
+  if PARENT_DEV=$(lsblk -dno PKNAME "/dev/$BLOCK_DEV_NAME") && [[ -n "$PARENT_DEV" ]]; then
+    BLOCK_DEV_NAME="$PARENT_DEV"
+  fi
+  echo "$BLOCK_DEV_NAME"
+  return 0
+}
+
+block_dev_to_storcli_ctrlr() {
+  local BLOCK_DEV_NAME=$1
+  local BLOCK_DEV_SYS_PATH
+  BLOCK_DEV_SYS_PATH=$(realpath -- "/sys/block/$BLOCK_DEV_NAME") || perror $? "Failed to resolve $BLOCK_DEV_NAME sys path" || return $?
+  cat "$BLOCK_DEV_SYS_PATH"/../../../../scsi_host/host*/unique_id || perror $? "Failed to get $BLOCK_DEV_NAME storcli2 controller index"
+  return $?
+}
+
+slot_num_to_storcli_ctrlr() {
+  local SLOT_NUM=$1
+  (
+    set -o pipefail
+    /opt/45drives/tools/storcli2 /call/eall show all J | jq -re '
+[
+  .Controllers[] |
+  select( (."Command Status"."Status" == "Success") and ()  ) |
+  ."Response Data"."Drives List"[] |
+  select(
+    ."Drive Detailed Information"."OS Drive Name" | endswith("'"$BLOCK_DEV_NAME"'")
+  )
+] | if length == 1 then .[] else empty end |
+."Drive Information"."EID:Slt" | split(":")[1]
+'
+  ) || perror $? "Failed to get slot number for $BLOCK_DEV_NAME"
+  return $?
+}
+
+block_dev_to_slot_num() {
+  local BLOCK_DEV_NAME=$1
+  local CTRL_NUM=$2
+  [[ -n "$CTRL_NUM" ]] || CTRL_NUM=$(block_dev_to_storcli_ctrlr "$BLOCK_DEV_NAME") || return $?
+  (
+    set -o pipefail
+    /opt/45drives/tools/storcli2 /c"$CTRL_NUM"/eall/sall show all J | jq -re '
+[
+  .Controllers[] |
+  select(."Command Status"."Status" == "Success") |
+  ."Response Data"."Drives List"[] |
+  select(
+    ."Drive Detailed Information"."OS Drive Name" | endswith("'"$BLOCK_DEV_NAME"'")
+  )
+] | if length == 1 then .[] else empty end |
+."Drive Information"."EID:Slt" | split(":")[1]
+'
+  ) || perror $? "Failed to get slot number for $BLOCK_DEV_NAME"
+  return $?
+}
+
+slot_num_to_slot_name() {
+  local SLOT_NUM=$1
+  [[ $SLOT_NUM =~ ^[0-9]+$ ]] || perror 1 Invalid slot number: "$SLOT_NUM" || return $?
+  awk '
+  BEGIN {
+    found_style = 0
+  }
+  $1 == "'"$(get_map_key)"'" {
+    found_style = 1
+    print $'"$((2 + "$SLOT_NUM"))"'
+  }
+  END {
+    if (!found_style) {
+      print "map key lookup failed ('"$(get_map_key)"')" > "/dev/stderr"
+      exit 1
+    }
+    exit 0
+  }
+  ' "$SLOT_NAME_MAP_FILE" || perror $? "Failed to lookup slot name for slot $SLOT_NUM"
+  return $?
+}
+
+slot_name_to_slot_num() {
+  local SLOT_NAME=$1
+  awk '
+  BEGIN {
+    found_style = 0
+    found_slot = 0
+  }
+  $1 == "'"$(get_map_key)"'" {
+    found_style = 1
+    for (i = 2; i<=NF; ++i) {
+      if ($i == "'"$SLOT_NAME"'") {
+        found_slot = 1
+        print i - 2
+      }
+    }
+  }
+  END {
+    if (!found_style) {
+      print "map key lookup failed" > "/dev/stderr"
+      exit 1
+    }
+    if (!found_slot) {
+      print "slot column lookup failed" > "/dev/stderr"
+      exit 1
+    }
+    exit 0
+  }
+  ' "$SLOT_NAME_MAP_FILE"
+  return $?
+}
+
+block_dev_to_slot_name() {
+  local BLOCK_DEV_NAME=$1
+  local CTRL_NUM=$2
+  [[ -n "$CTRL_NUM" ]] || CTRL_NUM=$(block_dev_to_storcli_ctrlr "$BLOCK_DEV_NAME") || return $?
+  local SLOT_NUM
+  SLOT_NUM=$(block_dev_to_slot_num "$BLOCK_DEV_NAME" "$CTRL_NUM") || return $?
+  slot_num_to_slot_name "$SLOT_NUM"
+}
+
+all_slot_names() {
+  (
+    set -o pipefail
+    grep "^$(get_map_key)" "$SLOT_NAME_MAP_FILE" | cut -d' ' -f2- | xargs printf '%s\n'
+  ) || perror $? "Failed to lookup all slot names"
+  return $?
+}
diff --git a/ubm/ubm_id_disk b/ubm/ubm_id_disk
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# /opt/45drives/ubm/ubm_id_disk %k
+#
+# Outputs env variable values for UBM_SLOT_NUM and UBM_SLOT_NAME
+#
+# UDEV rule example:
+#
+# ENV{DEVTYPE}=="disk", IMPORT{program}="/opt/45drives/ubm/ubm_id_disk %k"
+#
+# Part of 45drives-tools
+#
+# Authors
+# Josh Boudreau <jboudreau@45drives.com>
+
+[ -z "$BASH" ] && echo "Must use bash" >&2 && exit 1
+
+BLOCK_DEV_KERNEL_NAME=$1
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")"
+# shellcheck source=./ubm_funcs.sh
+source "$SCRIPT_DIR/ubm_funcs.sh" || { RESULT=$?; echo Failed to source "$SCRIPT_DIR/ubm_funcs.sh" >&2; exit $RESULT; }
+
+BLOCK_DEV_KERNEL_NAME=$(normalize_block_dev_name "$BLOCK_DEV_KERNEL_NAME") || exit 0
+
+CTRL_NUM=$(block_dev_to_storcli_ctrlr "$BLOCK_DEV_KERNEL_NAME") || exit 0
+echo "MPI3MR_CTRL_NUM=$CTRL_NUM"
+
+SLOT_NUM=$(block_dev_to_slot_num "$BLOCK_DEV_KERNEL_NAME") || exit 0
+echo "UBM_SLOT_NUM=$SLOT_NUM"
+
+SLOT_NAME=$(slot_num_to_slot_name "$SLOT_NUM") || exit 0
+echo "UBM_SLOT_NAME=$SLOT_NAME"
diff --git a/ubm/ubm_id_enc b/ubm/ubm_id_enc
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#
+# /opt/45drives/ubm/ubm_id_enc %k
+#
+# Outputs env variable values for UBM_BP_NUM
+#
+# Part of 45drives-tools
+#
+# Authors
+# Josh Boudreau <jboudreau@45drives.com>
+
+# TODO