From b3442b646414e3216f7fdbf41920c7fe7973d324 Mon Sep 17 00:00:00 2001 From: ibmmqmet Date: Thu, 29 Feb 2024 15:44:30 +0000 Subject: [PATCH] Update for MQ 9.3.5 --- CHANGELOG.md | 4 +- DEPRECATIONS.md | 3 +- Dockerfile | 4 +- go.mod | 3 +- ibmmq/cmqc_aix.go | 9 +- ibmmq/cmqc_darwin.go | 9 +- ibmmq/cmqc_linux_amd64.go | 9 +- ibmmq/cmqc_linux_arm64.go | 9 +- ibmmq/cmqc_linux_ppc64le.go | 9 +- ibmmq/cmqc_linux_s390x.go | 9 +- ibmmq/cmqc_windows.go | 9 +- ibmmq/mqiPCF.go | 43 +++- ibmmq/mqistr.go | 4 +- mqmetric/channel.go | 14 +- mqmetric/channelamqp.go | 6 +- mqmetric/discover.go | 68 +----- mqmetric/globals.go | 2 +- mqmetric/mapping.go | 343 +++++++++++++++++++++++++++++++ mqmetric/metrics.txt | 14 +- mqmetric/queue.go | 4 +- mqmetric/status.go | 8 +- mqmetric/sub.go | 2 +- mqmetric/topic.go | 6 +- samples/amqsconn.go | 3 +- samples/amqsconntls.go | 3 +- samples/amqsjwt.go | 80 +++---- samples/amqspcf.go | 2 +- samples/runSample.deb.Dockerfile | 6 +- samples/runSample.gomod | 2 +- samples/runSample.ubi.Dockerfile | 6 +- 30 files changed, 516 insertions(+), 177 deletions(-) create mode 100644 mqmetric/mapping.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d3513a1..d11d58e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,11 @@ # Changelog Newest updates are at the top of this file. -## xxx xx 2023 - v5.5.x +## Feb 29 2024 - v5.5.4 +- Update for MQ 9.3.5 - ibmmq - Add simple tracing for MQI calls (MQIGO_TRACE env var) - samples - Add sample obtaining and using a JWT token +- Make Go 1.18 baseline compiler ## Nov 13 2023 - v5.5.3 - mqmetric - MQ 9.3 permits resource subscriptions for queues with '/' in name diff --git a/DEPRECATIONS.md b/DEPRECATIONS.md index e4d40d7..3b63bd7 100644 --- a/DEPRECATIONS.md +++ b/DEPRECATIONS.md @@ -7,8 +7,7 @@ Removal of function will only happen on a major version change. **Note:** There is no date currently planned for a new major release. ## In next minor version -The compiler will be set to use Go 1.17 at minimum from -the +build lines in the directives +Go compiler version moved forward if necessary ## In next major version The following interfaces are planned to be removed: diff --git a/Dockerfile b/Dockerfile index a550f44..2d20224 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG BASE_IMAGE=ubuntu:18.04 FROM $BASE_IMAGE ARG GOPATH_ARG="/go" -ARG GOVERSION=1.17 +ARG GOVERSION=1.18 ARG GOARCH=amd64 ARG MQARCH=X64 @@ -60,7 +60,7 @@ RUN mkdir -p $GOPATH/src $GOPATH/bin $GOPATH/pkg \ # Location of the downloadable MQ client package \ ENV RDURL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist" \ RDTAR="IBM-MQC-Redist-Linux${MQARCH}.tar.gz" \ - VRMF=9.3.4.0 + VRMF=9.3.5.0 # Install the MQ client from the Redistributable package. This also contains the # header files we need to compile against. Setup the subset of the package diff --git a/go.mod b/go.mod index b1e90f7..f09dc3f 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,4 @@ module github.com/ibm-messaging/mq-golang/v5 -go 1.16 +go 1.18 + diff --git a/ibmmq/cmqc_aix.go b/ibmmq/cmqc_aix.go index bc79c38..8f73a8a 100644 --- a/ibmmq/cmqc_aix.go +++ b/ibmmq/cmqc_aix.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/cmqc_darwin.go b/ibmmq/cmqc_darwin.go index a639067..dffd0d5 100644 --- a/ibmmq/cmqc_darwin.go +++ b/ibmmq/cmqc_darwin.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/cmqc_linux_amd64.go b/ibmmq/cmqc_linux_amd64.go index 644ca12..373d2b9 100644 --- a/ibmmq/cmqc_linux_amd64.go +++ b/ibmmq/cmqc_linux_amd64.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/cmqc_linux_arm64.go b/ibmmq/cmqc_linux_arm64.go index 241a0a4..3f0824e 100644 --- a/ibmmq/cmqc_linux_arm64.go +++ b/ibmmq/cmqc_linux_arm64.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/cmqc_linux_ppc64le.go b/ibmmq/cmqc_linux_ppc64le.go index 2c8e9f5..3e1ad4b 100644 --- a/ibmmq/cmqc_linux_ppc64le.go +++ b/ibmmq/cmqc_linux_ppc64le.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/cmqc_linux_s390x.go b/ibmmq/cmqc_linux_s390x.go index 2bbd827..a96f347 100644 --- a/ibmmq/cmqc_linux_s390x.go +++ b/ibmmq/cmqc_linux_s390x.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/cmqc_windows.go b/ibmmq/cmqc_windows.go index 6cc0551..5301050 100644 --- a/ibmmq/cmqc_windows.go +++ b/ibmmq/cmqc_windows.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,8 +30,8 @@ package ibmmq **************************************************************** * * -* Generated on: 9/27/23 11:53 AM -* Build Level: p934-L230927 +* Generated on: 2/1/24 12:04 PM +* Build Level: p935-L240201 * Build Type: Production * */ @@ -1032,7 +1032,7 @@ const ( MQCMDI_SEC_SIGNOFF_ERROR int32 = 17 MQCMDI_SEC_TIMER_ZERO int32 = 14 MQCMDI_SEC_UPPERCASE int32 = 21 - MQCMDL_CURRENT_LEVEL int32 = 934 + MQCMDL_CURRENT_LEVEL int32 = 935 MQCMDL_LEVEL_1 int32 = 100 MQCMDL_LEVEL_101 int32 = 101 MQCMDL_LEVEL_110 int32 = 110 @@ -1084,6 +1084,7 @@ const ( MQCMDL_LEVEL_932 int32 = 932 MQCMDL_LEVEL_933 int32 = 933 MQCMDL_LEVEL_934 int32 = 934 + MQCMDL_LEVEL_935 int32 = 935 MQCMD_ACCOUNTING_MQI int32 = 167 MQCMD_ACCOUNTING_Q int32 = 168 MQCMD_ACTIVITY_MSG int32 = 69 diff --git a/ibmmq/mqiPCF.go b/ibmmq/mqiPCF.go index afff2d1..d339993 100644 --- a/ibmmq/mqiPCF.go +++ b/ibmmq/mqiPCF.go @@ -1,7 +1,7 @@ package ibmmq /* - Copyright (c) IBM Corporation 2016,2019 + Copyright (c) IBM Corporation 2016,2024 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -62,6 +62,7 @@ type MQEPH struct { Format string Control int32 Flags int32 + Cfh MQCFH } /* @@ -105,12 +106,22 @@ NewMQEPH returns a PCF Embedded Header structure with correct initialisation func NewMQEPH() *MQEPH { eph := new(MQEPH) eph.StrucLength = C.MQEPH_STRUC_LENGTH_FIXED - eph.Version = C.MQCFH_VERSION_1 + eph.Version = C.MQEPH_VERSION_1 eph.Encoding = 0 eph.CodedCharSetId = C.MQCCSI_UNDEFINED eph.Format = C.MQFMT_NONE eph.Flags = C.MQEPH_NONE + eph.Cfh.Type = C.MQCFT_NONE + eph.Cfh.StrucLength = C.MQCFH_STRUC_LENGTH + eph.Cfh.Version = C.MQCFH_VERSION_3 + eph.Cfh.Command = C.MQCMD_NONE + eph.Cfh.MsgSeqNumber = 1 + eph.Cfh.Control = C.MQCFC_LAST + eph.Cfh.CompCode = C.MQCC_OK + eph.Cfh.Reason = C.MQRC_NONE + eph.Cfh.ParameterCount = 0 + return eph } @@ -144,6 +155,34 @@ func (cfh *MQCFH) Bytes() []byte { return buf } +func (eph *MQEPH) Bytes() []byte { + + // There's no constant defining the length of just the "EPH" wrapper. + // The STRUC_LENGTH_FIXED includes the CFH length. So we have to start + // by calculating it + buf := make([]byte, C.MQEPH_STRUC_LENGTH_FIXED-C.MQCFH_STRUC_LENGTH) + offset := 0 + + copy(buf[offset:], "EPH ") + offset += 4 + endian.PutUint32(buf[offset:], uint32(eph.Version)) + offset += 4 + endian.PutUint32(buf[offset:], uint32(eph.StrucLength)) + offset += 4 + endian.PutUint32(buf[offset:], uint32(eph.Encoding)) + offset += 4 + endian.PutUint32(buf[offset:], uint32(eph.CodedCharSetId)) + offset += 4 + copy(buf[offset:], (eph.Format + space8)[0:8]) + offset += 8 + endian.PutUint32(buf[offset:], uint32(eph.Flags)) + offset += 4 + + buf = append(buf, eph.Cfh.Bytes()...) + + return buf +} + /* Bytes serialises a PCFParameter into the C structure corresponding to its type. diff --git a/ibmmq/mqistr.go b/ibmmq/mqistr.go index 6a719c2..1d0d431 100644 --- a/ibmmq/mqistr.go +++ b/ibmmq/mqistr.go @@ -14,7 +14,7 @@ package ibmmq * for each value; those can be found in other header files such as * cmqc.h. **************************************************************** -* Copyright (c) IBM Corporation 1993, 2023 +* Copyright (c) IBM Corporation 1993, 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1461,6 +1461,8 @@ func MQItoString(class string, value int) string { s = "MQCMDL_LEVEL_933" case 934: s = "MQCMDL_LEVEL_934" + case 935: + s = "MQCMDL_LEVEL_935" default: s = "" } diff --git a/mqmetric/channel.go b/mqmetric/channel.go index 5e6ae75..0e76645 100644 --- a/mqmetric/channel.go +++ b/mqmetric/channel.go @@ -108,22 +108,22 @@ func ChannelInitAttributes() { // These are the integer status fields that are of interest attr = ATTR_CHL_MESSAGES st.Attributes[attr] = newStatusAttribute(attr, "Messages (API Calls for SVRCONN)", ibmmq.MQIACH_MSGS) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_BYTES_SENT st.Attributes[attr] = newStatusAttribute(attr, "Bytes sent", ibmmq.MQIACH_BYTES_SENT) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_BYTES_RCVD st.Attributes[attr] = newStatusAttribute(attr, "Bytes rcvd", ibmmq.MQIACH_BYTES_RCVD) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_BUFFERS_SENT st.Attributes[attr] = newStatusAttribute(attr, "Buffers sent", ibmmq.MQIACH_BUFFERS_SENT) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_BUFFERS_RCVD st.Attributes[attr] = newStatusAttribute(attr, "Buffers rcvd", ibmmq.MQIACH_BUFFERS_RCVD) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_BATCHES st.Attributes[attr] = newStatusAttribute(attr, "Completed Batches", ibmmq.MQIACH_BATCHES) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values // This is decoded by MQCHS_* values attr = ATTR_CHL_STATUS @@ -243,7 +243,7 @@ func CollectChannelStatus(patterns string) error { // Need to clean out the prevValues elements to stop short-lived channels // building up in the map for a, _ := range st.Attributes { - if st.Attributes[a].delta { + if st.Attributes[a].Delta { m := st.Attributes[a].prevValues for key, _ := range m { if _, ok := os.objectSeen[key]; ok { diff --git a/mqmetric/channelamqp.go b/mqmetric/channelamqp.go index 5ccaad7..756cad7 100644 --- a/mqmetric/channelamqp.go +++ b/mqmetric/channelamqp.go @@ -80,10 +80,10 @@ func ChannelAMQPInitAttributes() { // These are the integer status fields that are of interest attr = ATTR_CHL_AMQP_MESSAGES_RECEIVED st.Attributes[attr] = newStatusAttribute(attr, "Messages Received", ibmmq.MQIACH_MSGS_RCVD) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_AMQP_MESSAGES_SENT st.Attributes[attr] = newStatusAttribute(attr, "Messages Sent", ibmmq.MQIACH_MSGS_SENT) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_CHL_AMQP_CONNECTIONS st.Attributes[attr] = newStatusAttribute(attr, "Connections", ibmmq.MQIACF_CONNECTION_COUNT) @@ -166,7 +166,7 @@ func CollectAMQPChannelStatus(patterns string) error { // Need to clean out the prevValues elements to stop short-lived channels // building up in the map for a, _ := range st.Attributes { - if st.Attributes[a].delta { + if st.Attributes[a].Delta { m := st.Attributes[a].prevValues for key, _ := range m { if _, ok := os.objectSeen[key]; ok { diff --git a/mqmetric/discover.go b/mqmetric/discover.go index 08fd631..065f2fa 100644 --- a/mqmetric/discover.go +++ b/mqmetric/discover.go @@ -6,7 +6,7 @@ storage mechanisms including Prometheus and InfluxDB. package mqmetric /* - Copyright (c) IBM Corporation 2016, 2023 + Copyright (c) IBM Corporation 2016, 2024 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,7 +36,6 @@ import ( "bufio" "fmt" "os" - "regexp" "strings" "unicode/utf8" @@ -1307,71 +1306,6 @@ func parsePCFResponse(buf []byte) ([]*ibmmq.PCFParameter, bool) { return elemList, rc } -/* -Need to turn the "friendly" name of each element into something -that is suitable for metric names. - -Should also have consistency of units (always use seconds, -bytes etc), and organisation of the elements of the name (units last) - -While we can't change the MQ-generated descriptions for its statistics, -we can reformat most of them heuristically here. -*/ -func formatDescription(elem *MonElement) string { - s := elem.Description - s = strings.Replace(s, " ", "_", -1) - s = strings.Replace(s, "/", "_", -1) - s = strings.Replace(s, "-", "_", -1) - - /* Make sure we don't have multiple underscores */ - multiunder := regexp.MustCompile("__*") - s = multiunder.ReplaceAllLiteralString(s, "_") - - /* make it all lowercase. Not essential, but looks better */ - s = strings.ToLower(s) - - /* Remove all cases of bytes, seconds, count or percentage (we add them back in later) */ - s = strings.Replace(s, "_count", "", -1) - s = strings.Replace(s, "_bytes", "", -1) - s = strings.Replace(s, "_byte", "", -1) - s = strings.Replace(s, "_seconds", "", -1) - s = strings.Replace(s, "_second", "", -1) - s = strings.Replace(s, "_percentage", "", -1) - - // Switch round a couple of specific names - s = strings.Replace(s, "messages_expired", "expired_messages", -1) - - // Add the unit at end - switch elem.Datatype { - case ibmmq.MQIAMO_MONITOR_PERCENT, ibmmq.MQIAMO_MONITOR_HUNDREDTHS: - s = s + "_percentage" - case ibmmq.MQIAMO_MONITOR_MB, ibmmq.MQIAMO_MONITOR_GB: - s = s + "_bytes" - case ibmmq.MQIAMO_MONITOR_MICROSEC: - s = s + "_seconds" - default: - if strings.Contains(s, "_total") { - /* If we specify it is a total in description put that at the end */ - s = strings.Replace(s, "_total", "", -1) - s = s + "_total" - } else if strings.Contains(s, "log_") { - /* Weird case where the log datatype is not MB or GB but should be bytes */ - s = s + "_bytes" - } - - // There are some metrics that have both "count" and "byte count" in - // the descriptions. They were getting mapped to the same string, so - // we have to ensure uniqueness. - if strings.Contains(elem.Description, "byte count") { - s = s + "_bytes" - } else if strings.HasSuffix(elem.Description, " count") && !strings.Contains(s, "_count") { - s = s + "_count" - } - } - logTrace(" [%s] in:%s out:%s", "formatDescription", elem.Description, s) - return s -} - /* ReadPatterns is called during the initial configuration step to read a file containing object name patterns if they are not explicitly given diff --git a/mqmetric/globals.go b/mqmetric/globals.go index f717155..7e0a9ee 100644 --- a/mqmetric/globals.go +++ b/mqmetric/globals.go @@ -1,7 +1,7 @@ package mqmetric /* - Copyright (c) IBM Corporation 2016, 2023 + Copyright (c) IBM Corporation 2016, 2024 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/mqmetric/mapping.go b/mqmetric/mapping.go new file mode 100644 index 0000000..7ab29bf --- /dev/null +++ b/mqmetric/mapping.go @@ -0,0 +1,343 @@ +/* +Package mqmetric contains a set of routines common to several +commands used to export MQ metrics to different backend +storage mechanisms including Prometheus and InfluxDB. +*/ +package mqmetric + +/* + Copyright (c) IBM Corporation 2016, 2024 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Contributors: + Mark Taylor - Initial Contribution +*/ + +/* +Need to turn the "friendly" name of each element into something +that is suitable for metric names. + +Should also have consistency of units (always use seconds, +bytes etc), and organisation of the elements of the name (units last) + +While we can't change the MQ-generated descriptions for its statistics, +we can reformat most of them heuristically here. +*/ +import ( + "os" + "regexp" + "strings" + + "github.com/ibm-messaging/mq-golang/v5/ibmmq" +) + +var ( + mHeur = make(map[string]string) + mManual = make(map[string]string) + + mapsFilled = false + + UseManualMetricMaps = false // May move to a config option at some point + +) + +// Convert the description of the resource publication element into a metric name +// This has always been done using a heuristic algorithm, but we may want to change some of them +// to a format preferred by the backend (eg OpenTelemetry). +func formatDescription(elem *MonElement) string { + s := "" + + if !mapsFilled { + // Have to opt into using the non-heuristic maps for now + if os.Getenv("IBMMQ_MANUAL_METRIC_MAPS") != "" { + UseManualMetricMaps = true + } else { + UseManualMetricMaps = false + } + fillMapManual() + fillMapHeur() + + mapsFilled = true + } + + if UseManualMetricMaps { + s = formatDescriptionManual(elem.Description) + } + if s == "" { + s = FormatDescriptionHeuristic(elem, true) + } + return s +} + +func formatDescriptionManual(s string) string { + desc := strings.ReplaceAll(s, " ", "_") + desc = strings.ToLower(desc) + + // Is there an overriding metric name + if s, ok := mManual[desc]; ok { + return s + } else { + return "" + } +} + +// This function is exported so it can be called during the build/test process for automatic generation +// of some of the mHeur map elements. +func FormatDescriptionHeuristic(elem *MonElement, useMap bool) string { + + // The map has been generated once by hand and we will try to use it. + desc := strings.ReplaceAll(elem.Description, " ", "_") + desc = strings.ToLower(desc) + + if useMap { + if s, ok := mHeur[desc]; ok { + return s + } else { + logWarn("Element %s does not have a defined metric name", elem.Description) + } + } + + // If that fails, we go through generating the metric name using this set of + // rules. + s := elem.Description + s = strings.Replace(s, " ", "_", -1) + s = strings.Replace(s, "/", "_", -1) + s = strings.Replace(s, "-", "_", -1) + + /* Make sure we don't have multiple underscores */ + multiunder := regexp.MustCompile("__*") + s = multiunder.ReplaceAllLiteralString(s, "_") + + /* make it all lowercase. Not essential, but looks better */ + s = strings.ToLower(s) + + /* Remove all cases of bytes, seconds, count or percentage (we add them back in later) */ + s = strings.Replace(s, "_count", "", -1) + s = strings.Replace(s, "_bytes", "", -1) + s = strings.Replace(s, "_byte", "", -1) + s = strings.Replace(s, "_seconds", "", -1) + s = strings.Replace(s, "_second", "", -1) + s = strings.Replace(s, "_percentage", "", -1) + + // Switch round a couple of specific names + s = strings.Replace(s, "messages_expired", "expired_messages", -1) + + // Add the unit at end + switch elem.Datatype { + case ibmmq.MQIAMO_MONITOR_PERCENT, ibmmq.MQIAMO_MONITOR_HUNDREDTHS: + s = s + "_percentage" + case ibmmq.MQIAMO_MONITOR_MB, ibmmq.MQIAMO_MONITOR_GB: + s = s + "_bytes" + case ibmmq.MQIAMO_MONITOR_MICROSEC: + s = s + "_seconds" + default: + if strings.Contains(s, "_total") { + /* If we specify it is a total in description put that at the end */ + s = strings.Replace(s, "_total", "", -1) + s = s + "_total" + } else if strings.Contains(s, "log_") { + /* Weird case where the log datatype is not MB or GB but should be bytes */ + s = s + "_bytes" + } + + // There are some metrics that have both "count" and "byte count" in + // the descriptions. They were getting mapped to the same string, so + // we have to ensure uniqueness. + if strings.Contains(elem.Description, "byte count") { + s = s + "_bytes" + } else if strings.HasSuffix(elem.Description, " count") && !strings.Contains(s, "_count") { + s = s + "_count" + } + } + + logTrace(" [%s] in:%s out:%s", "formatDescription", elem.Description, s) + + return s +} + +// These are the original heuristically-derived metric names. This was built from running +// the code once and capturing info from traces. Any new metrics should show up from tools run +// during the release process. +func fillMapHeur() { + // Class: CPU + mHeur["user_cpu_time_percentage"] = "user_cpu_time_percentage" + mHeur["system_cpu_time_percentage"] = "system_cpu_time_percentage" + mHeur["cpu_load_-_one_minute_average"] = "cpu_load_one_minute_average_percentage" + mHeur["cpu_load_-_five_minute_average"] = "cpu_load_five_minute_average_percentage" + mHeur["cpu_load_-_fifteen_minute_average"] = "cpu_load_fifteen_minute_average_percentage" + mHeur["ram_free_percentage"] = "ram_free_percentage" + mHeur["ram_total_bytes"] = "ram_total_bytes" + mHeur["user_cpu_time_-_percentage_estimate_for_queue_manager"] = "user_cpu_time_estimate_for_queue_manager_percentage" + mHeur["system_cpu_time_-_percentage_estimate_for_queue_manager"] = "system_cpu_time_estimate_for_queue_manager_percentage" + mHeur["ram_total_bytes_-_estimate_for_queue_manager"] = "ram_total_estimate_for_queue_manager_bytes" + + // Class: Disk + mHeur["mq_trace_file_system_-_bytes_in_use"] = "mq_trace_file_system_in_use_bytes" + mHeur["mq_trace_file_system_-_free_space"] = "mq_trace_file_system_free_space_percentage" + mHeur["mq_errors_file_system_-_bytes_in_use"] = "mq_errors_file_system_in_use_bytes" + mHeur["mq_errors_file_system_-_free_space"] = "mq_errors_file_system_free_space_percentage" + mHeur["mq_fdc_file_count"] = "mq_fdc_file_count" + mHeur["queue_manager_file_system_-_bytes_in_use"] = "queue_manager_file_system_in_use_bytes" + mHeur["queue_manager_file_system_-_free_space"] = "queue_manager_file_system_free_space_percentage" + mHeur["log_-_bytes_in_use"] = "log_in_use_bytes" + mHeur["log_-_bytes_max"] = "log_max_bytes" + mHeur["log_file_system_-_bytes_in_use"] = "log_file_system_in_use_bytes" + mHeur["log_file_system_-_bytes_max"] = "log_file_system_max_bytes" + mHeur["log_-_physical_bytes_written"] = "log_physical_written_bytes" + mHeur["log_-_logical_bytes_written"] = "log_logical_written_bytes" + mHeur["log_-_write_latency"] = "log_write_latency_seconds" + mHeur["log_-_current_primary_space_in_use"] = "log_current_primary_space_in_use_percentage" + mHeur["log_-_workload_primary_space_utilization"] = "log_workload_primary_space_utilization_percentage" + mHeur["log_-_bytes_required_for_media_recovery"] = "log_required_for_media_recovery_bytes" + mHeur["log_-_bytes_occupied_by_reusable_extents"] = "log_occupied_by_reusable_extents_bytes" + mHeur["log_-_bytes_occupied_by_extents_waiting_to_be_archived"] = "log_occupied_by_extents_waiting_to_be_archived_bytes" + mHeur["log_-_write_size"] = "log_write_size_bytes" + + mHeur["appliance_data_-_bytes_in_use"] = "appliance_data_in_use_bytes" + mHeur["appliance_data_-_free_space"] = "appliance_data_free_space_percentage" + mHeur["system_volume_-_bytes_in_use"] = "system_volume_in_use_bytes" + mHeur["system_volume_-_free_space"] = "system_volume_free_space_percentage" + + // Class: STATQ and STATMQI + mHeur["mqinq_count"] = "mqinq_count" + mHeur["failed_mqinq_count"] = "failed_mqinq_count" + mHeur["mqset_count"] = "mqset_count" + mHeur["failed_mqset_count"] = "failed_mqset_count" + mHeur["interval_total_mqput/mqput1_count"] = "interval_mqput_mqput1_total_count" + mHeur["interval_total_mqput/mqput1_byte_count"] = "interval_mqput_mqput1_total_bytes" + mHeur["non-persistent_message_mqput_count"] = "non_persistent_message_mqput_count" + mHeur["persistent_message_mqput_count"] = "persistent_message_mqput_count" + mHeur["failed_mqput_count"] = "failed_mqput_count" + mHeur["non-persistent_message_mqput1_count"] = "non_persistent_message_mqput1_count" + mHeur["persistent_message_mqput1_count"] = "persistent_message_mqput1_count" + mHeur["failed_mqput1_count"] = "failed_mqput1_count" + mHeur["put_non-persistent_messages_-_byte_count"] = "put_non_persistent_messages_bytes" + mHeur["put_persistent_messages_-_byte_count"] = "put_persistent_messages_bytes" + mHeur["mqstat_count"] = "mqstat_count" + mHeur["interval_total_destructive_get-_count"] = "interval_destructive_get_total_count" + mHeur["interval_total_destructive_get_-_byte_count"] = "interval_destructive_get_total_bytes" + mHeur["non-persistent_message_destructive_get_-_count"] = "non_persistent_message_destructive_get_count" + mHeur["persistent_message_destructive_get_-_count"] = "persistent_message_destructive_get_count" + mHeur["failed_mqget_-_count"] = "failed_mqget_count" + mHeur["got_non-persistent_messages_-_byte_count"] = "got_non_persistent_messages_bytes" + mHeur["got_persistent_messages_-_byte_count"] = "got_persistent_messages_bytes" + mHeur["non-persistent_message_browse_-_count"] = "non_persistent_message_browse_count" + mHeur["persistent_message_browse_-_count"] = "persistent_message_browse_count" + mHeur["failed_browse_count"] = "failed_browse_count" + mHeur["non-persistent_message_browse_-_byte_count"] = "non_persistent_message_browse_bytes" + mHeur["persistent_message_browse_-_byte_count"] = "persistent_message_browse_bytes" + mHeur["expired_message_count"] = "expired_message_count" + mHeur["purged_queue_count"] = "purged_queue_count" + mHeur["mqcb_count"] = "mqcb_count" + mHeur["failed_mqcb_count"] = "failed_mqcb_count" + mHeur["mqctl_count"] = "mqctl_count" + mHeur["commit_count"] = "commit_count" + mHeur["rollback_count"] = "rollback_count" + mHeur["create_durable_subscription_count"] = "create_durable_subscription_count" + mHeur["alter_durable_subscription_count"] = "alter_durable_subscription_count" + mHeur["resume_durable_subscription_count"] = "resume_durable_subscription_count" + mHeur["create_non-durable_subscription_count"] = "create_non_durable_subscription_count" + mHeur["failed_create/alter/resume_subscription_count"] = "failed_create_alter_resume_subscription_count" + mHeur["delete_durable_subscription_count"] = "delete_durable_subscription_count" + mHeur["delete_non-durable_subscription_count"] = "delete_non_durable_subscription_count" + mHeur["subscription_delete_failure_count"] = "subscription_delete_failure_count" + mHeur["mqsubrq_count"] = "mqsubrq_count" + mHeur["failed_mqsubrq_count"] = "failed_mqsubrq_count" + mHeur["durable_subscriber_-_high_water_mark"] = "durable_subscriber_high_water_mark" + mHeur["durable_subscriber_-_low_water_mark"] = "durable_subscriber_low_water_mark" + mHeur["non-durable_subscriber_-_high_water_mark"] = "non_durable_subscriber_high_water_mark" + mHeur["non-durable_subscriber_-_low_water_mark"] = "non_durable_subscriber_low_water_mark" + mHeur["topic_mqput/mqput1_interval_total"] = "topic_mqput_mqput1_interval_total" + mHeur["interval_total_topic_bytes_put"] = "interval_topic_put_total" + mHeur["published_to_subscribers_-_message_count"] = "published_to_subscribers_message_count" + mHeur["published_to_subscribers_-_byte_count"] = "published_to_subscribers_bytes" + mHeur["non-persistent_-_topic_mqput/mqput1_count"] = "non_persistent_topic_mqput_mqput1_count" + mHeur["persistent_-_topic_mqput/mqput1_count"] = "persistent_topic_mqput_mqput1_count" + mHeur["failed_topic_mqput/mqput1_count"] = "failed_topic_mqput_mqput1_count" + mHeur["mqconn/mqconnx_count"] = "mqconn_mqconnx_count" + mHeur["failed_mqconn/mqconnx_count"] = "failed_mqconn_mqconnx_count" + mHeur["concurrent_connections_-_high_water_mark"] = "concurrent_connections_high_water_mark" + mHeur["mqdisc_count"] = "mqdisc_count" + mHeur["mqopen_count"] = "mqopen_count" + mHeur["failed_mqopen_count"] = "failed_mqopen_count" + mHeur["mqclose_count"] = "mqclose_count" + mHeur["failed_mqclose_count"] = "failed_mqclose_count" + mHeur["mqput/mqput1_count"] = "mqput_mqput1_count" + mHeur["mqput_byte_count"] = "mqput_bytes" + mHeur["mqput_non-persistent_message_count"] = "mqput_non_persistent_message_count" + mHeur["mqput_persistent_message_count"] = "mqput_persistent_message_count" + mHeur["mqput1_non-persistent_message_count"] = "mqput1_non_persistent_message_count" + mHeur["mqput1_persistent_message_count"] = "mqput1_persistent_message_count" + mHeur["non-persistent_byte_count"] = "non_persistent_bytes" + mHeur["persistent_byte_count"] = "persistent_bytes" + mHeur["queue_avoided_puts"] = "queue_avoided_puts_percentage" + mHeur["queue_avoided_bytes"] = "queue_avoided_percentage" + mHeur["lock_contention"] = "lock_contention_percentage" + mHeur["rolled_back_mqput_count"] = "rolled_back_mqput_count" + mHeur["mqget_count"] = "mqget_count" + mHeur["mqget_byte_count"] = "mqget_bytes" + mHeur["destructive_mqget_non-persistent_message_count"] = "destructive_mqget_non_persistent_message_count" + mHeur["destructive_mqget_persistent_message_count"] = "destructive_mqget_persistent_message_count" + mHeur["destructive_mqget_non-persistent_byte_count"] = "destructive_mqget_non_persistent_bytes" + mHeur["destructive_mqget_persistent_byte_count"] = "destructive_mqget_persistent_bytes" + mHeur["mqget_browse_non-persistent_message_count"] = "mqget_browse_non_persistent_message_count" + mHeur["mqget_browse_persistent_message_count"] = "mqget_browse_persistent_message_count" + mHeur["mqget_browse_non-persistent_byte_count"] = "mqget_browse_non_persistent_bytes" + mHeur["mqget_browse_persistent_byte_count"] = "mqget_browse_persistent_bytes" + mHeur["destructive_mqget_fails"] = "destructive_mqget_fails" + mHeur["destructive_mqget_fails_with_mqrc_no_msg_available"] = "destructive_mqget_fails_with_mqrc_no_msg_available" + mHeur["destructive_mqget_fails_with_mqrc_truncated_msg_failed"] = "destructive_mqget_fails_with_mqrc_truncated_msg_failed" + mHeur["mqget_browse_fails"] = "mqget_browse_fails" + mHeur["mqget_browse_fails_with_mqrc_no_msg_available"] = "mqget_browse_fails_with_mqrc_no_msg_available" + mHeur["mqget_browse_fails_with_mqrc_truncated_msg_failed"] = "mqget_browse_fails_with_mqrc_truncated_msg_failed" + mHeur["rolled_back_mqget_count"] = "rolled_back_mqget_count" + mHeur["messages_expired"] = "expired_messages" + mHeur["queue_purged_count"] = "queue_purged_count" + mHeur["average_queue_time"] = "average_queue_time_seconds" + mHeur["queue_depth"] = "queue_depth" + + // Class: Native HA + mHeur["synchronous_log_bytes_sent"] = "synchronous_log_sent_bytes" + mHeur["catch-up_log_bytes_sent"] = "catch_up_log_sent_bytes" + mHeur["log_write_average_acknowledgement_latency"] = "log_write_average_acknowledgement_latency" + mHeur["log_write_average_acknowledgement_size"] = "log_write_average_acknowledgement_size" + mHeur["backlog_bytes"] = "backlog_bytes" + mHeur["backlog_average_bytes"] = "backlog_average_bytes" +} + +// This map will contain only the additional elements where the heuristic version might not be suitable or +// match well-enough to some other implementations like the MQ Cloud package +func fillMapManual() { + mManual["ram_total_bytes"] = "ram_size_bytes" + + // Don't need the "mq_" on the front + mManual["mq_trace_file_system_-_bytes_in_use"] = "trace_file_system_in_use_bytes" + mManual["mq_trace_file_system_-_free_space"] = "trace_file_system_free_space_percentage" + mManual["mq_errors_file_system_-_bytes_in_use"] = "errors_file_system_in_use_bytes" + mManual["mq_errors_file_system_-_free_space"] = "errors_file_system_free_space_percentage" + mManual["mq_fdc_file_count"] = "fdc_files" + + // Flip around some of the elements + mManual["create_durable_subscription_count"] = "durable_subscription_create_count" + mManual["alter_durable_subscription_count"] = "durable_subscription_alter_count" + mManual["resume_durable_subscription_count"] = "durable_subscription_resume_count" + mManual["delete_durable_subscription_count"] = "durable_subscription_delete_count" + + mManual["create_non-durable_subscription_count"] = "non_durable_subscription_create_count" + mManual["delete_non-durable_subscription_count"] = "non_durable_subscription_delete_count" + + mManual["failed_create/alter/resume_subscription_count"] = "failed_subscription_create_alter_resume_count" + mManual["subscription_delete_failure_count"] = "failed_subscription_delete_count" + +} diff --git a/mqmetric/metrics.txt b/mqmetric/metrics.txt index 36335c8..8ac919a 100644 --- a/mqmetric/metrics.txt +++ b/mqmetric/metrics.txt @@ -56,13 +56,13 @@ Class: DISK Log - bytes max Log file system - bytes in use Log file system - bytes max - Log - physical bytes written - Log - logical bytes written + Log - physical bytes written for the current interval X + Log - logical bytes written for the current interval Log - write latency uSec - Log - write size - Log - bytes occupied by extents waiting to be archived - Log - bytes required for media recovery - Log - bytes occupied by reusable extents + Log - write size X, also rolling average + Log - occupied by extents waiting to be archived X + Log - space in MB required for media recovery X + Log - space in MB occupied by reusable extents Log - current primary space in use Log - workload primary space utilization @@ -309,4 +309,4 @@ Class: bufferpool/pageset ATTR_PS_TOTAL : pages_total ATTR_PS_UNUSED : pages_unused -This page was automatically generated from the source tree on 2023-10-10 07:32:28 +This page was automatically generated from the source tree on 2024-02-22 11:01:40 diff --git a/mqmetric/queue.go b/mqmetric/queue.go index e59ff6c..43a7dd5 100644 --- a/mqmetric/queue.go +++ b/mqmetric/queue.go @@ -6,7 +6,7 @@ storage mechanisms including Prometheus and InfluxDB. package mqmetric /* - Copyright (c) IBM Corporation 2018,2023 + Copyright (c) IBM Corporation 2018,2024 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -269,7 +269,7 @@ func collectQueueStatus(pattern string, instanceType int32) error { parseQData(instanceType, cfh, buf) } } - logDebug("collectQueueStatus response count: %d", statusMsgCount) + //logDebug("collectQueueStatus response count: %d", statusMsgCount) traceExitErr("collectQueueStatus", 0, err) return err } diff --git a/mqmetric/status.go b/mqmetric/status.go index dec7106..12db520 100644 --- a/mqmetric/status.go +++ b/mqmetric/status.go @@ -44,9 +44,9 @@ type StatusAttribute struct { Description string MetricName string Pseudo bool + Delta bool pcfAttr int32 squash bool - delta bool index int Values map[string]*StatusValue prevValues map[string]int64 @@ -69,9 +69,9 @@ func newStatusAttribute(n string, d string, p int32) *StatusAttribute { s := new(StatusAttribute) s.MetricName = n s.Description = d + s.Delta = false s.pcfAttr = p s.squash = false - s.delta = false s.index = -1 s.Values = make(map[string]*StatusValue) s.prevValues = make(map[string]int64) @@ -301,7 +301,7 @@ func statusGetIntAttributes(s *StatusSet, elem *ibmmq.PCFParameter, key string) // metric attribute. if index == -1 { v := elem.Int64Value[0] - if s.Attributes[attr].delta { + if s.Attributes[attr].Delta { // If we have already got a value for this attribute and queue // then use it to create the delta. Otherwise make the initial // value 0. @@ -323,7 +323,7 @@ func statusGetIntAttributes(s *StatusSet, elem *ibmmq.PCFParameter, key string) } } else { v := elem.Int64Value - if s.Attributes[attr].delta { + if s.Attributes[attr].Delta { // If we have already got a value for this attribute and queue // then use it to create the delta. Otherwise make the initial // value 0. diff --git a/mqmetric/sub.go b/mqmetric/sub.go index 7169e4c..4b483f8 100644 --- a/mqmetric/sub.go +++ b/mqmetric/sub.go @@ -82,7 +82,7 @@ func SubInitAttributes() { // These are the integer status fields that are of interest attr = ATTR_SUB_MESSAGES st.Attributes[attr] = newStatusAttribute(attr, "Messages Received", ibmmq.MQIACF_MESSAGE_COUNT) - st.Attributes[attr].delta = true + st.Attributes[attr].Delta = true os.init = true traceExit("SubInitAttributes", 0) diff --git a/mqmetric/topic.go b/mqmetric/topic.go index ee71365..8ea41e5 100644 --- a/mqmetric/topic.go +++ b/mqmetric/topic.go @@ -78,10 +78,10 @@ func TopicInitAttributes() { // These are the integer status fields that are of interest attr = ATTR_TOPIC_PUB_MESSAGES st.Attributes[attr] = newStatusAttribute(attr, "Published Messages", ibmmq.MQIACF_PUBLISH_COUNT) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_TOPIC_SUB_MESSAGES st.Attributes[attr] = newStatusAttribute(attr, "Received Messages", ibmmq.MQIACF_MESSAGE_COUNT) - st.Attributes[attr].delta = true // We have to manage the differences as MQ reports cumulative values + st.Attributes[attr].Delta = true // We have to manage the differences as MQ reports cumulative values attr = ATTR_TOPIC_PUBLISHER_COUNT st.Attributes[attr] = newStatusAttribute(attr, "Number of publishers", ibmmq.MQIA_PUB_COUNT) @@ -155,7 +155,7 @@ func CollectTopicStatus(patterns string) error { // Need to clean out the prevValues elements to stop short-lived topics // building up in the map for a, _ := range st.Attributes { - if st.Attributes[a].delta { + if st.Attributes[a].Delta { m := st.Attributes[a].prevValues for key, _ := range m { if _, ok := os.objectSeen[key]; ok { diff --git a/samples/amqsconn.go b/samples/amqsconn.go index f4dd956..653c3f9 100644 --- a/samples/amqsconn.go +++ b/samples/amqsconn.go @@ -6,7 +6,8 @@ parameters are needed here - channel name and connection information - along with the queue manager name. For example, run as - amqsconn QMGR1 "SYSTEM.DEF.SVRCONN" "myhost.example.com(1414)" + + amqsconn QMGR1 "SYSTEM.DEF.SVRCONN" "myhost.example.com(1414)" If the MQSAMP_USER_ID environment variable is set, then a userid/password flow is also made to authenticate to the queue manager. diff --git a/samples/amqsconntls.go b/samples/amqsconntls.go index 67f8e9b..3c56bb2 100644 --- a/samples/amqsconntls.go +++ b/samples/amqsconntls.go @@ -6,7 +6,8 @@ parameters are still needed here - channel name and connection information - along with the queue manager name. For example, run as - amqsconn QMGR1 "SYSTEM.SSL.SVRCONN" "myhost.example.com(1414)" + + amqsconn QMGR1 "SYSTEM.SSL.SVRCONN" "myhost.example.com(1414)" If the MQSAMP_USER_ID environment variable is set, then a userid/password flow is also made to authenticate to the queue manager. diff --git a/samples/amqsjwt.go b/samples/amqsjwt.go index 719ea8b..f185116 100644 --- a/samples/amqsjwt.go +++ b/samples/amqsjwt.go @@ -3,7 +3,8 @@ This is a short sample to show how to connect to a remote queue manager in a Go program by using a JWT token. The sample makes an API call to the Token Server to authenticate a user, -and uses the returned token to connect to the queue manager. +and uses the returned token to connect to the queue manager which must have been +configured to recognise tokens. There is no attempt in this sample to configure advanced security features such as TLS for the queue manager connection. It does, however, use a minimal @@ -57,7 +58,8 @@ const ( defaultConnectionName = "localhost(1414)" /* Get these values from the Token issuer. */ - defaultTokenAddress = "localhost:8443" + defaultTokenHost = "localhost" + defaultTokenPort = 8443 defaultTokenUserName = "jwtuser" defaultTokenPassword = "passw0rd" defaultTokenClientId = "jwtcid" @@ -65,10 +67,11 @@ const ( ) type Config struct { - qMgrName string - connectionName string - channel string - tokenAddress string + qMgrName string + connectionName string + channel string + tokenHost string + tokenPort int tokenUserName string tokenPassword string tokenClientId string @@ -77,17 +80,20 @@ type Config struct { // We only care about one field in the JSON data returned from // the call to the JWT server -type Token struct { +type JWTResponse struct { AccessToken string `json:"access_token"` } var cf Config -var tokenStruct Token +var jwtResponseStruct JWTResponse func main() { var err error var qMgr ibmmq.MQQueueManager var rc int + token := "" + + fmt.Println("Sample AMQSJWT.GO start") initParms() err = parseParms() @@ -108,12 +114,12 @@ func main() { cno.ClientConn = cd cno.Options = ibmmq.MQCNO_CLIENT_BINDING - err = obtainToken() + token, err = obtainToken() if err == nil { - if tokenStruct.AccessToken != "" { + if token != "" { csp := ibmmq.NewMQCSP() - csp.Token = tokenStruct.AccessToken - fmt.Printf("Using token: %s\n", tokenStruct.AccessToken) + csp.Token = token + fmt.Printf("Using token: %s\n", token) // Make the CNO refer to the CSP structure so it gets used during the connection cno.SecurityParms = csp @@ -129,52 +135,54 @@ func main() { // And now we can try to connect. Wait a short time before disconnecting. qMgr, err = ibmmq.Connx(cf.qMgrName, cno) if err == nil { - fmt.Printf("Connection to %s succeeded.\n", cf.qMgrName) + fmt.Printf("MQCONN to QM %s succeeded.\n", cf.qMgrName) d, _ := time.ParseDuration("3s") time.Sleep(d) qMgr.Disc() // Ignore errors from disconnect as we can't do much about it anyway rc = 0 } else { - fmt.Printf("Connection to %s failed.\n", cf.qMgrName) + fmt.Printf("MQCONN to %s failed.\n", cf.qMgrName) fmt.Println(err) rc = int(err.(*ibmmq.MQReturn).MQCC) } fmt.Println("Done.") os.Exit(rc) - } /* * Function to query a token from the token endpoint. Build the * command that is used to retrieve a JSON response from the token - * server. Parse the response via RetrieveTokenFromResponse to - * retrieve the token to be added into the MQCSP. + * server. Parse the response to find the token to be added into the MQCSP. */ -func obtainToken() error { +func obtainToken() (string, error) { var resp *http.Response /* This curl command is the basis of the call to get a token. It uses form data to set the various parameters - curl -k -X POST "https://$host/realms/$realm/protocol/openid-connect/token" \ + curl -k -X POST "https://$host:$port/realms/$realm/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "username=$user" -d "password=$password" \ -d "grant_type=password" -d "client_id=$cid" \ -o $output -Ss */ - /* - * NOTE: This is not a good idea for production, but it means we don't need to set up a truststore - * for the server's certificate. We will simply trust it - useful if it's a development-level server - * with a self-signed cert. - */ + /* + NOTE 1: The SkipVerify is is not a good idea for production, but it means we don't need to + set up a truststore for the token server's certificate. We will simply trust it - useful if it's a + development-level server with a self-signed cert. + + NOTE 2: If you do choose to set up a truststore/keystore for the connection to the token server, + then they must be in a suitable format for OpenSSL (such as pem, p12), not the kdb format usually + used for an MQ connection. + */ tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} client := &http.Client{Transport: tr} - // Build the URL. We will assume HTTPS - endpoint := fmt.Sprintf("https://%s/realms/%s/protocol/openid-connect/token", cf.tokenAddress, cf.tokenRealm) + // Build the URL. We will assume HTTPS. The path may need to change for different token servers. + endpoint := fmt.Sprintf("https://%s:%d/realms/%s/protocol/openid-connect/token", cf.tokenHost, cf.tokenPort, cf.tokenRealm) // Fill in the pieces of data that the server expects formData := url.Values{ @@ -195,34 +203,36 @@ func obtainToken() error { if err != nil { // we will get an error at this stage if the request fails, such as if the // requested URL is not found, or if the server is not reachable. - return err + return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { err = fmt.Errorf("status code error: %d %s", resp.StatusCode, resp.Status) - return err + return "", err } // If it all worked, we can parse the response. We don't need all of the returned // fields, only the token. data, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return "", err } else { // fmt.Printf("Got back a response: %s\n", data) - err = json.Unmarshal(data, &tokenStruct) + err = json.Unmarshal(data, &jwtResponseStruct) } - return err + return jwtResponseStruct.AccessToken, err } // Command line parameters - set flags and defaults func initParms() { - flag.StringVar(&cf.qMgrName, "m", defaultQMgrName, "Queue Manager for connection") - flag.StringVar(&cf.connectionName, "connection", defaultConnectionName, "Connection Name") - flag.StringVar(&cf.channel, "channel", defaultChannel, "Channel Name") - flag.StringVar(&cf.tokenAddress, "address", defaultTokenAddress, "Address for the token server in URL-style") + flag.StringVar(&cf.qMgrName, "m", defaultQMgrName, "Queue Manager") + flag.StringVar(&cf.connectionName, "connection", defaultConnectionName, "MQ Connection Name") + flag.StringVar(&cf.channel, "channel", defaultChannel, "MQ Channel Name") + flag.StringVar(&cf.tokenHost, "host", defaultTokenHost, "Hostname for the token server") + flag.IntVar(&cf.tokenPort, "port", defaultTokenPort, "Portnumber for the token server") + flag.StringVar(&cf.tokenUserName, "user", defaultTokenUserName, "UserName") flag.StringVar(&cf.tokenPassword, "password", defaultTokenPassword, "Password") flag.StringVar(&cf.tokenClientId, "clientId", defaultTokenClientId, "ClientId") diff --git a/samples/amqspcf.go b/samples/amqspcf.go index a5c1980..c37b252 100644 --- a/samples/amqspcf.go +++ b/samples/amqspcf.go @@ -236,7 +236,7 @@ func getReplies() error { getmqmd := ibmmq.NewMQMD() gmo := ibmmq.NewMQGMO() - gmo.Options = ibmmq.MQGMO_NO_SYNCPOINT + gmo.Options = ibmmq.MQGMO_NO_SYNCPOINT | ibmmq.MQGMO_CONVERT // Set options to wait for a maximum of 3 seconds for any new message to arrive gmo.Options |= ibmmq.MQGMO_WAIT diff --git a/samples/runSample.deb.Dockerfile b/samples/runSample.deb.Dockerfile index b3157ac..57ac687 100644 --- a/samples/runSample.deb.Dockerfile +++ b/samples/runSample.deb.Dockerfile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2019, 2023 +# © Copyright IBM Corporation 2019, 2024 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ # Start by setting some global variables that can still be overridden on the build command line. ARG BASE_IMAGE=ubuntu:18.04 ARG GOPATH_ARG="/go" -ARG GOVERSION=1.17 +ARG GOVERSION=1.18 ########################################################### # This starts the BUILD phase @@ -77,7 +77,7 @@ RUN mkdir -p $GOPATH/src $GOPATH/bin $GOPATH/pkg \ # Location of the downloadable MQ client package \ ENV RDURL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist" \ RDTAR="IBM-MQC-Redist-Linux${MQARCH}.tar.gz" \ - VRMF=9.3.4.0 + VRMF=9.3.5.0 # Install the MQ client from the Redistributable package. This also contains the # header files we need to compile against. Setup the subset of the package diff --git a/samples/runSample.gomod b/samples/runSample.gomod index 179364b..ccb274a 100644 --- a/samples/runSample.gomod +++ b/samples/runSample.gomod @@ -1,6 +1,6 @@ module amqsput/v5 -go 1.17 +go 1.18 require ( github.com/ibm-messaging/mq-golang/v5 latest diff --git a/samples/runSample.ubi.Dockerfile b/samples/runSample.ubi.Dockerfile index ce50207..0b468fb 100644 --- a/samples/runSample.ubi.Dockerfile +++ b/samples/runSample.ubi.Dockerfile @@ -1,4 +1,4 @@ -# © Copyright IBM Corporation 2019, 2023 +# © Copyright IBM Corporation 2019, 2024 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ FROM $BASE_IMAGE AS builder ARG GOPATH_ARG ENV GOPATH=$GOPATH_ARG \ ORG="github.com/ibm-messaging" -ARG GOVERSION=1.17 +ARG GOVERSION=1.18 ARG GOARCH=amd64 ARG MQARCH=X64 @@ -56,7 +56,7 @@ RUN mkdir -p $GOPATH/src $GOPATH/bin $GOPATH/pkg \ # Location of the downloadable MQ client package \ ENV RDURL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist" \ RDTAR="IBM-MQC-Redist-Linux${MQARCH}.tar.gz" \ - VRMF=9.3.4.0 + VRMF=9.3.5.0 # Install the MQ client from the Redistributable package. This also contains the # header files we need to compile against. Setup the subset of the package