diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/asup/OntapAsupManager.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/asup/OntapAsupManager.java new file mode 100644 index 000000000000..9a808157f159 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/asup/OntapAsupManager.java @@ -0,0 +1,561 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.asup; + +import com.cloud.storage.Volume; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.net.NetUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.poll.BackgroundPollManager; +import org.apache.cloudstack.poll.BackgroundPollTask; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; +import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.utils.OntapStorageConstants; +import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.time.Duration; +import java.time.Instant; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Periodic ASUP (AutoSupport) telemetry pusher for the NetApp ONTAP plugin. + * + *

This manager runs on a fixed interval and, for each + * ONTAP-backed primary storage pool, pushes two minimal EMS application-log messages to + * the backing ONTAP cluster:

+ * + * + */ +public class OntapAsupManager extends ManagerBase implements Configurable { + + public static final ConfigKey AsupEnabled = new ConfigKey<>( + OntapStorageConstants.ADVANCED_CONFIG_KEY_CATEGORY, Boolean.class, + OntapStorageConstants.ASUP_ENABLED_CONFIG_KEY, OntapStorageConstants.ASUP_ENABLED_DEFAULT, + OntapStorageConstants.ASUP_ENABLED_DESCRIPTION, + true, ConfigKey.Scope.Global); + + public static final ConfigKey AsupIntervalSeconds = new ConfigKey<>( + OntapStorageConstants.ADVANCED_CONFIG_KEY_CATEGORY, Integer.class, + OntapStorageConstants.ASUP_INTERVAL_CONFIG_KEY, + String.valueOf(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_SECONDS), + OntapStorageConstants.ASUP_INTERVAL_DESCRIPTION, + true, ConfigKey.Scope.Global); + + /** Time (in seconds) to wait while acquiring the single-emitter global lock. */ + private static final int ASUP_LOCK_TIMEOUT_SECONDS = 5; + + /** + * Fixed wakeup interval (ms) for {@link OntapAsupPollTask}. The task wakes on this + * cadence and checks whether the configured push interval ({@link #AsupIntervalSeconds}) + * has elapsed. This decouples the scheduler's fixed delay from the live config value, + * so changes made in the CloudStack UI take effect without a management-server restart. + */ + static final long ASUP_POLL_CHECK_INTERVAL_MS = 60_000L; + + /** + * Volume states that guarantee a physical object exists on the ONTAP FlexVolume. + * States like {@link Volume.State#Allocated} have a CloudStack DB row pointing to this + * pool but ONTAP provisioning has not been called yet — they must be excluded to avoid + * inflating disk counts and provisioned-size totals. Upload-family states live on + * secondary storage, not on the primary ONTAP volume, so they are also excluded. + */ + private static final Set CS_VOLUME_STATES = EnumSet.of( + Volume.State.Ready, + Volume.State.Snapshotting, + Volume.State.RevertSnapshotting, + Volume.State.Attaching, + Volume.State.Restoring, + Volume.State.Expunging, + Volume.State.Destroying + ); + + /** Serializes the structured event-description payloads to JSON. */ + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Timestamp of the last successful ASUP push. Starts at {@link Instant#EPOCH} so the + * very first wakeup always fires immediately. {@code volatile} ensures the poll-task + * thread's write is visible without synchronization overhead. + */ + volatile Instant lastPushTime = Instant.EPOCH; + + @Inject + private PrimaryDataStoreDao storagePoolDao; + @Inject + private StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + private VolumeDao volumeDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMSnapshotDao vmSnapshotDao; + @Inject + private BackgroundPollManager backgroundPollManager; + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + // Submit the periodic ASUP task to CloudStack's shared background poll manager. + // This must happen in the configure-phase: the poll manager schedules all submitted + // tasks during its own start-phase and rejects late submissions. Using the shared + // scheduler means this plugin does not create or manage its own thread. + backgroundPollManager.submitTask(new OntapAsupPollTask()); + logger.info("OntapAsupManager configured; ASUP poll task submitted to BackgroundPollManager"); + return true; + } + + /** + * Iterates all ONTAP-backed primary storage pools and pushes ASUP telemetry for each. + * + *

Guarded by a {@link GlobalLock} so that, in a multi-management-server deployment, + * only one node emits per cycle.

+ */ + protected void pushAsupForAllStoragePools() { + if (Boolean.FALSE.equals(AsupEnabled.value())) { + logger.debug("ONTAP ASUP: telemetry is disabled ({}=false); skipping this cycle.", AsupEnabled.key()); + return; + } + List pools = storagePoolDao.findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME); + if (CollectionUtils.isEmpty(pools)) { + logger.debug("ONTAP ASUP: no ONTAP-backed storage pools found; nothing to push."); + return; + } + + GlobalLock lock = GlobalLock.getInternLock(OntapStorageConstants.ASUP_GLOBAL_LOCK_NAME); + try { + if (!lock.lock(ASUP_LOCK_TIMEOUT_SECONDS)) { + logger.debug("ONTAP ASUP: another management server holds the ASUP lock; skipping this cycle."); + return; + } + String cloudStackVersion = getCloudStackVersion(); + String computerName = getComputerName(); + logger.debug("ONTAP ASUP: pushing telemetry for {} pool(s) [CloudStack version={}]", + pools.size(), cloudStackVersion); + // Tracks clusters that have already received a heartbeat this cycle, so that multiple + // pools backed by the same ONTAP cluster emit only a single heartbeat (event-id 0), + // while each distinct cluster still gets its own heartbeat per cycle. + Set clustersHeartBeated = new HashSet<>(); + for (StoragePoolVO pool : pools) { + pushAsupForStoragePool(pool, cloudStackVersion, computerName, clustersHeartBeated); + } + } finally { + lock.unlock(); + } + } + + /** + * Pushes the heartbeat (event-id 0) and pool (event-id 1) ASUP messages for a single pool. + * + *

The heartbeat is emitted at most once per distinct ONTAP cluster per cycle: the cluster's + * UUID (or its storage IP when the UUID is unavailable) is recorded in {@code clustersHeartbeated}, + * and subsequent pools backed by the same cluster skip the heartbeat. The pool mapping message + * is always emitted, once per pool.

+ * + *

Best-effort: any failure is logged and swallowed.

+ */ + protected void pushAsupForStoragePool(StoragePoolVO pool, String cloudStackVersion, String computerName, Set clustersHeartbeated) { + try { + Map details = storagePoolDetailsDao.listDetailsKeyPairs(pool.getId()); + if (details == null || details.isEmpty()) { + logger.warn("ONTAP ASUP: storage pool [{}] has no details; skipping.", pool.getId()); + return; + } + + StorageStrategy strategy = OntapStorageUtils.getStrategyByStoragePoolDetails(details); + // Fetch the ONTAP cluster once and reuse its identity (uuid, name) and version + // for both messages, avoiding extra REST round-trips. + Cluster cluster = strategy.getClusterInfo(); + String ontapVersion = strategy.getClusterVersion(cluster); + String clusterUuid = cluster != null ? cluster.getUuid() : null; + String clusterName = cluster != null ? cluster.getName() : null; + String appVersion = buildAppVersion(cloudStackVersion, ontapVersion); + + // event-id 0: CloudStack -> ONTAP cluster heartbeat (versions), emitted once per cluster. + // Key on the cluster UUID; fall back to the storage IP if the UUID is unavailable. + String clusterKey = StringUtils.isNotBlank(clusterUuid) ? clusterUuid + : details.get(OntapStorageConstants.STORAGE_IP); + if (clusterKey == null || clustersHeartbeated.add(clusterKey)) { + EmsApplicationLog heartbeat = buildBaseMessage(computerName, appVersion); + heartbeat.setEventId(OntapStorageConstants.ASUP_EVENT_ID_HEARTBEAT); + heartbeat.setEventDescription( + buildHeartbeatDescription(cloudStackVersion, ontapVersion, clusterUuid)); + strategy.sendAsupMessage(heartbeat); + } else { + logger.debug("ONTAP ASUP: heartbeat already sent this cycle for cluster [{}]; skipping for pool [{}]", + defaultUnknown(clusterName), pool.getId()); + } + + // event-id 1: CloudStack storage pool -> backing ONTAP volume mapping, once per pool. + // The description also includes disk usage and snapshot telemetry + EmsApplicationLog poolMessage = buildBaseMessage(computerName, appVersion); + poolMessage.setEventId(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL); + poolMessage.setEventDescription(buildPoolDescription(pool, details, clusterUuid)); + strategy.sendAsupMessage(poolMessage); + + logger.debug("ONTAP ASUP: pushed telemetry for pool [{}] (ONTAP version={})", + pool.getId(), defaultUnknown(ontapVersion)); + } catch (Exception e) { + // Best-effort telemetry; never propagate. + logger.warn("ONTAP ASUP: failed to push telemetry for pool [{}]: {}", pool.getId(), e.getMessage()); + } + } + + /** + * Builds the heartbeat (event-id 0) description as a JSON object carrying the CloudStack and + * ONTAP versions, the management-server operating system platform, plus the ONTAP cluster UUID. + * Example: {@code {"message":"CloudStack connected to ONTAP cluster","cloudstackVersion": + * "4.23.0.0","platform":"Linux 5.15.0-91-generic (amd64)","ontapVersion":"9.17.1", + * "clusterUuid":"..."}} + */ + private String buildHeartbeatDescription(String cloudStackVersion, String ontapVersion, + String clusterUuid) { + Map payload = new LinkedHashMap<>(); + payload.put("message", "CloudStack connected to ONTAP cluster"); + payload.put("cloudstackVersion", defaultUnknown(cloudStackVersion)); + payload.put("platform", getOperatingSystem()); + payload.put("ontapVersion", defaultUnknown(ontapVersion)); + payload.put("clusterUuid", defaultUnknown(clusterUuid)); + return toJson(payload); + } + + /** + * Builds the pool description (event-id 1) as a JSON object combining the backing-volume + * mapping, disk usage, and snapshot telemetry into a single EMS message. + * + *

Example: {@code {"message":"CloudStack storage pool backed by ONTAP volume", + * "poolName":"...","protocol":"nfs","clusterUuid":"...","svm":"...", + * "ontapVolumeUuid":"...","rootDiskCount":12,"dataDiskCount":18, + * "totalProvisionedSizeBytes":322122547200, + * "csVolumeSnapshotCount":5,"csVolumeSnapshotProvisionedSizeBytes":107374182400, + * "vmSnapshotCount":3,"vmSnapshotSizeBytes":322122547200}}

+ */ + private String buildPoolDescription(StoragePoolVO pool, Map details, + String clusterUuid) { + Map payload = new LinkedHashMap<>(); + payload.put("message", "CloudStack storage pool backed by ONTAP volume"); + payload.put("poolName", defaultUnknown(pool.getName())); + payload.put("protocol", defaultUnknown(details.get(OntapStorageConstants.PROTOCOL))); + payload.put("clusterUuid", defaultUnknown(clusterUuid)); + payload.put("svm", defaultUnknown(details.get(OntapStorageConstants.SVM_NAME))); + payload.put("ontapVolumeUuid", defaultUnknown(details.get(OntapStorageConstants.VOLUME_UUID))); + addStoragePoolUsage(pool, payload); + addSnapshotUsage(pool, payload); + return toJson(payload); + } + + /** + * Computes pool usage from CloudStack's volume records and adds it to the payload: + *
    + *
  • {@code rootDiskCount} - number of ROOT (boot) disks physically on this pool
  • + *
  • {@code dataDiskCount} - number of DATADISK disks physically on this pool
  • + *
  • {@code attachedVmCount} - number of distinct VMs that have at least one volume on + * this pool; always ≥ {@code rootDiskCount} because a VM's root disk may live on a + * different pool while one of its data disks resides here
  • + *
  • {@code totalProvisionedSizeBytes} - sum of those volumes' provisioned (logical) sizes + * in bytes; for thin-provisioned volumes this is the logical size requested at + * creation time, not the physical space consumed on ONTAP
  • + *
+ * All volume types are counted (both ROOT and DATADISK); the {@code null} type argument + * disables the type filter in the DAO. All derived values are computed in-memory from the + * same single query (no extra round-trips). Best-effort: any failure leaves the usage + * fields out and never breaks telemetry. + */ + private void addStoragePoolUsage(StoragePoolVO pool, Map payload) { + try { + // Pass null volume-type to include ALL volumes (ROOT + DATADISK). The single-arg + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); + + // Only count volumes that definitely have a physical object on the ONTAP FlexVolume. + // "Allocated" volumes have a pool_id row in the CS DB but ONTAP provisioning has not + // yet been called, so including them would inflate counts and provisioned size. + // Upload-family states live on secondary storage, not on this primary pool. + List cstackVolumes = volumes.stream() + .filter(v -> CS_VOLUME_STATES.contains(v.getState())) + .collect(java.util.stream.Collectors.toList()); + + long rootDiskCount = cstackVolumes.stream() + .filter(v -> Volume.Type.ROOT.equals(v.getVolumeType())).count(); + long dataDiskCount = cstackVolumes.stream() + .filter(v -> Volume.Type.DATADISK.equals(v.getVolumeType())).count(); + // Count distinct VMs that have at least one volume on this pool. + // A VM whose root disk is on a different pool but has a data disk here is still counted, + long attachedVmCount = cstackVolumes.stream() + .map(VolumeVO::getInstanceId) + .filter(id -> id != null) + .distinct() + .count(); + + long totalProvisionedSizeBytes = cstackVolumes.stream() + .mapToLong(v -> v.getSize() != null ? v.getSize() : 0L).sum(); + payload.put("rootDiskCount", rootDiskCount); + payload.put("dataDiskCount", dataDiskCount); + payload.put("attachedVmCount", attachedVmCount); + payload.put("totalProvisionedSizeBytes", totalProvisionedSizeBytes); + } catch (Exception e) { + logger.error("ONTAP ASUP: failed to compute usage for pool [{}]: {}", pool.getId(), e.getMessage()); + } + } + + /** + * Computes and adds two groups of snapshot telemetry to the pool description payload. + * + *

Volume-snapshot metrics ({@code csVolumeSnapshotCount}, + * {@code csVolumeSnapshotSizeBytes}): counts all non-destroyed CloudStack + * volume-level snapshots for volumes on this pool. Where an ONTAP-side size has been + * recorded in {@code snapshot_details} (key {@code ontap_snap_size}) that value is used; + * otherwise the source volume's provisioned size is used as a conservative upper bound.

+ * + *

VM-snapshot metrics ({@code vmSnapshotCount}, + * {@code vmSnapshotSizeBytes}): counts all active (non-expunging, non-removed) VM + * snapshots for VMs that have at least one volume on this pool. Because ONTAP stores + * VM snapshots as FlexVol-level snapshots, the ONTAP-space estimate is computed as + * {@code sum(poolVolumes.size) × vmSnapshotCount} — a pool-level approximation.

+ * + *

Best-effort: any failure leaves the fields out without breaking telemetry.

+ */ + private void addSnapshotUsage(StoragePoolVO pool, Map payload) { + addVmSnapshotMetrics(pool, payload); + addVolumeSnapshotMetrics(pool, payload); + } + + /** + * Adds {@code csVolumeSnapshotCount} to the payload. + * Counts all non-destroyed CloudStack volume-level snapshots for volumes on this pool. + */ + private void addVolumeSnapshotMetrics(StoragePoolVO pool, Map payload) { + try { + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); + if (volumes == null || volumes.isEmpty()) { + payload.put("csVolumeSnapshotCount", 0); + return; + } + + List volumeIds = new java.util.ArrayList<>(); + for (VolumeVO v : volumes) { + volumeIds.add(v.getId()); + } + + List snapshots = snapshotDao.searchByVolumes(volumeIds); + long snapCount = 0; + if (snapshots != null) { + for (SnapshotVO snap : snapshots) { + if (!com.cloud.storage.Snapshot.State.Destroyed.equals(snap.getState())) { + snapCount++; + } + } + } + + payload.put("csVolumeSnapshotCount", snapCount); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to compute volume-snapshot metrics for pool [{}]: {}", + pool.getId(), e.getMessage()); + } + } + + /** + * Adds {@code vmSnapshotCount} to the payload. + * Counts all active (non-expunging, non-removed) VM snapshots for VMs that have at + * least one volume on this pool. + */ + private void addVmSnapshotMetrics(StoragePoolVO pool, Map payload) { + try { + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); + if (volumes == null || volumes.isEmpty()) { + payload.put("vmSnapshotCount", 0); + return; + } + + java.util.Set vmIds = new java.util.HashSet<>(); + for (VolumeVO v : volumes) { + if (v.getInstanceId() != null) { + vmIds.add(v.getInstanceId()); + } + } + + if (vmIds.isEmpty()) { + payload.put("vmSnapshotCount", 0); + return; + } + + List vmSnapshots = vmSnapshotDao.searchByVms(new java.util.ArrayList<>(vmIds)); + long vmSnapCount = 0; + if (vmSnapshots != null) { + for (VMSnapshotVO vmSnap : vmSnapshots) { + if (!VMSnapshot.State.Expunging.equals(vmSnap.getState()) && vmSnap.getRemoved() == null) { + vmSnapCount++; + } + } + } + + payload.put("vmSnapshotCount", vmSnapCount); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to compute VM-snapshot metrics for pool [{}]: {}", + pool.getId(), e.getMessage()); + } + } + + /** + * Serializes a payload map to a JSON string. Falls back to the map's {@code toString()} if + * serialization unexpectedly fails, so telemetry is still emitted (best-effort). + */ + private String toJson(Map payload) { + try { + return objectMapper.writeValueAsString(payload); + } catch (Exception e) { + logger.warn("ONTAP ASUP: failed to serialize event description to JSON: {}", e.getMessage()); + return String.valueOf(payload); + } + } + + /** Builds the common EMS message envelope shared by all ASUP messages. */ + private EmsApplicationLog buildBaseMessage(String computerName, String appVersion) { + EmsApplicationLog message = new EmsApplicationLog(); + message.setComputerName(computerName); + message.setEventSource(OntapStorageConstants.ASUP_EVENT_SOURCE); + message.setAppVersion(appVersion); + message.setCategory(OntapStorageConstants.ASUP_CATEGORY); + message.setSeverity(OntapStorageConstants.ASUP_SEVERITY); + message.setAutosupportRequired(Boolean.FALSE); + return message; + } + + /** Composes "cloudstack-version|ontap-version" for the EMS app-version field. */ + private String buildAppVersion(String cloudStackVersion, String ontapVersion) { + return "cloudstack-" + cloudStackVersion + "|ontap-" + defaultUnknown(ontapVersion); + } + + /** + * Resolves the CloudStack management server version from the running artifact's manifest. + * Falls back to "unknown" when not available (e.g. running from an IDE without a manifest). + */ + protected String getCloudStackVersion() { + String version = this.getClass().getPackage().getImplementationVersion(); + if (StringUtils.isBlank(version)) { + version = OntapAsupManager.class.getPackage().getImplementationVersion(); + } + return StringUtils.isBlank(version) ? OntapStorageConstants.ASUP_UNKNOWN : version; + } + + /** Resolves the management server hostname for the EMS computer-name field. */ + protected String getComputerName() { + String hostName = NetUtils.getCanonicalHostName(); + return StringUtils.isBlank(hostName) ? OntapStorageConstants.ASUP_UNKNOWN : hostName; + } + + /** + * Resolves the management server operating system (name, version and architecture) from JVM + * system properties, e.g. {@code "Linux 5.15.0-91-generic (amd64)"}. Falls back to "unknown". + */ + protected String getOperatingSystem() { + String osName = System.getProperty("os.name"); + String osVersion = System.getProperty("os.version"); + String osArch = System.getProperty("os.arch"); + if (StringUtils.isBlank(osName)) { + return OntapStorageConstants.ASUP_UNKNOWN; + } + StringBuilder sb = new StringBuilder(osName); + if (StringUtils.isNotBlank(osVersion)) { + sb.append(' ').append(osVersion); + } + if (StringUtils.isNotBlank(osArch)) { + sb.append(" (").append(osArch).append(')'); + } + return sb.toString(); + } + + private String defaultUnknown(String value) { + return StringUtils.isBlank(value) ? OntapStorageConstants.ASUP_UNKNOWN : value; + } + + @Override + public String getConfigComponentName() { + return OntapAsupManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {AsupEnabled, AsupIntervalSeconds}; + } + + /** + * Background poll task that runs the ASUP push within a managed CloudStack context. + * + *

Wakes every {@link #ASUP_POLL_CHECK_INTERVAL_MS} ms. On each wakeup it reads + * the live value of {@link #AsupIntervalSeconds} and only pushes if enough time has + * elapsed since {@link #lastPushTimeMs}. This means changing the interval in the + * CloudStack UI takes effect within one check cycle — no restart required.

+ */ + protected class OntapAsupPollTask extends ManagedContextRunnable implements BackgroundPollTask { + @Override + protected void runInContext() { + try { + int intervalSeconds = AsupIntervalSeconds.value() != null + ? AsupIntervalSeconds.value() : OntapStorageConstants.ASUP_DEFAULT_INTERVAL_SECONDS; + if (intervalSeconds <= 0) { + intervalSeconds = OntapStorageConstants.ASUP_DEFAULT_INTERVAL_SECONDS; + } + Duration configuredInterval = Duration.ofSeconds(intervalSeconds); + Instant now = Instant.now(); + if (Duration.between(lastPushTime, now).compareTo(configuredInterval) < 0) { + return; // configured interval has not elapsed yet + } + lastPushTime = now; + pushAsupForAllStoragePools(); + } catch (Exception e) { + logger.warn("ONTAP ASUP: unexpected error during periodic push: {}", e.getMessage()); + } + } + + @Override + public Long getDelay() { + return ASUP_POLL_CHECK_INTERVAL_MS; + } + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/EmsFeignClient.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/EmsFeignClient.java new file mode 100644 index 000000000000..c3e4c73d6165 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/EmsFeignClient.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.feign.client; + +import feign.Headers; +import feign.Param; +import feign.RequestLine; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; + +public interface EmsFeignClient { + + @RequestLine("POST /api/support/ems/application-logs") + @Headers({"Authorization: {authHeader}", "Content-Type: application/json"}) + void sendEmsApplicationLog(@Param("authHeader") String authHeader, EmsApplicationLog emsApplicationLog); +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/EmsApplicationLog.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/EmsApplicationLog.java new file mode 100644 index 000000000000..24289c8d33b2 --- /dev/null +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/EmsApplicationLog.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.feign.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EmsApplicationLog { + + @JsonProperty("computer_name") + private String computerName; + + @JsonProperty("event_source") + private String eventSource; + + @JsonProperty("app_version") + private String appVersion; + + @JsonProperty("category") + private String category; + + @JsonProperty("severity") + private String severity; + + @JsonProperty("autosupport_required") + private Boolean autosupportRequired; + + @JsonProperty("event_id") + private String eventId; + + @JsonProperty("event_description") + private String eventDescription; + + public EmsApplicationLog() { + } + + public String getComputerName() { + return computerName; + } + + public void setComputerName(String computerName) { + this.computerName = computerName; + } + + public String getEventSource() { + return eventSource; + } + + public void setEventSource(String eventSource) { + this.eventSource = eventSource; + } + + public String getAppVersion() { + return appVersion; + } + + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getSeverity() { + return severity; + } + + public void setSeverity(String severity) { + this.severity = severity; + } + + public Boolean getAutosupportRequired() { + return autosupportRequired; + } + + public void setAutosupportRequired(Boolean autosupportRequired) { + this.autosupportRequired = autosupportRequired; + } + + public String getEventId() { + return eventId; + } + + public void setEventId(String eventId) { + this.eventId = eventId; + } + + public String getEventDescription() { + return eventDescription; + } + + public void setEventDescription(String eventDescription) { + this.eventDescription = eventDescription; + } + + @Override + public String toString() { + return "EmsApplicationLog{" + + "computerName='" + computerName + '\'' + + ", eventSource='" + eventSource + '\'' + + ", appVersion='" + appVersion + '\'' + + ", category='" + category + '\'' + + ", severity='" + severity + '\'' + + ", autosupportRequired=" + autosupportRequired + + ", eventId='" + eventId + '\'' + + ", eventDescription='" + eventDescription + '\'' + + '}'; + } +} diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java index f4ab806d6885..6d59b588ccad 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java @@ -23,20 +23,25 @@ import feign.FeignException; import org.apache.cloudstack.storage.feign.FeignClientFactory; import org.apache.cloudstack.storage.feign.client.AggregateFeignClient; +import org.apache.cloudstack.storage.feign.client.ClusterFeignClient; import org.apache.cloudstack.storage.feign.client.JobFeignClient; import org.apache.cloudstack.storage.feign.client.NetworkFeignClient; import org.apache.cloudstack.storage.feign.client.NASFeignClient; import org.apache.cloudstack.storage.feign.client.SANFeignClient; import org.apache.cloudstack.storage.feign.client.SnapshotFeignClient; +import org.apache.cloudstack.storage.feign.client.EmsFeignClient; import org.apache.cloudstack.storage.feign.client.SvmFeignClient; import org.apache.cloudstack.storage.feign.client.VolumeFeignClient; import org.apache.cloudstack.storage.feign.model.Aggregate; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; import org.apache.cloudstack.storage.feign.model.IpInterface; import org.apache.cloudstack.storage.feign.model.IscsiService; import org.apache.cloudstack.storage.feign.model.Job; import org.apache.cloudstack.storage.feign.model.Nas; import org.apache.cloudstack.storage.feign.model.OntapStorage; import org.apache.cloudstack.storage.feign.model.Svm; +import org.apache.cloudstack.storage.feign.model.Version; import org.apache.cloudstack.storage.feign.model.Volume; import org.apache.cloudstack.storage.feign.model.response.JobResponse; import org.apache.cloudstack.storage.feign.model.response.OntapResponse; @@ -72,6 +77,8 @@ public abstract class StorageStrategy { protected SANFeignClient sanFeignClient; protected NASFeignClient nasFeignClient; protected SnapshotFeignClient snapshotFeignClient; + protected ClusterFeignClient clusterFeignClient; + protected EmsFeignClient emsFeignClient; protected OntapStorage storage; @@ -96,6 +103,74 @@ public StorageStrategy(OntapStorage ontapStorage) { this.sanFeignClient = feignClientFactory.createClient(SANFeignClient.class, baseURL); this.nasFeignClient = feignClientFactory.createClient(NASFeignClient.class, baseURL); this.snapshotFeignClient = feignClientFactory.createClient(SnapshotFeignClient.class, baseURL); + this.clusterFeignClient = feignClientFactory.createClient(ClusterFeignClient.class, baseURL); + this.emsFeignClient = feignClientFactory.createClient(EmsFeignClient.class, baseURL); + } + + /** + * Fetches the full ONTAP {@link Cluster} object (name, uuid, version) in a single REST call, + * for ASUP telemetry. Best-effort: returns {@code null} if it cannot be retrieved, and callers + * must never fail a storage operation because of it. + * + * @return the ONTAP {@link Cluster}, or {@code null} if it cannot be resolved + */ + public Cluster getClusterInfo() { + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + return clusterFeignClient.getCluster(authHeader, true); + } catch (Exception e) { + logger.warn("getClusterInfo: failed to fetch ONTAP cluster info for storage IP {}: {}", + storage.getStorageIP(), e.getMessage()); + return null; + } + } + + /** + * Extracts a clean, parser-friendly ONTAP version string from a {@link Cluster}. + * + *

Prefers the compact "generation.major.minor" numeric form (e.g. "9.17.1"), which avoids + * the colon/date noise in {@code version.full}. Falls back to the verbose {@code version.full} + * banner only when the numeric fields are unavailable.

+ * + * @param cluster the cluster (may be {@code null}) + * @return the ONTAP version string, or {@code null} if it cannot be resolved + */ + public String getClusterVersion(Cluster cluster) { + if (cluster == null || cluster.getVersion() == null) { + return null; + } + Version version = cluster.getVersion(); + if (version.getGeneration() != null && version.getMajor() != null && version.getMinor() != null) { + return version.getGeneration() + OntapStorageConstants.DOT + version.getMajor() + + OntapStorageConstants.DOT + version.getMinor(); + } + if (version.getFull() != null && !version.getFull().isEmpty()) { + return version.getFull(); + } + return null; + } + + /** + * Pushes a single ASUP (AutoSupport) EMS application-log message to the ONTAP cluster. + * + *

This is strictly best-effort telemetry: any failure is logged and swallowed so that + * it can never affect a storage operation or the periodic scheduler.

+ * + * @param message the EMS message to send + */ + public void sendAsupMessage(EmsApplicationLog message) { + if (message == null) { + return; + } + try { + String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword()); + emsFeignClient.sendEmsApplicationLog(authHeader, message); + logger.debug("sendAsupMessage: ASUP EMS message [event-id={}] sent to ONTAP cluster at {}", + message.getEventId(), storage.getStorageIP()); + } catch (Exception e) { + logger.error("sendAsupMessage: failed to send ASUP EMS message [event-id={}] to ONTAP cluster at {}: {}", + message.getEventId(), storage.getStorageIP(), e.getMessage()); + } } // Connect method to validate ONTAP cluster, credentials, protocol, and SVM diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java index 6ecfa3967470..3792717a49a5 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java @@ -80,6 +80,7 @@ public class OntapStorageConstants { public static final String SEMICOLON = ";"; public static final String COMMA = ","; public static final String HYPHEN = "-"; + public static final String DOT = "."; public static final String VOLUME_PATH_PREFIX = "/vol/"; @@ -109,4 +110,20 @@ public class OntapStorageConstants { /** vm_snapshot_details key for ONTAP FlexVolume-level VM snapshots. */ public static final String ONTAP_FLEXVOL_SNAPSHOT = "ontapFlexVolSnapshot"; + + // ASUP (AutoSupport) / EMS telemetry + public static final String ADVANCED_CONFIG_KEY_CATEGORY = "Advanced"; + public static final String ASUP_CATEGORY = "provisioning"; + public static final String ASUP_SEVERITY = "notice"; + public static final String ASUP_EVENT_SOURCE = "CloudStack ONTAP plugin"; + public static final String ASUP_EVENT_ID_HEARTBEAT = "0"; + public static final String ASUP_EVENT_ID_STORAGE_POOL = "1"; + public static final String ASUP_UNKNOWN = "unknown"; + public static final String ASUP_GLOBAL_LOCK_NAME = "ontap.asup.push"; + public static final String ASUP_ENABLED_CONFIG_KEY = "ontap.asup.enabled"; + public static final String ASUP_ENABLED_DEFAULT = "true"; + public static final String ASUP_ENABLED_DESCRIPTION = "Enable periodic ASUP (AutoSupport) telemetry push from the CloudStack ONTAP plugin to the ONTAP cluster."; + public static final String ASUP_INTERVAL_CONFIG_KEY = "ontap.asup.interval"; + public static final int ASUP_DEFAULT_INTERVAL_SECONDS = 43200; // 12 hours (twice a day) + public static final String ASUP_INTERVAL_DESCRIPTION = "Interval (in seconds) between periodic ASUP telemetry pushes from the CloudStack ONTAP plugin."; } diff --git a/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml b/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml index bb907871469c..16efb56d136e 100644 --- a/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml +++ b/plugins/storage/volume/ontap/src/main/resources/META-INF/cloudstack/storage-volume-ontap/spring-storage-volume-ontap-context.xml @@ -33,4 +33,7 @@ + + diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/asup/OntapAsupManagerTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/asup/OntapAsupManagerTest.java new file mode 100644 index 000000000000..1f56af360d2e --- /dev/null +++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/asup/OntapAsupManagerTest.java @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.asup; + +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.feign.model.Cluster; +import org.apache.cloudstack.storage.feign.model.EmsApplicationLog; +import org.apache.cloudstack.storage.service.StorageStrategy; +import org.apache.cloudstack.storage.utils.OntapStorageConstants; +import org.apache.cloudstack.storage.utils.OntapStorageUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class OntapAsupManagerTest { + + // ── DAOs ────────────────────────────────────────────────────────────────── + @Mock private PrimaryDataStoreDao storagePoolDao; + @Mock private StoragePoolDetailsDao storagePoolDetailsDao; + @Mock private VolumeDao volumeDao; + @Mock private SnapshotDao snapshotDao; + @Mock private VMSnapshotDao vmSnapshotDao; + + @InjectMocks + private OntapAsupManager asupManager; + + // ── Common fixtures ────────────────────────────────────────────────────── + private StoragePoolVO pool; + private Map poolDetails; + private StorageStrategy mockStrategy; + private Cluster mockCluster; + + @BeforeEach + void setUp() { + pool = mock(StoragePoolVO.class); + lenient().when(pool.getId()).thenReturn(1L); + lenient().when(pool.getName()).thenReturn("ontap-pool-1"); + + poolDetails = new HashMap<>(); + poolDetails.put(OntapStorageConstants.STORAGE_IP, "192.168.1.10"); + poolDetails.put(OntapStorageConstants.PROTOCOL, "NFS3"); + poolDetails.put(OntapStorageConstants.SVM_NAME, "svm1"); + poolDetails.put(OntapStorageConstants.VOLUME_UUID, "fv-uuid-1"); + poolDetails.put(OntapStorageConstants.VOLUME_NAME, "fv-name-1"); + + mockStrategy = mock(StorageStrategy.class); + + mockCluster = mock(Cluster.class); + lenient().when(mockCluster.getUuid()).thenReturn("cluster-uuid-1"); + lenient().when(mockCluster.getName()).thenReturn("ontap-cluster-1"); + } + + // ────────────────────────────────────────────────────────────────────────── + // pushAsupForAllStoragePools – no pools + // ────────────────────────────────────────────────────────────────────────── + + @Test + void pushAsupForAllPools_noOntapPools_sendsNoMessages() { + when(storagePoolDao.findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME)) + .thenReturn(Collections.emptyList()); + + asupManager.pushAsupForAllStoragePools(); + + verify(mockStrategy, never()).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Message count / event-id routing + // ────────────────────────────────────────────────────────────────────────── + + @Test + void pushAsupForStoragePool_newCluster_sendsHeartbeatThenPoolMessage() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + // heartbeat (event-id 0) + pool (event-id 1) = 2 messages + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + + List msgs = cap.getAllValues(); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_HEARTBEAT, msgs.get(0).getEventId()); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL, msgs.get(1).getEventId()); + } + + @Test + void pushAsupForStoragePool_clusterAlreadyHeartbeated_sendsOnlyPoolMessage() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + HashSet clustersHeartbeated = new HashSet<>(); + clustersHeartbeated.add("cluster-uuid-1"); // already sent this cycle + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", clustersHeartbeated); + } + + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(1)).sendAsupMessage(cap.capture()); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL, cap.getValue().getEventId()); + } + + @Test + void pushAsupForStoragePool_strategyThrows_doesNotPropagateException() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenThrow(new RuntimeException("connection refused")); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + verify(mockStrategy, never()).sendAsupMessage(any()); + } + + @Test + void pushAsupForStoragePool_poolDetailsEmpty_skipsPool() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(Collections.emptyMap()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())) + .thenThrow(new RuntimeException("no details")); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + verify(mockStrategy, never()).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Pool message — content verification + // ────────────────────────────────────────────────────────────────────────── + + @Test + void poolMessage_containsPoolNameClusterUuidAndSnapshotKeys() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("ontap-pool-1"), "should contain pool name"); + assertTrue(desc.contains("cluster-uuid-1"), "should contain cluster UUID"); + assertTrue(desc.contains("csVolumeSnapshotCount"), "should contain csVolumeSnapshotCount"); + assertTrue(desc.contains("vmSnapshotCount"), "should contain vmSnapshotCount"); + } + + @Test + void poolMessage_volumeSnapshots_zeroWhenNoVolumes() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"csVolumeSnapshotCount\":0"), "desc=" + desc); + assertTrue(desc.contains("\"vmSnapshotCount\":0"), "desc=" + desc); + } + + @Test + void poolMessage_volumeSnapshots_countExcludesDestroyed() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + // instanceId=null → no VM IDs → vmSnapshotDao never called + VolumeVO vol = mockVolume(10L, null, 10_737_418_240L); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.singletonList(vol)); + + // 2 active + 1 Destroyed → only 2 counted + SnapshotVO s1 = makeSnapshot(1L, 10L, Snapshot.State.BackedUp); + SnapshotVO s2 = makeSnapshot(2L, 10L, Snapshot.State.Creating); + SnapshotVO s3 = makeSnapshot(3L, 10L, Snapshot.State.Destroyed); + when(snapshotDao.searchByVolumes(anyList())).thenReturn(Arrays.asList(s1, s2, s3)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"csVolumeSnapshotCount\":2"), "Destroyed must be excluded; desc=" + desc); + } + + // ────────────────────────────────────────────────────────────────────────── + // Pool message — VM-snapshot fields (vmSnapshotCount) + // ────────────────────────────────────────────────────────────────────────── + + @Test + void poolMessage_vmSnapshots_countsActiveSnapshotsAcrossDistinctVMs() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + VolumeVO vol1 = mockVolume(10L, 100L, 10_737_418_240L); // vm 100 + VolumeVO vol2 = mockVolume(20L, 200L, 10_737_418_240L); // vm 200 + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())) + .thenReturn(Arrays.asList(vol1, vol2)); + when(snapshotDao.searchByVolumes(anyList())).thenReturn(Collections.emptyList()); + + // 2 active VM snapshots; 1 is Expunging (should be excluded) + VMSnapshotVO vmSnap1 = makeVmSnapshot(VMSnapshot.State.Ready, null); + VMSnapshotVO vmSnap2 = makeVmSnapshot(VMSnapshot.State.Ready, null); + VMSnapshotVO vmSnapExp = makeVmSnapshot(VMSnapshot.State.Expunging, null); + when(vmSnapshotDao.searchByVms(anyList())).thenReturn(Arrays.asList(vmSnap1, vmSnap2, vmSnapExp)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":2"), "Expunging must be excluded; desc=" + desc); + } + + @Test + void poolMessage_vmSnapshots_removedSnapshotsExcluded() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + VolumeVO vol = mockVolume(10L, 100L, 1_073_741_824L); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.singletonList(vol)); + when(snapshotDao.searchByVolumes(anyList())).thenReturn(Collections.emptyList()); + + VMSnapshotVO active = makeVmSnapshot(VMSnapshot.State.Ready, null); + VMSnapshotVO deleted = makeVmSnapshot(VMSnapshot.State.Ready, new java.util.Date()); // removed + when(vmSnapshotDao.searchByVms(anyList())).thenReturn(Arrays.asList(active, deleted)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":1"), "removed snapshots must be excluded; desc=" + desc); + } + + @Test + void poolMessage_vmSnapshots_zeroWhenNoVmIdsOnPool() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + + // instanceId=null → detached data disk → vmSnapshotDao must NOT be called + VolumeVO vol = mockVolume(10L, null, 1_073_741_824L); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.singletonList(vol)); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + String desc = capturePoolMessage(); + assertTrue(desc.contains("\"vmSnapshotCount\":0"), "desc=" + desc); + verify(vmSnapshotDao, never()).searchByVms(anyList()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Best-effort: DAO failures must never suppress the pool message + // ────────────────────────────────────────────────────────────────────────── + + @Test + void poolMessage_snapshotDaoThrows_poolMessageStillSent() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())) + .thenThrow(new RuntimeException("DB error")); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + // heartbeat + pool both sent even when DAO fails + verify(mockStrategy, times(2)).sendAsupMessage(any()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Multi-pool: same cluster → single heartbeat + // ────────────────────────────────────────────────────────────────────────── + + @Test + void twoPoolsSameCluster_singleHeartbeat() { + StoragePoolVO pool2 = mock(StoragePoolVO.class); + when(pool2.getId()).thenReturn(2L); + when(pool2.getName()).thenReturn("ontap-pool-2"); + + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(storagePoolDetailsDao.listDetailsKeyPairs(2L)).thenReturn(new HashMap<>(poolDetails)); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(anyLong(), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + + HashSet clustersHeartbeated = new HashSet<>(); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", clustersHeartbeated); + asupManager.pushAsupForStoragePool(pool2, "4.20.0", "mgmt-host", clustersHeartbeated); + } + + // 1 heartbeat + 2 pool messages = 3 total + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(3)).sendAsupMessage(cap.capture()); + + long heartbeats = cap.getAllValues().stream() + .filter(m -> OntapStorageConstants.ASUP_EVENT_ID_HEARTBEAT.equals(m.getEventId())) + .count(); + assertEquals(1, heartbeats, "exactly 1 heartbeat for two pools sharing a cluster"); + } + + // ────────────────────────────────────────────────────────────────────────── + // Common EMS envelope fields + // ────────────────────────────────────────────────────────────────────────── + + @Test + void allMessages_haveCorrectEnvelopeFields() { + when(storagePoolDetailsDao.listDetailsKeyPairs(1L)).thenReturn(poolDetails); + when(mockStrategy.getClusterInfo()).thenReturn(mockCluster); + when(mockStrategy.getClusterVersion(mockCluster)).thenReturn("9.17.1"); + when(volumeDao.findNonDestroyedVolumesByPoolId(eq(1L), isNull())).thenReturn(Collections.emptyList()); + + try (MockedStatic u = mockStatic(OntapStorageUtils.class)) { + u.when(() -> OntapStorageUtils.getStrategyByStoragePoolDetails(any())).thenReturn(mockStrategy); + asupManager.pushAsupForStoragePool(pool, "4.20.0", "mgmt-host", new HashSet<>()); + } + + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + + for (EmsApplicationLog msg : cap.getAllValues()) { + assertEquals(OntapStorageConstants.ASUP_EVENT_SOURCE, msg.getEventSource()); + assertEquals(OntapStorageConstants.ASUP_CATEGORY, msg.getCategory()); + assertEquals(OntapStorageConstants.ASUP_SEVERITY, msg.getSeverity()); + assertFalse(msg.getAutosupportRequired(), "autosupport_required should be false"); + assertEquals("mgmt-host", msg.getComputerName()); + } + } + + // ────────────────────────────────────────────────────────────────────────── + // Config defaults + // ────────────────────────────────────────────────────────────────────────── + + @Test + void asupIntervalSeconds_defaultIsProductionValue() { + assertEquals(String.valueOf(OntapStorageConstants.ASUP_DEFAULT_INTERVAL_SECONDS), + OntapAsupManager.AsupIntervalSeconds.defaultValue()); + } + + @Test + void asupEnabled_defaultIsTrue() { + assertEquals("true", OntapAsupManager.AsupEnabled.defaultValue()); + } + + // ────────────────────────────────────────────────────────────────────────── + // OntapAsupPollTask – self-throttle (interval change takes effect without restart) + // ────────────────────────────────────────────────────────────────────────── + + @Test + void pollTask_getDelay_returnsFixedCheckInterval() { + OntapAsupManager.OntapAsupPollTask task = asupManager.new OntapAsupPollTask(); + assertEquals(OntapAsupManager.ASUP_POLL_CHECK_INTERVAL_MS, task.getDelay()); + } + + @Test + void pollTask_skipsWhenIntervalNotElapsed() { + asupManager.lastPushTime = Instant.now(); // just pushed + OntapAsupManager.OntapAsupPollTask task = asupManager.new OntapAsupPollTask(); + task.run(); + verify(storagePoolDao, never()).findPoolsByProvider(any()); + } + + @Test + void pollTask_pushesWhenIntervalElapsed() { + asupManager.lastPushTime = Instant.EPOCH; // never pushed + when(storagePoolDao.findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME)) + .thenReturn(Collections.emptyList()); + OntapAsupManager.OntapAsupPollTask task = asupManager.new OntapAsupPollTask(); + task.run(); + verify(storagePoolDao).findPoolsByProvider(OntapStorageConstants.ONTAP_PLUGIN_NAME); + } + + // ────────────────────────────────────────────────────────────────────────── + // Utility helpers + // ────────────────────────────────────────────────────────────────────────── + + @Test + void getCloudStackVersion_returnsNonBlank() { + String v = asupManager.getCloudStackVersion(); + assertNotNull(v); + assertFalse(v.isEmpty()); + } + + @Test + void getComputerName_returnsNonEmpty() { + String host = asupManager.getComputerName(); + assertNotNull(host); + assertFalse(host.isEmpty()); + } + + @Test + void getOperatingSystem_returnsNonEmpty() { + String os = asupManager.getOperatingSystem(); + assertNotNull(os); + assertFalse(os.isEmpty()); + } + + // ────────────────────────────────────────────────────────────────────────── + // Helpers + // ────────────────────────────────────────────────────────────────────────── + + /** + * Captures and returns the event-id 1 (pool) message description. + * Expects exactly 2 messages to have been sent (heartbeat + pool). + */ + private String capturePoolMessage() { + ArgumentCaptor cap = ArgumentCaptor.forClass(EmsApplicationLog.class); + verify(mockStrategy, times(2)).sendAsupMessage(cap.capture()); + EmsApplicationLog poolMsg = cap.getAllValues().get(1); + assertEquals(OntapStorageConstants.ASUP_EVENT_ID_STORAGE_POOL, poolMsg.getEventId()); + String desc = poolMsg.getEventDescription(); + assertNotNull(desc); + return desc; + } + + /** + * Creates a mock VolumeVO with state=Ready so that CS_VOLUME_STATES filter includes it + * and getSize() is exercised (avoiding UnnecessaryStubbingException in strict mode). + */ + private VolumeVO mockVolume(long id, Long instanceId, long size) { + VolumeVO vol = mock(VolumeVO.class); + when(vol.getId()).thenReturn(id); + when(vol.getInstanceId()).thenReturn(instanceId); + when(vol.getSize()).thenReturn(size); + when(vol.getState()).thenReturn(Volume.State.Ready); + return vol; + } + + /** Creates a mock SnapshotVO with the given id, volumeId and state. */ + private SnapshotVO makeSnapshot(long id, long volumeId, Snapshot.State state) { + SnapshotVO snap = mock(SnapshotVO.class); + when(snap.getState()).thenReturn(state); + return snap; + } + + /** Creates a mock VMSnapshotVO with the given state and removed timestamp. */ + private VMSnapshotVO makeVmSnapshot(VMSnapshot.State state, java.util.Date removed) { + VMSnapshotVO vmSnap = mock(VMSnapshotVO.class); + when(vmSnap.getState()).thenReturn(state); + lenient().when(vmSnap.getRemoved()).thenReturn(removed); + return vmSnap; + } +}