Skip to content

Rewrite of email alert about VR health checks #14403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
erikbocks wants to merge 2 commits into
base: main
Choose a base branch
from
+25 −17
Open

Rewrite of email alert about VR health checks #14403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6838f62
Rewrite of email alert about VR health checks
erikbocks Sep 28, 2026
575a871
Merge remote-tracking branch 'origin/main' into rewrite-vr-alerts
erikbocks Nov 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public RouterHealthCheckResultVO getRouterHealthCheckResult(long routerId, Strin
SearchCriteria<RouterHealthCheckResultVO> sc = RouterChecksSearchBuilder.create();
sc.setParameters("routerId", routerId);
sc.setParameters("checkName", checkName);
sc.setParameters("checkType", checkType);
sc.setParametersIfNotNull("checkType", checkType);
List<RouterHealthCheckResultVO> checks = listBy(sc);
if (checks.size() > 1) {
logger.error("Found multiple entries for router Id: " + routerId + ", check name: " + checkName);
Expand Down
40 changes: 24 additions & 16 deletions server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import com.google.gson.reflect.TypeToken;

import org.apache.cloudstack.acl.ApiKeyPairVO;
import org.apache.cloudstack.alert.AlertService;
import org.apache.cloudstack.alert.AlertService.AlertType;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
Expand Down Expand Up @@ -861,7 +860,7 @@ protected void updateSite2SiteVpnConnectionState(final List<DomainRouterVO> rout
"Site-to-site Vpn Connection to %s on router %s(%s) just switched from %s to %s",
gw.getName(), router.getHostName(), router, oldState, conn.getState());
logger.info(context);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
}
}
} finally {
Expand Down Expand Up @@ -924,7 +923,7 @@ protected void updateRoutersRedundantState(final List<DomainRouterVO> routers) {
router, router.getHostName(), prevState, currState);
logger.info(context);
if (currState == RedundantState.PRIMARY) {
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
if (router.getVpcId() != null && vpcManager.isProviderSupportServiceInVpc(router.getVpcId(), Service.SourceNat, Provider.Netris)) {
// re-push VPC VR static NAT/VPN routes so Netris picks up the new PRIMARY
vpcManager.reconfigStaticNatForVpcVr(router.getVpcId());
Expand Down Expand Up @@ -1011,7 +1010,7 @@ private void checkDuplicatePrimary(final List<DomainRouterVO> routers) {
"with hostname %s are both in PRIMARY state! " +
"If the problem persist, restart both of routers. ",
router, router.getHostName(), dupRouter, dupRouter.getHostName());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), title, context);
logger.warn(context);
} else {
networkRouterMaps.put(routerGuestNtwkId, router);
Expand Down Expand Up @@ -1169,23 +1168,33 @@ private void handleFailingChecks(DomainRouterVO router, List<String> failingChec
if (CollectionUtils.isEmpty(failingChecks)) {
return;
}
String subject = String.format("Failed health checks on router [%s]", router.getName());
String alertMessage = String.format("The following health checks have failed on router [%s] with UUID [%s]: ", router.getName(), router.getUuid());
ArrayList<String> failedChecks = new ArrayList<>();
for (String failedCheckName : failingChecks) {
RouterHealthCheckResultVO routerHealthCheckResultVO = routerHealthCheckResultDao.getRouterHealthCheckResult(router.getId(), failedCheckName, null);
String failedCheckDetails = routerHealthCheckResultVO.getParsedCheckDetails();

@Damans227 Damans227 Oct 14, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens if there is no saved result for a failing check? looks like it crashes here and the router recreate and the other routers get skipped

@erikbocks erikbocks Nov 13, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hello, @Damans227

I checked the behavior that you mentioned about no saved result for a failing check and could not find a flow where this happens.

In the code, the router checks are parsed and saved to the database in the getFailingChecks method, before this piece of code. Even in scenarios where the command is not received by the router, or the router's file system is not writable, there is a separate handling to tackle those. This means that, even if the router does not have any check yet, the new checks will be persisted before the failing check handling is done.

To validate this, I created a VR locally and filled its volume with a dummy file created by dd, for the disk_space_check.py test to fail. With no tests registered yet, I requested fresh checks and validated with the IDE's debugging tool that all the checks were persisted before the handling part.

Have you seen any specific scenario of this happening, so we can test this solution and validate its behavior?

failedCheckDetails = failedCheckDetails.replace("\n", " ");
String failedCheck = failedCheckName + ": " + failedCheckDetails;
failedChecks.add(failedCheck);
}
alertMessage = alertMessage + failedChecks;

String alertMessage = String.format("Health checks failed: %d failing checks on router %s / %s", failingChecks.size(), router.getName(), router.getUuid());
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(),
alertMessage, alertMessage);
logger.warn(alertMessage + ". Checking failed health checks to see if router needs recreate");
subject, alertMessage);
logger.warn("{}. Checking failed health checks to see if router needs recreation", alertMessage);

String checkFailsToRecreateVr = RouterHealthChecksFailuresToRecreateVr.valueIn(router.getDataCenterId());
StringBuilder failingChecksEvent = new StringBuilder();
boolean recreateRouter = false;
for (int i = 0; i < failingChecks.size(); i++) {
String failedCheck = failingChecks.get(i);
if (i == 0) {
failingChecksEvent.append("Router ")
failingChecksEvent.append("Router [")
.append(router.getName())
.append(" / ")
.append("] with UUID [")
.append(router.getUuid())
.append(" has failing checks: ");
.append("] has failing checks: ");
}

failingChecksEvent.append(failedCheck);
Expand All @@ -1202,8 +1211,7 @@ private void handleFailingChecks(DomainRouterVO router, List<String> failingChec
Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS, failingChecksEvent.toString(), router.getId(), ApiCommandResourceType.DomainRouter.toString());

if (recreateRouter) {
logger.warn("Health Check Alert: Found failing checks in " +
RouterHealthChecksFailuresToRecreateVrCK + ", attempting recreating router.");
logger.warn("Health Check Alert: Found failing checks in [{}], attempting to recreate router with id [{}].", RouterHealthChecksFailuresToRecreateVrCK, router.getId());
recreateRouter(router);
}
}
Expand Down Expand Up @@ -2312,7 +2320,7 @@ private Provider getVrProvider(DomainRouterVO router) {
if (vrProvider == null) {
throw new CloudRuntimeException("Cannot find related virtual router provider of router: " + router.getHostName());
}
final Provider provider = Network.Provider.getProvider(vrProvider.getType().toString());
final Provider provider = Provider.getProvider(vrProvider.getType().toString());
if (provider == null) {
throw new CloudRuntimeException("Cannot find related provider of virtual router provider: " + vrProvider.getType().toString());
}
Expand Down Expand Up @@ -2722,7 +2730,7 @@ protected void finalizeIpAssocForNetwork(final Commands cmds, final VirtualRoute
}

protected ArrayList<? extends PublicIpAddress> getPublicIpsToApply(final Provider provider, final Long guestNetworkId,
final com.cloud.network.IpAddress.State... skipInStates) {
final IpAddress.State... skipInStates) {

final List<? extends IpAddress> userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null);

Expand Down Expand Up @@ -2770,7 +2778,7 @@ public boolean finalizeStart(final VirtualMachineProfile profile, final long hos
final String errorMessage = "Command: " + cmdClassName + " failed while starting virtual router";
final String errorDetails = "Details: " + answer.getDetails() + " " + answer;
// add alerts for the failed commands
_alertMgr.sendAlert(AlertService.AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), errorMessage, errorDetails);
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), errorMessage, errorDetails);
logger.error(answer.getDetails());
logger.warn(errorMessage);
// Stop the router if any of the commands failed
Expand Down Expand Up @@ -3189,7 +3197,7 @@ public <T extends VirtualRouter> void collectNetworkStatistics(final T router, f
continue;
}
if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest
&& network.getGuestType() == Network.GuestType.Isolated) {
&& network.getGuestType() == GuestType.Isolated) {
final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address());
final String routerType = router.getType().toString();
final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(),
Expand Down
Loading