LoadBalancingRulesManagerImpl.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
// 
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.network.lb;

import com.cloud.agent.api.to.LoadBalancerTO;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenter.NetworkType;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.VlanDao;
import com.cloud.domain.dao.DomainDao;
import com.cloud.event.ActionEvent;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.event.dao.EventDao;
import com.cloud.event.dao.UsageEventDao;
import com.cloud.exception.*;
import com.cloud.network.*;
import com.cloud.network.Network.Capability;
import com.cloud.network.Network.Provider;
import com.cloud.network.Network.Service;
import com.cloud.network.as.*;
import com.cloud.network.as.Condition;
import com.cloud.network.as.dao.*;
import com.cloud.network.dao.*;
import com.cloud.network.element.LoadBalancingServiceProvider;
import com.cloud.network.lb.LoadBalancingRule.*;
import com.cloud.network.rules.*;
import com.cloud.network.rules.FirewallRule.FirewallRuleType;
import com.cloud.network.rules.FirewallRule.Purpose;
import com.cloud.network.rules.LbStickinessMethod.LbStickinessMethodParam;
import com.cloud.network.vpc.VpcManager;
import com.cloud.offering.NetworkOffering;
import com.cloud.projects.Project.ListProjectResourcesCriteria;
import com.cloud.server.ResourceTag.TaggedResourceType;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.tags.ResourceTagVO;
import com.cloud.tags.dao.ResourceTagDao;
import com.cloud.user.*;
import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.UserDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.db.*;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.NetUtils;
import com.cloud.vm.Nic;
import com.cloud.vm.UserVmVO;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.dao.UserVmDao;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.user.loadbalancer.*;
import org.apache.cloudstack.api.response.ServiceResponse;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import javax.ejb.Local;
import javax.inject.Inject;
import java.security.InvalidParameterException;
import java.util.*;

@Component
@Local(value = { LoadBalancingRulesManager.class, LoadBalancingRulesService.class })
public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements LoadBalancingRulesManager,
        LoadBalancingRulesService, NetworkRuleApplier {
    private static final Logger s_logger = Logger.getLogger(LoadBalancingRulesManagerImpl.class);

    @Inject
    NetworkManager _networkMgr;
    @Inject
    NetworkModel _networkModel;
    @Inject
    RulesManager _rulesMgr;
    @Inject
    AccountManager _accountMgr;
    @Inject
    IPAddressDao _ipAddressDao;
    @Inject
    LoadBalancerDao _lbDao;
    @Inject
    VlanDao _vlanDao;
    @Inject
    EventDao _eventDao;
    @Inject
    LoadBalancerVMMapDao _lb2VmMapDao;
    @Inject
    LBStickinessPolicyDao _lb2stickinesspoliciesDao;
    @Inject
    LBHealthCheckPolicyDao _lb2healthcheckDao;
    @Inject
    UserVmDao _vmDao;
    @Inject
    AccountDao _accountDao;
    @Inject
    DomainDao _domainDao;
    @Inject
    NicDao _nicDao;
    @Inject
    UsageEventDao _usageEventDao;
    @Inject
    FirewallRulesCidrsDao _firewallCidrsDao;
    @Inject
    FirewallManager _firewallMgr;
    @Inject
    NetworkDao _networkDao;
    @Inject
    FirewallRulesDao _firewallDao;
    @Inject
    DomainService _domainMgr;
    @Inject
    ConfigurationManager _configMgr;

    @Inject
    ExternalLoadBalancerUsageManager _externalLBUsageMgr;
    @Inject
    NetworkServiceMapDao _ntwkSrvcDao;
    @Inject
    ResourceTagDao _resourceTagDao;
    @Inject
    VpcManager _vpcMgr;
    @Inject
    VMTemplateDao _templateDao;
    @Inject
    ServiceOfferingDao _offeringsDao;
    @Inject
    CounterDao _counterDao;
    @Inject
    ConditionDao _conditionDao;
    @Inject
    AutoScaleVmProfileDao _autoScaleVmProfileDao;
    @Inject
    AutoScalePolicyDao _autoScalePolicyDao;
    @Inject
    AutoScalePolicyConditionMapDao _autoScalePolicyConditionMapDao;
    @Inject
    AutoScaleVmGroupDao _autoScaleVmGroupDao;
    @Inject
    AutoScaleVmGroupPolicyMapDao _autoScaleVmGroupPolicyMapDao;
    @Inject
    ConfigurationDao _configDao;
    @Inject
    DataCenterDao _dcDao = null;
    @Inject
    UserDao _userDao;
    @Inject
    List<LoadBalancingServiceProvider> _lbProviders;

    // Will return a string. For LB Stickiness this will be a json, for
    // autoscale this will be "," separated values
    @Override
    public String getLBCapability(long networkid, String capabilityName) {
        Map<Service, Map<Capability, String>> serviceCapabilitiesMap = _networkModel.getNetworkCapabilities(networkid);
        if (serviceCapabilitiesMap != null) {
            for (Service service : serviceCapabilitiesMap.keySet()) {
                ServiceResponse serviceResponse = new ServiceResponse();
                serviceResponse.setName(service.getName());
                if ("Lb".equalsIgnoreCase(service.getName())) {
                    Map<Capability, String> serviceCapabilities = serviceCapabilitiesMap.get(service);
                    if (serviceCapabilities != null) {
                        for (Capability capability : serviceCapabilities.keySet()) {
                            if (capabilityName.equals(capability.getName())) {
                                return serviceCapabilities.get(capability);
                            }
                        }
                    }
                }
            }
        }
        return null;
    }

    private LbAutoScaleVmGroup getLbAutoScaleVmGroup(AutoScaleVmGroupVO vmGroup, String currentState, LoadBalancerVO lb) {
        long lbNetworkId = lb.getNetworkId();
        String lbName = lb.getName();
        List<AutoScaleVmGroupPolicyMapVO> vmGroupPolicyMapList = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(vmGroup
                .getId());
        List<LbAutoScalePolicy> autoScalePolicies = new ArrayList<LbAutoScalePolicy>();
        for (AutoScaleVmGroupPolicyMapVO vmGroupPolicyMap : vmGroupPolicyMapList) {
            AutoScalePolicy autoScalePolicy = _autoScalePolicyDao.findById(vmGroupPolicyMap.getPolicyId());
            List<AutoScalePolicyConditionMapVO> autoScalePolicyConditionMapList = _autoScalePolicyConditionMapDao
                    .listByAll(autoScalePolicy.getId(), null);
            List<LbCondition> lbConditions = new ArrayList<LbCondition>();
            for (AutoScalePolicyConditionMapVO autoScalePolicyConditionMap : autoScalePolicyConditionMapList) {
                Condition condition = _conditionDao.findById(autoScalePolicyConditionMap.getConditionId());
                Counter counter = _counterDao.findById(condition.getCounterid());
                lbConditions.add(new LbCondition(counter, condition));
            }
            autoScalePolicies.add(new LbAutoScalePolicy(autoScalePolicy, lbConditions));
        }
        AutoScaleVmProfile autoScaleVmProfile = _autoScaleVmProfileDao.findById(vmGroup.getProfileId());
        Long autoscaleUserId = autoScaleVmProfile.getAutoScaleUserId();
        User user = _userDao.findByIdIncludingRemoved(autoscaleUserId);
        String apiKey = user.getApiKey();
        String secretKey = user.getSecretKey();
        String csUrl = _configDao.getValue(Config.EndpointeUrl.key());
        String zoneId = _dcDao.findById(autoScaleVmProfile.getZoneId()).getUuid();
        String domainId = _domainDao.findById(autoScaleVmProfile.getDomainId()).getUuid();
        String serviceOfferingId = _offeringsDao.findById(autoScaleVmProfile.getServiceOfferingId()).getUuid();
        String templateId = _templateDao.findById(autoScaleVmProfile.getTemplateId()).getUuid();
        String vmName = "AutoScale-LB-" + lbName;
        String lbNetworkUuid = null;

        DataCenter zone = _configMgr.getZone(vmGroup.getZoneId());
        if (zone == null) {
            // This should never happen, but still a cautious check
            s_logger.warn("Unable to find zone while packaging AutoScale Vm Group, zoneid: " + vmGroup.getZoneId());
            throw new InvalidParameterValueException("Unable to find zone");
        } else {
            if (zone.getNetworkType() == NetworkType.Advanced) {
                NetworkVO lbNetwork = _networkDao.findById(lbNetworkId);
                lbNetworkUuid = lbNetwork.getUuid();
            }
        }

        if (apiKey == null) {
            throw new InvalidParameterValueException("apiKey for user: " + user.getUsername()
                    + " is empty. Please generate it");
        }

        if (secretKey == null) {
            throw new InvalidParameterValueException("secretKey for user: " + user.getUsername()
                    + " is empty. Please generate it");
        }

        if (csUrl == null || csUrl.contains("localhost")) {
            throw new InvalidParameterValueException(
                    "Global setting endpointe.url has to be set to the Management Server's API end point");
        }

        LbAutoScaleVmProfile lbAutoScaleVmProfile = new LbAutoScaleVmProfile(autoScaleVmProfile, apiKey, secretKey,
                csUrl, zoneId, domainId, serviceOfferingId, templateId, vmName, lbNetworkUuid);
        return new LbAutoScaleVmGroup(vmGroup, autoScalePolicies, lbAutoScaleVmProfile, currentState);
    }

    private boolean applyAutoScaleConfig(LoadBalancerVO lb, AutoScaleVmGroupVO vmGroup, String currentState)
            throws ResourceUnavailableException {
        LbAutoScaleVmGroup lbAutoScaleVmGroup = getLbAutoScaleVmGroup(vmGroup, currentState, lb);
        /*
         * Regular config like destinations need not be packed for applying
         * autoscale config as of today.
         */
	List<LbStickinessPolicy> policyList = getStickinessPolicies(lb.getId());
        LoadBalancingRule rule = new LoadBalancingRule(lb, null, policyList, null);
        rule.setAutoScaleVmGroup(lbAutoScaleVmGroup);

        if (!isRollBackAllowedForProvider(lb)) {
            // this is for Netscaler type of devices. if their is failure the db
            // entries will be rollbacked.
            return false;
        }

        List<LoadBalancingRule> rules = Arrays.asList(rule);

        if (!_networkMgr.applyRules(rules, FirewallRule.Purpose.LoadBalancing, this, false)) {
            s_logger.debug("LB rules' autoscale config are not completely applied");
            return false;
        }

        return true;
    }

    @Override
    @DB
    public boolean configureLbAutoScaleVmGroup(long vmGroupid, String currentState) throws ResourceUnavailableException {
        AutoScaleVmGroupVO vmGroup = _autoScaleVmGroupDao.findById(vmGroupid);
        boolean success = false;

        LoadBalancerVO loadBalancer = _lbDao.findById(vmGroup.getLoadBalancerId());

        FirewallRule.State backupState = loadBalancer.getState();

        if (vmGroup.getState().equals(AutoScaleVmGroup.State_New)) {
            loadBalancer.setState(FirewallRule.State.Add);
            _lbDao.persist(loadBalancer);
        } else if (loadBalancer.getState() == FirewallRule.State.Active
                && vmGroup.getState().equals(AutoScaleVmGroup.State_Revoke)) {
            loadBalancer.setState(FirewallRule.State.Add);
            _lbDao.persist(loadBalancer);
        }

        try {
            success = applyAutoScaleConfig(loadBalancer, vmGroup, currentState);
        } catch (ResourceUnavailableException e) {
            s_logger.warn("Unable to configure AutoScaleVmGroup to the lb rule: " + loadBalancer.getId()
                    + " because resource is unavaliable:", e);
            if (isRollBackAllowedForProvider(loadBalancer)) {
                loadBalancer.setState(backupState);
                _lbDao.persist(loadBalancer);
                s_logger.debug("LB Rollback rule id: " + loadBalancer.getId()
                        + " lb state rolback while creating AutoscaleVmGroup");
            }
            throw e;
        } finally {
            if (!success) {
                s_logger.warn("Failed to configure LB Auto Scale Vm Group with Id:" + vmGroupid);
            }
        }

        if (success) {
            if (vmGroup.getState().equals(AutoScaleVmGroup.State_New)) {
                Transaction.currentTxn().start();
                loadBalancer.setState(FirewallRule.State.Active);
                s_logger.debug("LB rule " + loadBalancer.getId() + " state is set to Active");
                _lbDao.persist(loadBalancer);
                vmGroup.setState(AutoScaleVmGroup.State_Enabled);
                _autoScaleVmGroupDao.persist(vmGroup);
                s_logger.debug("LB Auto Scale Vm Group with Id: " + vmGroupid + " is set to Enabled state.");
                Transaction.currentTxn().commit();
            }
            s_logger.info("Successfully configured LB Autoscale Vm Group with Id: " + vmGroupid);
        }
        return success;
    }

    private boolean validateHealthCheck(CreateLBHealthCheckPolicyCmd cmd) {
        LoadBalancerVO loadBalancer = _lbDao.findById(cmd.getLbRuleId());
        String capability = getLBCapability(loadBalancer.getNetworkId(), Capability.HealthCheckPolicy.getName());
        if (capability != null) {
            return true;
        }
        return false;
    }

    private boolean genericValidator(CreateLBStickinessPolicyCmd cmd) throws InvalidParameterValueException {
        LoadBalancerVO loadBalancer = _lbDao.findById(cmd.getLbRuleId());
        /* Validation : check for valid Method name and params */
        List<LbStickinessMethod> stickinessMethodList = getStickinessMethods(loadBalancer.getNetworkId());
        boolean methodMatch = false;

        if (stickinessMethodList == null) {
            throw new InvalidParameterValueException("Failed:  No Stickiness method available for LB rule:"
                    + cmd.getLbRuleId());
        }
        for (LbStickinessMethod method : stickinessMethodList) {
            if (method.getMethodName().equalsIgnoreCase(cmd.getStickinessMethodName())) {
                methodMatch = true;
                Map apiParamList = cmd.getparamList();
                List<LbStickinessMethodParam> methodParamList = method.getParamList();
                Map<String, String> tempParamList = new HashMap<String, String>();

                /*
                 * validation-1: check for any extra params that are not
                 * required by the policymethod(capability), FIXME: make the
                 * below loop simple without using raw data type
                 */
                if (apiParamList != null) {
                    Collection userGroupCollection = apiParamList.values();
                    Iterator iter = userGroupCollection.iterator();
                    while (iter.hasNext()) {
                        HashMap<String, String> paramKVpair = (HashMap) iter.next();
                        String paramName = paramKVpair.get("name");
                        String paramValue = paramKVpair.get("value");

                        tempParamList.put(paramName, paramValue);
                        Boolean found = false;
                        for (LbStickinessMethodParam param : methodParamList) {
                            if (param.getParamName().equalsIgnoreCase(paramName)) {
                                if ((param.getIsflag() == false) && (paramValue == null)) {
                                    throw new InvalidParameterValueException("Failed : Value expected for the Param :"
                                            + param.getParamName());
                                }
                                found = true;
                                break;
                            }
                        }
                        if (!found) {
                            throw new InvalidParameterValueException(
                                    "Failed : Stickiness policy does not support param name :" + paramName);
                        }
                    }
                }

                /* validation-2: check for mandatory params */
                for (LbStickinessMethodParam param : methodParamList) {
                    if (param.getRequired()) {
                        if (tempParamList.get(param.getParamName()) == null) {
                            throw new InvalidParameterValueException("Failed : Missing Manadatory Param :"
                                    + param.getParamName());
                        }
                    }
                }
                /* Successfully completed the Validation */
                break;
            }
        }
        if (methodMatch == false) {
            throw new InvalidParameterValueException("Failed to match Stickiness method name for LB rule:"
                    + cmd.getLbRuleId());
        }

        /* Validation : check for the multiple policies to the rule id */
        List<LBStickinessPolicyVO> stickinessPolicies = _lb2stickinesspoliciesDao.listByLoadBalancerId(
                cmd.getLbRuleId(), false);
        if (stickinessPolicies.size() > 0) {
            throw new InvalidParameterValueException("Failed to create Stickiness policy: Already policy attached "
                    + cmd.getLbRuleId());
        }
        return true;
    }

    @SuppressWarnings("rawtypes")
    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_LB_STICKINESSPOLICY_CREATE, eventDescription = "create lb stickinesspolicy to load balancer", create = true)
    public StickinessPolicy createLBStickinessPolicy(CreateLBStickinessPolicyCmd cmd)
            throws NetworkRuleConflictException {
        UserContext caller = UserContext.current();

        /* Validation : check corresponding load balancer rule exist */
        LoadBalancerVO loadBalancer = _lbDao.findById(cmd.getLbRuleId());
        if (loadBalancer == null) {
            throw new InvalidParameterValueException("Failed: LB rule id: " + cmd.getLbRuleId() + " not present ");
        }

        _accountMgr.checkAccess(caller.getCaller(), null, true, loadBalancer);
        if (loadBalancer.getState() == FirewallRule.State.Revoke) {
            throw new InvalidParameterValueException("Failed:  LB rule id: " + cmd.getLbRuleId()
                    + " is in deleting state: ");
        }

        /* Generic validations */
        if (!genericValidator(cmd)) {
            throw new InvalidParameterValueException("Failed to create Stickiness policy: Validation Failed "
                    + cmd.getLbRuleId());
        }

        /*
         * Specific validations using network element validator for specific
         * validations
         */
        LBStickinessPolicyVO lbpolicy = new LBStickinessPolicyVO(loadBalancer.getId(), cmd.getLBStickinessPolicyName(),
                cmd.getStickinessMethodName(), cmd.getparamList(), cmd.getDescription());
        List<LbStickinessPolicy> policyList = new ArrayList<LbStickinessPolicy>();
        policyList.add(new LbStickinessPolicy(cmd.getStickinessMethodName(), lbpolicy.getParams()));
        LoadBalancingRule lbRule = new LoadBalancingRule(loadBalancer, getExistingDestinations(lbpolicy.getId()),
                policyList, null);
        if (!validateRule(lbRule)) {
            throw new InvalidParameterValueException("Failed to create Stickiness policy: Validation Failed "
                    + cmd.getLbRuleId());
        }

        /* Finally Insert into DB */
        LBStickinessPolicyVO policy = new LBStickinessPolicyVO(loadBalancer.getId(), cmd.getLBStickinessPolicyName(),
                cmd.getStickinessMethodName(), cmd.getparamList(), cmd.getDescription());
        policy = _lb2stickinesspoliciesDao.persist(policy);

        return policy;
    }

    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_LB_HEALTHCHECKPOLICY_CREATE, eventDescription = "create load balancer health check to load balancer", create = true)
    public HealthCheckPolicy createLBHealthCheckPolicy(CreateLBHealthCheckPolicyCmd cmd) {
        UserContext caller = UserContext.current();

        /*
         * Validation of cmd Monitor interval must be greater than response
         * timeout
         */
        Map<String, String> paramMap = cmd.getFullUrlParams();

        if (paramMap.containsKey(ApiConstants.HEALTHCHECK_RESPONSE_TIMEOUT)
                && paramMap.containsKey(ApiConstants.HEALTHCHECK_INTERVAL_TIME)) {
            if (cmd.getResponsTimeOut() > cmd.getHealthCheckInterval())
                throw new InvalidParameterValueException(
                        "Failed to create HealthCheck policy : Monitor interval must be greater than response timeout");
        }
        /* Validation : check corresponding load balancer rule exist */
        LoadBalancerVO loadBalancer = _lbDao.findById(cmd.getLbRuleId());
        if (loadBalancer == null) {
            throw new InvalidParameterValueException("Failed: LB rule id: " + cmd.getLbRuleId() + " not present ");
        }

        _accountMgr.checkAccess(caller.getCaller(), null, true, loadBalancer);

        if (loadBalancer.getState() == FirewallRule.State.Revoke) {
            throw new InvalidParameterValueException("Failed:  LB rule id: " + cmd.getLbRuleId()
                    + " is in deleting state: ");
        }

        /*
         * Validate Whether LB Provider has the capabilities to support Health
         * Checks
         */
        if (!validateHealthCheck(cmd)) {
            throw new InvalidParameterValueException(
                    "Failed to create HealthCheck policy: Validation Failed (HealthCheck Policy is not supported by LB Provider for the LB rule id :)"
                            + cmd.getLbRuleId());
        }

        /* Validation : check for the multiple hc policies to the rule id */
        List<LBHealthCheckPolicyVO> hcPolicies = _lb2healthcheckDao.listByLoadBalancerId(cmd.getLbRuleId(), false);
        if (hcPolicies.size() > 0) {
            throw new InvalidParameterValueException(
                    "Failed to create HealthCheck policy: Already policy attached  for the LB Rule id :"
                            + cmd.getLbRuleId());
        }
        /*
         * Specific validations using network element validator for specific
         * validations
         */
        LBHealthCheckPolicyVO hcpolicy = new LBHealthCheckPolicyVO(loadBalancer.getId(), cmd.getPingPath(),
                cmd.getDescription(), cmd.getResponsTimeOut(), cmd.getHealthCheckInterval(), cmd.getHealthyThreshold(),
                cmd.getUnhealthyThreshold());

        List<LbHealthCheckPolicy> hcPolicyList = new ArrayList<LbHealthCheckPolicy>();
        hcPolicyList.add(new LbHealthCheckPolicy(hcpolicy.getpingpath(), hcpolicy.getDescription(), hcpolicy
                .getResponseTime(), hcpolicy.getHealthcheckInterval(), hcpolicy.getHealthcheckThresshold(), hcpolicy
                .getUnhealthThresshold()));

        // Finally Insert into DB
        LBHealthCheckPolicyVO policy = new LBHealthCheckPolicyVO(loadBalancer.getId(), cmd.getPingPath(),
                cmd.getDescription(), cmd.getResponsTimeOut(), cmd.getHealthCheckInterval(), cmd.getHealthyThreshold(),
                cmd.getUnhealthyThreshold());

        policy = _lb2healthcheckDao.persist(policy);
        return policy;
    }

    private boolean validateRule(LoadBalancingRule lbRule) {
        Network network = _networkDao.findById(lbRule.getNetworkId());
        Purpose purpose = lbRule.getPurpose();
        if (purpose != Purpose.LoadBalancing) {
            s_logger.debug("Unable to validate network rules for purpose: " + purpose.toString());
            return false;
        }
        for (LoadBalancingServiceProvider ne : _lbProviders) {
            boolean validated = ne.validateLBRule(network, lbRule);
            if (!validated)
                return false;
        }
        return true;
    }

    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_LB_STICKINESSPOLICY_CREATE, eventDescription = "Apply Stickinesspolicy to load balancer ", async = true)
    public boolean applyLBStickinessPolicy(CreateLBStickinessPolicyCmd cmd) {
        boolean success = true;

        LoadBalancerVO loadBalancer = _lbDao.findById(cmd.getLbRuleId());
        if (loadBalancer == null) {
            throw new InvalidParameterException("Invalid Load balancer Id:" + cmd.getLbRuleId());
        }
        FirewallRule.State backupState = loadBalancer.getState();
        loadBalancer.setState(FirewallRule.State.Add);
        _lbDao.persist(loadBalancer);
        try {
            applyLoadBalancerConfig(cmd.getLbRuleId());
        } catch (ResourceUnavailableException e) {
            s_logger.warn("Unable to apply Stickiness policy to the lb rule: " + cmd.getLbRuleId()
                    + " because resource is unavaliable:", e);
            if (isRollBackAllowedForProvider(loadBalancer)) {
                loadBalancer.setState(backupState);
                _lbDao.persist(loadBalancer);
                s_logger.debug("LB Rollback rule id: " + loadBalancer.getId()
                        + " lb state rolback while creating sticky policy");
            }
            deleteLBStickinessPolicy(cmd.getEntityId(), false);
            success = false;
        }

        return success;
    }

    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_LB_HEALTHCHECKPOLICY_CREATE, eventDescription = "Apply HealthCheckPolicy to load balancer ", async = true)
    public boolean applyLBHealthCheckPolicy(CreateLBHealthCheckPolicyCmd cmd) {
        boolean success = true;

        LoadBalancerVO loadBalancer = _lbDao.findById(cmd.getLbRuleId());
        if (loadBalancer == null) {
            throw new InvalidParameterException("Invalid Load balancer Id:" + cmd.getLbRuleId());
        }
        FirewallRule.State backupState = loadBalancer.getState();
        loadBalancer.setState(FirewallRule.State.Add);
        _lbDao.persist(loadBalancer);
        try {
            applyLoadBalancerConfig(cmd.getLbRuleId());
        } catch (ResourceUnavailableException e) {
            s_logger.warn("Unable to apply healthcheck policy to the lb rule: " + cmd.getLbRuleId()
                    + " because resource is unavaliable:", e);
            if (isRollBackAllowedForProvider(loadBalancer)) {
                loadBalancer.setState(backupState);
                _lbDao.persist(loadBalancer);
                s_logger.debug("LB Rollback rule id: " + loadBalancer.getId()
                        + " lb state rolback while creating healthcheck policy");
            }
            deleteLBHealthCheckPolicy(cmd.getEntityId(), false);
            success = false;
        }
        return success;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_LB_STICKINESSPOLICY_DELETE, eventDescription = "revoking LB Stickiness policy ", async = true)
    public boolean deleteLBStickinessPolicy(long stickinessPolicyId, boolean apply) {
        boolean success = true;

        UserContext caller = UserContext.current();
        LBStickinessPolicyVO stickinessPolicy = _lb2stickinesspoliciesDao.findById(stickinessPolicyId);

        if (stickinessPolicy == null) {
            throw new InvalidParameterException("Invalid Stickiness policy id value: " + stickinessPolicyId);
        }
        LoadBalancerVO loadBalancer = _lbDao.findById(Long.valueOf(stickinessPolicy.getLoadBalancerId()));
        if (loadBalancer == null) {
            throw new InvalidParameterException("Invalid Load balancer : " + stickinessPolicy.getLoadBalancerId()
                    + " for Stickiness policy id: " + stickinessPolicyId);
        }
        long loadBalancerId = loadBalancer.getId();
        FirewallRule.State backupState = loadBalancer.getState();
        _accountMgr.checkAccess(caller.getCaller(), null, true, loadBalancer);

        if (apply) {
            if (loadBalancer.getState() == FirewallRule.State.Active) {
                loadBalancer.setState(FirewallRule.State.Add);
                _lbDao.persist(loadBalancer);
            }

            boolean backupStickyState = stickinessPolicy.isRevoke();
            stickinessPolicy.setRevoke(true);
            _lb2stickinesspoliciesDao.persist(stickinessPolicy);
            s_logger.debug("Set load balancer rule for revoke: rule id " + loadBalancerId + ", stickinesspolicyID "
                    + stickinessPolicyId);

            try {
                if (!applyLoadBalancerConfig(loadBalancerId)) {
                    s_logger.warn("Failed to remove load balancer rule id " + loadBalancerId
                            + " for stickinesspolicyID " + stickinessPolicyId);
                    throw new CloudRuntimeException("Failed to remove load balancer rule id " + loadBalancerId
                            + " for stickinesspolicyID " + stickinessPolicyId);
                }
            } catch (ResourceUnavailableException e) {
                if (isRollBackAllowedForProvider(loadBalancer)) {
                    stickinessPolicy.setRevoke(backupStickyState);
                    _lb2stickinesspoliciesDao.persist(stickinessPolicy);
                    loadBalancer.setState(backupState);
                    _lbDao.persist(loadBalancer);
                    s_logger.debug("LB Rollback rule id: " + loadBalancer.getId() + "  while deleting sticky policy: "
                            + stickinessPolicyId);
                }
                s_logger.warn("Unable to apply the load balancer config because resource is unavaliable.", e);
                success = false;
            }
        } else {
            _lb2stickinesspoliciesDao.remove(stickinessPolicy.getLoadBalancerId());
        }
        return success;
    }

    @DB
    @Override
    @ActionEvent(eventType = EventTypes.EVENT_LB_HEALTHCHECKPOLICY_DELETE, eventDescription = "revoking LB HealthCheck policy ", async = true)
    public boolean deleteLBHealthCheckPolicy(long healthCheckPolicyId, boolean apply) {
        boolean success = true;

        UserContext caller = UserContext.current();
        LBHealthCheckPolicyVO healthCheckPolicy = _lb2healthcheckDao.findById(healthCheckPolicyId);

        if (healthCheckPolicy == null) {
            throw new InvalidParameterException("Invalid HealthCheck policy id value: " + healthCheckPolicyId);
        }
        LoadBalancerVO loadBalancer = _lbDao.findById(Long.valueOf(healthCheckPolicy.getLoadBalancerId()));
        if (loadBalancer == null) {
            throw new InvalidParameterException("Invalid Load balancer : " + healthCheckPolicy.getLoadBalancerId()
                    + " for HealthCheck policy id: " + healthCheckPolicyId);
        }
        long loadBalancerId = loadBalancer.getId();
        FirewallRule.State backupState = loadBalancer.getState();
        _accountMgr.checkAccess(caller.getCaller(), null, true, loadBalancer);

        if (apply) {
            if (loadBalancer.getState() == FirewallRule.State.Active) {
                loadBalancer.setState(FirewallRule.State.Add);
                _lbDao.persist(loadBalancer);
            }

            boolean backupStickyState = healthCheckPolicy.isRevoke();
            healthCheckPolicy.setRevoke(true);
            _lb2healthcheckDao.persist(healthCheckPolicy);
            s_logger.debug("Set health check policy to revoke for loadbalancing rule id : " + loadBalancerId
                    + ", healthCheckpolicyID " + healthCheckPolicyId);

            // removing the state of services set by the monitor.
            List<LoadBalancerVMMapVO> maps = _lb2VmMapDao.listByLoadBalancerId(loadBalancerId);
            if (maps != null) {
                Transaction txn = Transaction.currentTxn();
                txn.start();
                s_logger.debug("Resetting health state policy for services in loadbalancing rule id : "
                        + loadBalancerId);
                for (LoadBalancerVMMapVO map : maps) {
                    map.setState(null);
                    _lb2VmMapDao.persist(map);
                }
                txn.commit();
            }

            try {
                if (!applyLoadBalancerConfig(loadBalancerId)) {
                    s_logger.warn("Failed to remove load balancer rule id " + loadBalancerId
                            + " for healthCheckpolicyID " + healthCheckPolicyId);
                    throw new CloudRuntimeException("Failed to remove load balancer rule id " + loadBalancerId
                            + " for healthCheckpolicyID " + healthCheckPolicyId);
                }
            } catch (ResourceUnavailableException e) {
                if (isRollBackAllowedForProvider(loadBalancer)) {
                    healthCheckPolicy.setRevoke(backupStickyState);
                    _lb2healthcheckDao.persist(healthCheckPolicy);
                    loadBalancer.setState(backupState);
                    _lbDao.persist(loadBalancer);
                    s_logger.debug("LB Rollback rule id: " + loadBalancer.getId()
                            + "  while deleting healthcheck policy: " + healthCheckPolicyId);
                }
                s_logger.warn("Unable to apply the load balancer config because resource is unavaliable.", e);
                success = false;
            }
        } else {
            _lb2healthcheckDao.remove(healthCheckPolicy.getLoadBalancerId());
        }
        return success;
    }

    // This method will check the status of services which has monitors created
    // by CloudStack and update them in lbvmmap table
    @DB
    @Override
    public void updateLBHealthChecks() throws ResourceUnavailableException {
        List<LoadBalancerVO> rules = _lbDao.listAll();
        List<NetworkVO> networks = _networkDao.listAll();
        List<LoadBalancerTO> stateRules = null;
        boolean isHandled = false;
        for (NetworkVO ntwk : networks) {
            Network network = _networkDao.findById(ntwk.getId());
            String capability = getLBCapability(network.getId(), Capability.HealthCheckPolicy.getName());

            if (capability != null && capability.equalsIgnoreCase("true")) {
                /*
                 * s_logger.debug(
                 * "HealthCheck Manager :: LB Provider in the Network has the Healthcheck policy capability :: "
                 * + provider.get(0).getName());
                 */
                rules = _lbDao.listByNetworkId(network.getId());
                if (rules != null && rules.size() > 0) {
                    List<LoadBalancingRule> lbrules = new ArrayList<LoadBalancingRule>();
                    for (LoadBalancerVO lb : rules) {
                        List<LbDestination> dstList = getExistingDestinations(lb.getId());
                        List<LbHealthCheckPolicy> hcPolicyList = getHealthCheckPolicies(lb.getId());
                        // adding to lbrules list only if the LB rule
                        // hashealtChecks
                        if (hcPolicyList != null && hcPolicyList.size() > 0) {
                            LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, null, hcPolicyList);
                            lbrules.add(loadBalancing);
                        }
                    }
                    if (lbrules.size() > 0) {
                        isHandled = false;
                        for (LoadBalancingServiceProvider lbElement : _lbProviders) {
                            stateRules = lbElement.updateHealthChecks(network, (List<LoadBalancingRule>) lbrules);
                            if (stateRules != null && stateRules.size() > 0) {
                                for (LoadBalancerTO lbto : stateRules) {
                                    LoadBalancerVO ulb = _lbDao.findByUuid(lbto.getUuid());
                                    List<LoadBalancerVMMapVO> lbVmMaps = _lb2VmMapDao.listByLoadBalancerId(ulb.getId());
                                    for (LoadBalancerVMMapVO lbVmMap : lbVmMaps) {
                                        UserVm vm = _vmDao.findById(lbVmMap.getInstanceId());
                                        Nic nic = _nicDao.findByInstanceIdAndNetworkIdIncludingRemoved(
                                                ulb.getNetworkId(), vm.getId());
                                        String dstIp = nic.getIp4Address();
                                        for (int i = 0; i < lbto.getDestinations().length; i++) {
                                            LoadBalancerTO.DestinationTO des = lbto.getDestinations()[i];
                                            if (dstIp.equalsIgnoreCase(lbto.getDestinations()[i].getDestIp())) {
                                                lbVmMap.setState(des.getMonitorState());
                                                _lb2VmMapDao.persist(lbVmMap);
                                                s_logger.debug("Updating the LB VM Map table with the service state");
                                            }
                                        }
                                    }
                                }
                                isHandled = true;
                            }
                            if (isHandled) {
                                break;
                            }
                        }
                    }
                }
            } else {
                // s_logger.debug("HealthCheck Manager :: LB Provider in the Network DNOT the Healthcheck policy capability ");
            }
        }
    }

    private boolean isRollBackAllowedForProvider(LoadBalancerVO loadBalancer) {
        Network network = _networkDao.findById(loadBalancer.getNetworkId());
        List<Provider> provider = _networkMgr.getProvidersForServiceInNetwork(network, Service.Lb);
        if (provider == null || provider.size() == 0) {
            return false;
        }
        if (provider.get(0) == Provider.Netscaler || provider.get(0) == Provider.F5BigIp) {
            return true;
        }
        return false;
    }

    @Override
    @DB
    @ActionEvent(eventType = EventTypes.EVENT_ASSIGN_TO_LOAD_BALANCER_RULE, eventDescription = "assigning to load balancer", async = true)
    public boolean assignToLoadBalancer(long loadBalancerId, List<Long> instanceIds) {
        UserContext ctx = UserContext.current();
        Account caller = ctx.getCaller();

        LoadBalancerVO loadBalancer = _lbDao.findById(loadBalancerId);
        if (loadBalancer == null) {
            throw new InvalidParameterValueException("Failed to assign to load balancer " + loadBalancerId
                    + ", the load balancer was not found.");
        }

        List<LoadBalancerVMMapVO> mappedInstances = _lb2VmMapDao.listByLoadBalancerId(loadBalancerId, false);
        Set<Long> mappedInstanceIds = new HashSet<Long>();
        for (LoadBalancerVMMapVO mappedInstance : mappedInstances) {
            mappedInstanceIds.add(Long.valueOf(mappedInstance.getInstanceId()));
        }

        List<UserVm> vmsToAdd = new ArrayList<UserVm>();

        for (Long instanceId : instanceIds) {
            if (mappedInstanceIds.contains(instanceId)) {
                throw new InvalidParameterValueException("VM " + instanceId + " is already mapped to load balancer.");
            }

            UserVm vm = _vmDao.findById(instanceId);
            if (vm == null || vm.getState() == State.Destroyed || vm.getState() == State.Expunging) {
                InvalidParameterValueException ex = new InvalidParameterValueException("Invalid instance id specified");
                ex.addProxyObject(vm, instanceId, "instanceId");
                throw ex;
            }

            _rulesMgr.checkRuleAndUserVm(loadBalancer, vm, caller);

            if (vm.getAccountId() != loadBalancer.getAccountId()) {
                throw new PermissionDeniedException("Cannot add virtual machines that do not belong to the same owner.");
            }

            // Let's check to make sure the vm has a nic in the same network as
            // the load balancing rule.
            List<? extends Nic> nics = _networkModel.getNics(vm.getId());
            Nic nicInSameNetwork = null;
            for (Nic nic : nics) {
                if (nic.getNetworkId() == loadBalancer.getNetworkId()) {
                    nicInSameNetwork = nic;
                    break;
                }
            }

            if (nicInSameNetwork == null) {
                InvalidParameterValueException ex = new InvalidParameterValueException("VM " + instanceId
                        + " cannot be added because it doesn't belong in the same network.");
                ex.addProxyObject(vm, instanceId, "instanceId");
                throw ex;
            }

            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Adding " + vm + " to the load balancer pool");
            }
            vmsToAdd.add(vm);
        }

        Transaction txn = Transaction.currentTxn();
        txn.start();
        for (UserVm vm : vmsToAdd) {
            LoadBalancerVMMapVO map = new LoadBalancerVMMapVO(loadBalancer.getId(), vm.getId(), false);
            map = _lb2VmMapDao.persist(map);
        }
        txn.commit();
        if (_autoScaleVmGroupDao.isAutoScaleLoadBalancer(loadBalancerId)) {
            // For autoscaled loadbalancer, the rules need not be applied,
            // meaning the call need not reach the resource layer.
            // We can consider the job done.
            return true;
        }
        boolean success = false;
        FirewallRule.State backupState = loadBalancer.getState();
        try {
            loadBalancer.setState(FirewallRule.State.Add);
            _lbDao.persist(loadBalancer);
            applyLoadBalancerConfig(loadBalancerId);
            success = true;
        } catch (ResourceUnavailableException e) {
            if (isRollBackAllowedForProvider(loadBalancer)) {
                List<Long> vmInstanceIds = new ArrayList<Long>();
                txn = Transaction.currentTxn();
                txn.start();
                for (UserVm vm : vmsToAdd) {
                    vmInstanceIds.add(vm.getId());
                }
                txn.commit();
                if (!vmInstanceIds.isEmpty()) {
                    _lb2VmMapDao.remove(loadBalancer.getId(), vmInstanceIds, null);
                    s_logger.debug("LB Rollback rule id: " + loadBalancer.getId() + "  while attaching VM: "
                            + vmInstanceIds);
                }
                loadBalancer.setState(backupState);
                _lbDao.persist(loadBalancer);
            }
            s_logger.warn("Unable to apply the load balancer config because resource is unavaliable.", e);
        }

        if (!success) {
            CloudRuntimeException ex = new CloudRuntimeException("Failed to add specified loadbalancerruleid for vms "
                    + instanceIds);
            ex.addProxyObject(loadBalancer, loadBalancerId, "loadBalancerId");
            // TBD: Also pack in the instanceIds in the exception using the
            // right VO object or table name.
            throw ex;
        }

        return success;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_REMOVE_FROM_LOAD_BALANCER_RULE, eventDescription = "removing from load balancer", async = true)
    public boolean removeFromLoadBalancer(long loadBalancerId, List<Long> instanceIds) {
        return removeFromLoadBalancerInternal(loadBalancerId, instanceIds, true);
    }

    private boolean removeFromLoadBalancerInternal(long loadBalancerId, List<Long> instanceIds, boolean rollBack) {
        UserContext caller = UserContext.current();

        LoadBalancerVO loadBalancer = _lbDao.findById(Long.valueOf(loadBalancerId));
        if (loadBalancer == null) {
            throw new InvalidParameterException("Invalid load balancer value: " + loadBalancerId);
        }

        _accountMgr.checkAccess(caller.getCaller(), null, true, loadBalancer);

        boolean success = false;
        FirewallRule.State backupState = loadBalancer.getState();
        try {
            loadBalancer.setState(FirewallRule.State.Add);
            _lbDao.persist(loadBalancer);

            for (long instanceId : instanceIds) {
                LoadBalancerVMMapVO map = _lb2VmMapDao.findByLoadBalancerIdAndVmId(loadBalancerId, instanceId);
                map.setRevoke(true);
                _lb2VmMapDao.persist(map);
                s_logger.debug("Set load balancer rule for revoke: rule id " + loadBalancerId + ", vmId " + instanceId);
            }

            if (_autoScaleVmGroupDao.isAutoScaleLoadBalancer(loadBalancerId)) {
                // For autoscaled loadbalancer, the rules need not be applied,
                // meaning the call need not reach the resource layer.
                // We can consider the job done and only need to remove the
                // rules in DB
                _lb2VmMapDao.remove(loadBalancer.getId(), instanceIds, null);
                return true;
            }

            if (!applyLoadBalancerConfig(loadBalancerId)) {
                s_logger.warn("Failed to remove load balancer rule id " + loadBalancerId + " for vms " + instanceIds);
                CloudRuntimeException ex = new CloudRuntimeException(
                        "Failed to remove specified load balancer rule id for vms " + instanceIds);
                ex.addProxyObject(loadBalancer, loadBalancerId, "loadBalancerId");
                throw ex;
            }
            success = true;
        } catch (ResourceUnavailableException e) {
            if (rollBack && isRollBackAllowedForProvider(loadBalancer)) {

                for (long instanceId : instanceIds) {
                    LoadBalancerVMMapVO map = _lb2VmMapDao.findByLoadBalancerIdAndVmId(loadBalancerId, instanceId);
                    map.setRevoke(false);
                    _lb2VmMapDao.persist(map);
                    s_logger.debug("LB Rollback rule id: " + loadBalancerId + ",while removing vmId " + instanceId);
                }

                loadBalancer.setState(backupState);
                _lbDao.persist(loadBalancer);