TemplateManagerImpl.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.template;

import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.ejb.Local;
import javax.inject.Inject;
import javax.naming.ConfigurationException;

import org.apache.log4j.Logger;

import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd;
import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd;
import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd;
import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd;
import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd;
import org.apache.cloudstack.api.command.user.iso.ListIsoPermissionsCmd;
import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd;
import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd;
import org.apache.cloudstack.api.command.user.iso.UpdateIsoPermissionsCmd;
import org.apache.cloudstack.api.command.user.template.CopyTemplateCmd;
import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd;
import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd;
import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd;
import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.storage.command.DettachCommand;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.to.TemplateObjectTO;

import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.ComputeChecksumCommand;
import com.cloud.agent.api.storage.DestroyCommand;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.query.dao.UserVmJoinDao;
import com.cloud.api.query.vo.UserVmJoinVO;
import com.cloud.configuration.Config;
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.domain.Domain;
import com.cloud.domain.dao.DomainDao;
import com.cloud.event.ActionEvent;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.event.UsageEventVO;
import com.cloud.event.dao.UsageEventDao;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.projects.Project;
import com.cloud.projects.ProjectManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.GuestOSVO;
import com.cloud.storage.LaunchPermissionVO;
import com.cloud.storage.Snapshot;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.StoragePoolStatus;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.Upload;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.GuestOSDao;
import com.cloud.storage.dao.LaunchPermissionDao;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.template.TemplateAdapter.TemplateAdapterType;
import com.cloud.template.VirtualMachineTemplate.BootloaderType;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountService;
import com.cloud.user.AccountVO;
import com.cloud.user.ResourceLimitService;
import com.cloud.user.User;
import com.cloud.user.dao.AccountDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.EnumUtils;
import com.cloud.utils.Pair;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.UserVmVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.VirtualMachineProfile;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDao;

@Local(value = {TemplateManager.class, TemplateApiService.class})
public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable {
    private final static Logger s_logger = Logger.getLogger(TemplateManagerImpl.class);

    @Inject
    private VMTemplateDao _tmpltDao;
    @Inject
    private TemplateDataStoreDao _tmplStoreDao;
    @Inject
    private VMTemplatePoolDao _tmpltPoolDao;
    @Inject
    private VMTemplateZoneDao _tmpltZoneDao;
    @Inject
    private VMInstanceDao _vmInstanceDao;
    @Inject
    private PrimaryDataStoreDao _poolDao;
    @Inject
    private StoragePoolHostDao _poolHostDao;
    @Inject
    private AccountDao _accountDao;
    @Inject
    private AgentManager _agentMgr;
    @Inject
    private AccountManager _accountMgr;
    @Inject
    private HostDao _hostDao;
    @Inject
    private DataCenterDao _dcDao;
    @Inject
    private UserVmDao _userVmDao;
    @Inject
    private VolumeDao _volumeDao;
    @Inject
    private SnapshotDao _snapshotDao;
    @Inject
    private ConfigurationDao _configDao;
    @Inject
    private DomainDao _domainDao;
    @Inject
    private GuestOSDao _guestOSDao;
    @Inject
    private StorageManager _storageMgr;
    @Inject
    private UsageEventDao _usageEventDao;
    @Inject
    private AccountService _accountService;
    @Inject
    private ResourceLimitService _resourceLimitMgr;
    @Inject
    private LaunchPermissionDao _launchPermissionDao;
    @Inject
    private ProjectManager _projectMgr;
    @Inject
    private VolumeDataFactory _volFactory;
    @Inject
    private TemplateDataFactory _tmplFactory;
    @Inject
    private SnapshotDataFactory _snapshotFactory;
    @Inject
    private TemplateService _tmpltSvr;
    @Inject
    private DataStoreManager _dataStoreMgr;
    @Inject
    private VolumeOrchestrationService _volumeMgr;
    @Inject
    private EndPointSelector _epSelector;
    @Inject
    private UserVmJoinDao _userVmJoinDao;

    @Inject
    MessageBus _messageBus;

    private boolean _disableExtraction = false;
    private ExecutorService _preloadExecutor;

    private List<TemplateAdapter> _adapters;

    @Inject
    private StorageCacheManager cacheMgr;
    @Inject
    private EndPointSelector selector;

    private TemplateAdapter getAdapter(HypervisorType type) {
        TemplateAdapter adapter = null;
        if (type == HypervisorType.BareMetal) {
            adapter = AdapterBase.getAdapterByName(_adapters, TemplateAdapterType.BareMetal.getName());
        } else {
            // see HypervisorTemplateAdapter
            adapter = AdapterBase.getAdapterByName(_adapters, TemplateAdapterType.Hypervisor.getName());
        }

        if (adapter == null) {
            throw new CloudRuntimeException("Cannot find template adapter for " + type.toString());
        }

        return adapter;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_ISO_CREATE, eventDescription = "creating iso")
    public VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws ResourceAllocationException {
        TemplateAdapter adapter = getAdapter(HypervisorType.None);
        TemplateProfile profile = adapter.prepare(cmd);
        VMTemplateVO template = adapter.create(profile);

        if (template != null) {
            return template;
        } else {
            throw new CloudRuntimeException("Failed to create ISO");
        }
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating template")
    public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws URISyntaxException, ResourceAllocationException {
        Account account = CallContext.current().getCallingAccount();
        if (cmd.getTemplateTag() != null) {
            if (!_accountService.isRootAdmin(account.getId())) {
                throw new PermissionDeniedException("Parameter templatetag can only be specified by a Root Admin, permission denied");
            }
        }
        if (cmd.isRoutingType() != null) {
            if (!_accountService.isRootAdmin(account.getId())) {
                throw new PermissionDeniedException("Parameter isrouting can only be specified by a Root Admin, permission denied");
            }
        }

        TemplateAdapter adapter = getAdapter(HypervisorType.getType(cmd.getHypervisor()));
        TemplateProfile profile = adapter.prepare(cmd);
        VMTemplateVO template = adapter.create(profile);

        if (template != null) {
            return template;
        } else {
            throw new CloudRuntimeException("Failed to create a template");
        }
    }

    @Override
    public DataStore getImageStore(String storeUuid, Long zoneId) {
        DataStore imageStore = null;
        if (storeUuid != null) {
            imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image);
        } else {
            imageStore = _dataStoreMgr.getImageStore(zoneId);
            if (imageStore == null) {
                throw new CloudRuntimeException("cannot find an image store for zone " + zoneId);
            }
        }

        return imageStore;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_ISO_EXTRACT, eventDescription = "extracting ISO", async = true)
    public String extract(ExtractIsoCmd cmd) {
        Account account = CallContext.current().getCallingAccount();
        Long templateId = cmd.getId();
        Long zoneId = cmd.getZoneId();
        String url = cmd.getUrl();
        String mode = cmd.getMode();
        Long eventId = cmd.getStartEventId();

        return extract(account, templateId, url, zoneId, mode, eventId, true);
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_EXTRACT, eventDescription = "extracting template", async = true)
    public String extract(ExtractTemplateCmd cmd) {
        Account caller = CallContext.current().getCallingAccount();
        Long templateId = cmd.getId();
        Long zoneId = cmd.getZoneId();
        String url = cmd.getUrl();
        String mode = cmd.getMode();
        Long eventId = cmd.getStartEventId();

        VirtualMachineTemplate template = _tmpltDao.findById(templateId);
        if (template == null) {
            throw new InvalidParameterValueException("unable to find template with id " + templateId);
        }

        return extract(caller, templateId, url, zoneId, mode, eventId, false);
    }

    @Override
    public VirtualMachineTemplate prepareTemplate(long templateId, long zoneId) {

        VMTemplateVO vmTemplate = _tmpltDao.findById(templateId);
        if (vmTemplate == null) {
            throw new InvalidParameterValueException("Unable to find template id=" + templateId);
        }

        _accountMgr.checkAccess(CallContext.current().getCallingAccount(), AccessType.OperateEntry, vmTemplate);

        prepareTemplateInAllStoragePools(vmTemplate, zoneId);
        return vmTemplate;
    }

    private String extract(Account caller, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO) {
        String desc = Upload.Type.TEMPLATE.toString();
        if (isISO) {
            desc = Upload.Type.ISO.toString();
        }
        eventId = eventId == null ? 0 : eventId;

        if (!_accountMgr.isRootAdmin(caller.getId()) && _disableExtraction) {
            throw new PermissionDeniedException("Extraction has been disabled by admin");
        }

        VMTemplateVO template = _tmpltDao.findById(templateId);
        if (template == null || template.getRemoved() != null) {
            throw new InvalidParameterValueException("Unable to find " + desc + " with id " + templateId);
        }

        if (template.getTemplateType() == Storage.TemplateType.SYSTEM) {
            throw new InvalidParameterValueException("Unable to extract the " + desc + " " + template.getName() + " as it is a default System template");
        } else if (template.getTemplateType() == Storage.TemplateType.PERHOST) {
            throw new InvalidParameterValueException("Unable to extract the " + desc + " " + template.getName() + " as it resides on host and not on SSVM");
        }

        if (isISO) {
            if (template.getFormat() != ImageFormat.ISO) {
                throw new InvalidParameterValueException("Unsupported format, could not extract the ISO");
            }
        } else {
            if (template.getFormat() == ImageFormat.ISO) {
                throw new InvalidParameterValueException("Unsupported format, could not extract the template");
            }
        }

        if (zoneId != null && _dcDao.findById(zoneId) == null) {
            throw new IllegalArgumentException("Please specify a valid zone.");
        }

        if (!_accountMgr.isRootAdmin(caller.getId()) && !template.isExtractable()) {
            throw new InvalidParameterValueException("Unable to extract template id=" + templateId + " as it's not extractable");
        }

        _accountMgr.checkAccess(caller, AccessType.OperateEntry, template);

        List<DataStore> ssStores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId));

        TemplateDataStoreVO tmpltStoreRef = null;
        ImageStoreEntity tmpltStore = null;
        if (ssStores != null) {
            for (DataStore store : ssStores) {
                tmpltStoreRef = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId);
                if (tmpltStoreRef != null) {
                    if (tmpltStoreRef.getDownloadState() == com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
                        tmpltStore = (ImageStoreEntity)store;
                        break;
                    }
                }
            }
        }

        if (tmpltStore == null) {
            throw new InvalidParameterValueException("The " + desc + " has not been downloaded ");
        }

        // Handle NFS to S3 object store migration case, we trigger template sync from NFS to S3 during extract template or copy template
        _tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore);

        TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);

        return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(), templateObject);
    }

    @Override
    public void prepareIsoForVmProfile(VirtualMachineProfile profile) {
        UserVmVO vm = _userVmDao.findById(profile.getId());
        if (vm.getIsoId() != null) {
            TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId());
            if (template == null){
                s_logger.error("Failed to prepare ISO on secondary or cache storage");
                throw new CloudRuntimeException("Failed to prepare ISO on secondary or cache storage");
            }
            if (template.isBootable()) {
                profile.setBootLoaderType(BootloaderType.CD);
            }

            GuestOSVO guestOS = _guestOSDao.findById(template.getGuestOSId());
            String displayName = null;
            if (guestOS != null) {
                displayName = guestOS.getDisplayName();
            }

            TemplateObjectTO iso = (TemplateObjectTO)template.getTO();
            iso.setGuestOsType(displayName);
            DiskTO disk = new DiskTO(iso, 3L, null, Volume.Type.ISO);
            profile.addDisk(disk);
        } else {
            TemplateObjectTO iso = new TemplateObjectTO();
            iso.setFormat(ImageFormat.ISO);
            DiskTO disk = new DiskTO(iso, 3L, null, Volume.Type.ISO);
            profile.addDisk(disk);
        }
    }

    public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) {
        List<StoragePoolVO> pools = _poolDao.listByStatus(StoragePoolStatus.Up);
        for (final StoragePoolVO pool : pools) {
            if (pool.getDataCenterId() == zoneId) {
                s_logger.info("Schedule to preload template " + template.getId() + " into primary storage " + pool.getId());
                _preloadExecutor.execute(new ManagedContextRunnable() {
                    @Override
                    protected void runInContext() {
                        try {
                            reallyRun();
                        } catch (Throwable e) {
                            s_logger.warn("Unexpected exception ", e);
                        }
                    }

                    private void reallyRun() {
                        s_logger.info("Start to preload template " + template.getId() + " into primary storage " + pool.getId());
                        StoragePool pol = (StoragePool)_dataStoreMgr.getPrimaryDataStore(pool.getId());
                        prepareTemplateForCreate(template, pol);
                        s_logger.info("End of preloading template " + template.getId() + " into primary storage " + pool.getId());
                    }
                });
            } else {
                s_logger.info("Skip loading template " + template.getId() + " into primary storage " + pool.getId() + " as pool zone " + pool.getDataCenterId() +
                        " is different from the requested zone " + zoneId);
            }
        }
    }

    @Override
    @DB
    public VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO templ, StoragePool pool) {
        VMTemplateVO template = _tmpltDao.findById(templ.getId(), true);

        long poolId = pool.getId();
        long templateId = template.getId();
        VMTemplateStoragePoolVO templateStoragePoolRef = null;
        TemplateDataStoreVO templateStoreRef = null;

        templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId);
        if (templateStoragePoolRef != null) {
            templateStoragePoolRef.setMarkedForGC(false);
            _tmpltPoolDao.update(templateStoragePoolRef.getId(), templateStoragePoolRef);

            if (templateStoragePoolRef.getDownloadState() == Status.DOWNLOADED) {
                if (s_logger.isDebugEnabled()) {
                    s_logger.debug("Template " + templateId + " has already been downloaded to pool " + poolId);
                }

                return templateStoragePoolRef;
            }
        }

        templateStoreRef = _tmplStoreDao.findByTemplateZoneDownloadStatus(templateId, pool.getDataCenterId(), VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
        if (templateStoreRef == null) {
            s_logger.error("Unable to find a secondary storage host who has completely downloaded the template.");
            return null;
        }

        List<StoragePoolHostVO> vos = _poolHostDao.listByHostStatus(poolId, com.cloud.host.Status.Up);
        if (vos == null || vos.isEmpty()) {
            throw new CloudRuntimeException("Cannot download " + templateId + " to poolId " + poolId + " since there is no host in the Up state connected to this pool");
        }

        if (templateStoragePoolRef == null) {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Downloading template " + templateId + " to pool " + poolId);
            }
            DataStore srcSecStore = _dataStoreMgr.getDataStore(templateStoreRef.getDataStoreId(), DataStoreRole.Image);
            TemplateInfo srcTemplate = _tmplFactory.getTemplate(templateId, srcSecStore);

            AsyncCallFuture<TemplateApiResult> future = _tmpltSvr.prepareTemplateOnPrimary(srcTemplate, pool);
            try {
                TemplateApiResult result = future.get();
                if (result.isFailed()) {
                    s_logger.debug("prepare template failed:" + result.getResult());
                    return null;
                }

                return _tmpltPoolDao.findByPoolTemplate(poolId, templateId);
            } catch (Exception ex) {
                s_logger.debug("failed to copy template from image store:" + srcSecStore.getName() + " to primary storage");
            }
        }

        return null;
    }

    @Override
    public String getChecksum(DataStore store, String templatePath) {
        EndPoint ep = _epSelector.select(store);
        ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath);
        Answer answer = null;
        if (ep == null) {
            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
            s_logger.error(errMsg);
            answer = new Answer(cmd, false, errMsg);
        } else {
            answer = ep.sendMessage(cmd);
        }
        if (answer != null && answer.getResult()) {
            return answer.getDetails();
        }
        return null;
    }

    @Override
    @DB
    public boolean resetTemplateDownloadStateOnPool(long templateStoragePoolRefId) {
        // have to use the same lock that prepareTemplateForCreate use to
        // maintain state consistency
        VMTemplateStoragePoolVO templateStoragePoolRef = _tmpltPoolDao.acquireInLockTable(templateStoragePoolRefId, 1200);

        if (templateStoragePoolRef == null) {
            s_logger.warn("resetTemplateDownloadStateOnPool failed - unable to lock TemplateStorgePoolRef " + templateStoragePoolRefId);
            return false;
        }

        try {
            templateStoragePoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED);
            _tmpltPoolDao.update(templateStoragePoolRefId, templateStoragePoolRef);
        } finally {
            _tmpltPoolDao.releaseFromLockTable(templateStoragePoolRefId);
        }

        return true;
    }

    @Override
    @DB
    public boolean copy(long userId, VMTemplateVO template, DataStore srcSecStore, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException {
        long tmpltId = template.getId();
        long dstZoneId = dstZone.getId();
        // find all eligible image stores for the destination zone
        List<DataStore> dstSecStores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(dstZoneId));
        if (dstSecStores == null || dstSecStores.isEmpty()) {
            throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, dstZone.getId());
        }
        AccountVO account = _accountDao.findById(template.getAccountId());
        // find the size of the template to be copied
        TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(srcSecStore.getId(), tmpltId);

        _resourceLimitMgr.checkResourceLimit(account, ResourceType.template);
        _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, new Long(srcTmpltStore.getSize()));

        // Event details
        String copyEventType;
        String createEventType;
        if (template.getFormat().equals(ImageFormat.ISO)) {
            copyEventType = EventTypes.EVENT_ISO_COPY;
            createEventType = EventTypes.EVENT_ISO_CREATE;
        } else {
            copyEventType = EventTypes.EVENT_TEMPLATE_COPY;
            createEventType = EventTypes.EVENT_TEMPLATE_CREATE;
        }

        TemplateInfo srcTemplate = _tmplFactory.getTemplate(template.getId(), srcSecStore);
        // Copy will just find one eligible image store for the destination zone
        // and copy template there, not propagate to all image stores
        // for that zone
        for (DataStore dstSecStore : dstSecStores) {
            TemplateDataStoreVO dstTmpltStore = _tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId);
            if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) {
                return true; // already downloaded on this image store
            }

            AsyncCallFuture<TemplateApiResult> future = _tmpltSvr.copyTemplate(srcTemplate, dstSecStore);
            try {
                TemplateApiResult result = future.get();
                if (result.isFailed()) {
                    s_logger.debug("copy template failed for image store " + dstSecStore.getName() + ":" + result.getResult());
                    continue; // try next image store
                }

                _tmpltDao.addTemplateToZone(template, dstZoneId);

                if (account.getId() != Account.ACCOUNT_ID_SYSTEM) {
                    UsageEventUtils.publishUsageEvent(copyEventType, account.getId(), dstZoneId, tmpltId, null, null, null, srcTmpltStore.getPhysicalSize(),
                            srcTmpltStore.getSize(), template.getClass().getName(), template.getUuid());
                }
                return true;
            } catch (Exception ex) {
                s_logger.debug("failed to copy template to image store:" + dstSecStore.getName() + " ,will try next one");
            }
        }
        return false;

    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_COPY, eventDescription = "copying template", async = true)
    public VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException {
        Long templateId = cmd.getId();
        Long userId = CallContext.current().getCallingUserId();
        Long sourceZoneId = cmd.getSourceZoneId();
        Long destZoneId = cmd.getDestinationZoneId();
        Account caller = CallContext.current().getCallingAccount();

        // Verify parameters
        VMTemplateVO template = _tmpltDao.findById(templateId);
        if (template == null || template.getRemoved() != null) {
            throw new InvalidParameterValueException("Unable to find template with id");
        }

        DataStore srcSecStore = null;
        if (sourceZoneId != null) {
            // template is on zone-wide secondary storage
            srcSecStore = getImageStore(sourceZoneId, templateId);
        } else {
            // template is on region store
            srcSecStore = getImageStore(templateId);
        }

        if (srcSecStore == null) {
            throw new InvalidParameterValueException("There is no template " + templateId + " ready on image store.");
        }

        if (template.isCrossZones()) {
            // sync template from cache store to region store if it is not there, for cases where we are going to migrate existing NFS to S3.
            _tmpltSvr.syncTemplateToRegionStore(templateId, srcSecStore);
            s_logger.debug("Template " + templateId + " is cross-zone, don't need to copy");
            return template;
        }

        if (sourceZoneId != null) {
            if (sourceZoneId.equals(destZoneId)) {
                throw new InvalidParameterValueException("Please specify different source and destination zones.");
            }

            DataCenterVO sourceZone = _dcDao.findById(sourceZoneId);
            if (sourceZone == null) {
                throw new InvalidParameterValueException("Please specify a valid source zone.");
            }
        }

        DataCenterVO dstZone = _dcDao.findById(destZoneId);
        if (dstZone == null) {
            throw new InvalidParameterValueException("Please specify a valid destination zone.");
        }

        DataStore dstSecStore = getImageStore(destZoneId, templateId);
        if (dstSecStore != null) {
            s_logger.debug("There is template " + templateId + " in secondary storage " + dstSecStore.getName() + " in zone " + destZoneId + " , don't need to copy");
            return template;
        }

        _accountMgr.checkAccess(caller, AccessType.OperateEntry, template);

        boolean success = copy(userId, template, srcSecStore, dstZone);

        if (success) {
            // increase resource count
            long accountId = template.getAccountId();
            if (template.getSize() != null) {
                _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.secondary_storage, template.getSize());
            }
            return template;
        } else {
            throw new CloudRuntimeException("Failed to copy template");
        }
    }

    @Override
    public boolean delete(long userId, long templateId, Long zoneId) {
        VMTemplateVO template = _tmpltDao.findById(templateId);
        if (template == null || template.getRemoved() != null) {
            throw new InvalidParameterValueException("Please specify a valid template.");
        }

        TemplateAdapter adapter = getAdapter(template.getHypervisorType());
        return adapter.delete(new TemplateProfile(userId, template, zoneId));
    }

    @Override
    public List<VMTemplateStoragePoolVO> getUnusedTemplatesInPool(StoragePoolVO pool) {
        List<VMTemplateStoragePoolVO> unusedTemplatesInPool = new ArrayList<VMTemplateStoragePoolVO>();
        List<VMTemplateStoragePoolVO> allTemplatesInPool = _tmpltPoolDao.listByPoolId(pool.getId());

        for (VMTemplateStoragePoolVO templatePoolVO : allTemplatesInPool) {
            VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());

            // If this is a routing template, consider it in use
            if (template.getTemplateType() == TemplateType.SYSTEM) {
                continue;
            }

            // If the template is not yet downloaded to the pool, consider it in
            // use
            if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) {
                continue;
            }

            if (template.getFormat() != ImageFormat.ISO && !_volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), pool.getId())) {
                unusedTemplatesInPool.add(templatePoolVO);
            }
        }

        return unusedTemplatesInPool;
    }

    @Override
    @DB
    public void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO) {
        //Need to hold the lock, otherwise, another thread may create a volume from the template at the same time.
        //Assumption here is that, we will hold the same lock during create volume from template
        VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.acquireInLockTable(templatePoolVO.getId());
        if (templatePoolRef == null) {
            s_logger.debug("can't aquire the lock for template pool ref:" + templatePoolVO.getId());
            return;
        }

        try {
            StoragePool pool = (StoragePool)_dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId());
            VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());

            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Evicting " + templatePoolVO);
            }
            DestroyCommand cmd = new DestroyCommand(pool, templatePoolVO);

            try {
                Answer answer = _storageMgr.sendToPool(pool, cmd);

                if (answer != null && answer.getResult()) {
                    // Remove the templatePoolVO
                    if (_tmpltPoolDao.remove(templatePoolVO.getId())) {
                        s_logger.debug("Successfully evicted template: " + template.getName() + " from storage pool: " + pool.getName());
                    }
                } else {
                    s_logger.info("Will retry evicte template: " + template.getName() + " from storage pool: " + pool.getName());
                }
            } catch (StorageUnavailableException e) {
                s_logger.info("Storage is unavailable currently.  Will retry evicte template: " + template.getName() + " from storage pool: " + pool.getName());
            }
        } finally {
            _tmpltPoolDao.releaseFromLockTable(templatePoolRef.getId());
        }

    }

    @Override
    public boolean start() {
        return true;
    }

    @Override
    public boolean stop() {
        return true;
    }

    @Override
    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {

        String disableExtraction = _configDao.getValue(Config.DisableExtraction.toString());
        _disableExtraction = (disableExtraction == null) ? false : Boolean.parseBoolean(disableExtraction);

        _preloadExecutor = Executors.newFixedThreadPool(8, new NamedThreadFactory("Template-Preloader"));

        return true;
    }

    protected TemplateManagerImpl() {
    }

    @Override
    public boolean templateIsDeleteable(VMTemplateHostVO templateHostRef) {
        VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templateHostRef.getTemplateId());
        long templateId = template.getId();
        HostVO secondaryStorageHost = _hostDao.findById(templateHostRef.getHostId());
        long zoneId = secondaryStorageHost.getDataCenterId();
        DataCenterVO zone = _dcDao.findById(zoneId);

        // Check if there are VMs running in the template host ref's zone that
        // use the template
        List<VMInstanceVO> nonExpungedVms = _vmInstanceDao.listNonExpungedByZoneAndTemplate(zoneId, templateId);

        if (!nonExpungedVms.isEmpty()) {
            s_logger.debug("Template " + template.getName() + " in zone " + zone.getName() +
                    " is not deleteable because there are non-expunged VMs deployed from this template.");
            return false;
        }
        List<UserVmVO> userVmUsingIso = _userVmDao.listByIsoId(templateId);
        // check if there is any VM using this ISO.
        if (!userVmUsingIso.isEmpty()) {
            s_logger.debug("ISO " + template.getName() + " in zone " + zone.getName() + " is not deleteable because it is attached to " + userVmUsingIso.size() + " VMs");
            return false;
        }
        // Check if there are any snapshots for the template in the template
        // host ref's zone
        List<VolumeVO> volumes = _volumeDao.findByTemplateAndZone(templateId, zoneId);
        for (VolumeVO volume : volumes) {
            List<SnapshotVO> snapshots = _snapshotDao.listByVolumeIdVersion(volume.getId(), "2.1");
            if (!snapshots.isEmpty()) {
                s_logger.debug("Template " + template.getName() + " in zone " + zone.getName() +
                        " is not deleteable because there are 2.1 snapshots using this template.");
                return false;
            }
        }

        return true;
    }

    @Override
    public boolean templateIsDeleteable(long templateId) {
        List<UserVmJoinVO> userVmUsingIso = _userVmJoinDao.listActiveByIsoId(templateId);
        // check if there is any Vm using this ISO. We only need to check the
        // case where templateId is an ISO since
        // VM can be launched from ISO in secondary storage, while template will
        // always be copied to
        // primary storage before deploying VM.
        if (!userVmUsingIso.isEmpty()) {
            s_logger.debug("ISO " + templateId + " is not deleteable because it is attached to " + userVmUsingIso.size() + " VMs");
            return false;
        }

        return true;
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_ISO_DETACH, eventDescription = "detaching ISO", async = true)
    public boolean detachIso(long vmId) {
        Account caller = CallContext.current().getCallingAccount();
        Long userId = CallContext.current().getCallingUserId();

        // Verify input parameters
        UserVmVO vmInstanceCheck = _userVmDao.findById(vmId);
        if (vmInstanceCheck == null) {
            throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
        }

        UserVm userVM = _userVmDao.findById(vmId);
        if (userVM == null) {
            throw new InvalidParameterValueException("Please specify a valid VM.");
        }

        _accountMgr.checkAccess(caller, null, userVM);

        Long isoId = userVM.getIsoId();
        if (isoId == null) {
            throw new InvalidParameterValueException("The specified VM has no ISO attached to it.");
        }
        CallContext.current().setEventDetails("Vm Id: " + vmId + " ISO Id: " + isoId);

        State vmState = userVM.getState();
        if (vmState != State.Running && vmState != State.Stopped) {
            throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running.");
        }

        boolean result = attachISOToVM(vmId, userId, isoId, false); // attach=false
        // => detach
        if (result) {
            return result;
        } else {
            throw new CloudRuntimeException("Failed to detach iso");
        }
    }

    @Override
    @ActionEvent(eventType = EventTypes.EVENT_ISO_ATTACH, eventDescription = "attaching ISO", async = true)
    public boolean attachIso(long isoId, long vmId) {
        Account caller = CallContext.current().getCallingAccount();
        Long userId = CallContext.current().getCallingUserId();

        // Verify input parameters
        UserVmVO vm = _userVmDao.findById(vmId);
        if (vm == null) {
            throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
        }

        VMTemplateVO iso = _tmpltDao.findById(isoId);
        if (iso == null || iso.getRemoved() != null) {
            throw new InvalidParameterValueException("Unable to find an ISO with id " + isoId);
        }

        // check permissions
        // check if caller has access to VM and ISO
        // and also check if the VM's owner has access to the ISO. This is a bit different from sameOwner check for attachVolumeToVM, where both volume and VM need
        // OperateEntry access type. Here VM needs OperateEntry access type, ISO needs UseEntry access type.
        _accountMgr.checkAccess(caller, null, iso, vm);
        Account vmOwner = _accountDao.findById(vm.getAccountId());
        _accountMgr.checkAccess(vmOwner, null, iso);

        State vmState = vm.getState();
        if (vmState != State.Running && vmState != State.Stopped) {
            throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running.");
        }

        if ("xen-pv-drv-iso".equals(iso.getDisplayText()) && vm.getHypervisorType() != Hypervisor.HypervisorType.XenServer) {
            throw new InvalidParameterValueException("Cannot attach Xenserver PV drivers to incompatible hypervisor " + vm.getHypervisorType());
        }

        if ("vmware-tools.iso".equals(iso.getName()) && vm.getHypervisorType() != Hypervisor.HypervisorType.VMware) {
            throw new InvalidParameterValueException("Cannot attach VMware tools drivers to incompatible hypervisor " + vm.getHypervisorType());
        }
        boolean result = attachISOToVM(vmId, userId, isoId, true);
        if (result) {
            return result;
        } else {
            throw new CloudRuntimeException("Failed to attach iso");
        }
    }

    // for ISO, we need to consider whether to copy to cache storage or not if it is not on NFS, since our hypervisor resource always assumes that they are in NFS
    @Override
    public TemplateInfo prepareIso(long isoId, long dcId) {
        TemplateInfo tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
        if (tmplt == null || tmplt.getFormat() != ImageFormat.ISO) {
            s_logger.warn("ISO: " + isoId + " does not exist in vm_template table");
            return null;
        }

        if (tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
            // if it is s3, need to download into cache storage first
            Scope destScope = new ZoneScope(dcId);
            TemplateInfo cacheData = (TemplateInfo)cacheMgr.createCacheObject(tmplt, destScope);
            if (cacheData == null) {
                s_logger.error("Failed in copy iso from S3 to cache storage");
                return null;
            }
            return cacheData;
        } else {
            return tmplt;