Skip to content

Commit

Permalink
Reserve memory for host
Browse files Browse the repository at this point in the history 
By default cloudstack reserves 1Gb of RAM in hosts
using _dom0_memory field. Add a global setting
"host.reserved.mem.mb" which can used to either
increase or decrese the amount of memory which can be reserved
  • Loading branch information
@benj-n
Sina Kashipazha authored and benj-n committed Jun 27, 2024
1 parent 22cd00f commit b071723
Show file tree
Hide file tree
Showing 6 changed files with 143 additions and 29 deletions.
2 changes: 2 additions & 0 deletions engine/components-api/src/main/java/com/cloud/agent/AgentManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,6 @@ public enum TapAgentsAction {
void notifyMonitorsOfRemovedHost(long hostId, long clusterId);

void propagateChangeToAgents(Map<String, String> params);

void updateCapacityOfHosts();
}
86 changes: 80 additions & 6 deletions engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Collections;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
Expand All @@ -34,6 +35,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.LongFunction;

import javax.inject.Inject;
import javax.naming.ConfigurationException;
Expand All @@ -51,6 +53,7 @@
import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.log4j.Logger;
Expand Down Expand Up @@ -81,7 +84,11 @@
import com.cloud.agent.transport.Request;
import com.cloud.agent.transport.Response;
import com.cloud.alert.AlertManager;
import com.cloud.capacity.Capacity;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.capacity.CapacityVO;
import com.cloud.configuration.ManagementServiceConfiguration;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
Expand Down Expand Up @@ -124,6 +131,8 @@
import com.cloud.utils.nio.Task;
import com.cloud.utils.time.InaccurateClock;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.ThreadContext;
import org.apache.commons.collections.CollectionUtils;

/**
* Implementation of the Agent Manager. This class controls the connection to the agents.
Expand Down Expand Up @@ -172,6 +181,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl

@Inject
protected IndirectAgentLB indirectAgentLB;
@Inject
protected CapacityDao capacityDao;

protected int _retry = 2;

Expand Down Expand Up @@ -1874,6 +1885,7 @@ public void processConnect(final Host host, final StartupCommand cmd, final bool
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
params.put(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), String.valueOf(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId())));

try {
SetHostParamsCommand cmds = new SetHostParamsCommand(params);
Expand Down Expand Up @@ -1932,25 +1944,87 @@ protected Map<Long, List<Long>> getHostsPerZone() {
return hostsByZone;
}

private void sendCommandToAgents(Map<Long, List<Long>> hostsPerZone, Map<String, String> params) {
SetHostParamsCommand cmds = new SetHostParamsCommand(params);
for (Long zoneId : hostsPerZone.keySet()) {
List<Long> hostIds = hostsPerZone.get(zoneId);
private void sendCommandToAgents(Map<Long, List<Long>> hostsPerZone, LongFunction<Map<String, String>> paramsGenerator ) {
for (List<Long> hostIds : hostsPerZone.values()) {
for (Long hostId : hostIds) {
Answer answer = easySend(hostId, cmds);
Answer answer = easySend(hostId, new SetHostParamsCommand(paramsGenerator.apply(hostId)));
if (answer == null || !answer.getResult()) {
s_logger.error("Error sending parameters to agent " + hostId);
}
}
}
}

private long calculateAvailableMemoryOfHost(HostVO host) {
long reservedMemory = ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId()));
return host.getTotalMemory() + host.getDom0MinMemory() - reservedMemory;
}

private void updateMemoriesInDb(HostVO host, long newMemoryValue) {
host.setTotalMemory(newMemoryValue);

// Update "dom0_memory" in host table
host.setDom0MinMemory(ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId())));
_hostDao.update(host.getId(), host);

// Update the "total_capacity" for all hosts in op_host_capacity
CapacityVO memCap = capacityDao.findByHostIdType(host.getId(), Capacity.CAPACITY_TYPE_MEMORY);
memCap.setTotalCapacity(host.getTotalMemory());
capacityDao.update(memCap.getId(), memCap);
}

private boolean updateHostMemory(HostVO host) {
try {
// Update the "ram" for all hosts
long newMemoryValue = calculateAvailableMemoryOfHost(host);
if (newMemoryValue > 0) {
updateMemoriesInDb(host, newMemoryValue);
return true;
}
} catch (Exception e) {
s_logger.error("Unable to update the reserved memory capacity for host id " + host.getId() + " : " + e.getMessage());
}
return false;
}

@Override
public void propagateChangeToAgents(Map<String, String> params) {
if (params != null && ! params.isEmpty()) {
s_logger.debug("Propagating changes on host parameters to the agents");
Map<Long, List<Long>> hostsPerZone = getHostsPerZone();
sendCommandToAgents(hostsPerZone, params);
sendCommandToAgents(hostsPerZone, id -> params);
}
}

@Override
public void updateCapacityOfHosts() {
Map<Long, List<Long>> hostsByZone = new HashMap<>();
boolean allHostMemoryValuesAreValid = true;

List<HostVO> allHosts = _resourceMgr.listAllHostsInAllZonesByType(Host.Type.Routing);
if (CollectionUtils.isEmpty(allHosts)) {
return;
}

for (HostVO host : allHosts) {
boolean updateWasSuccessful = updateHostMemory(host);

if (!updateWasSuccessful) {
allHostMemoryValuesAreValid = false;
continue;
}

Long zoneId = host.getDataCenterId();
List<Long> hostIds = hostsByZone.getOrDefault(zoneId, new ArrayList<>());
hostIds.add(host.getId());
hostsByZone.put(zoneId, hostIds);
}

if (allHostMemoryValuesAreValid) {
sendCommandToAgents(hostsByZone,
hostId -> Collections.singletonMap(
ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(),
ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(_hostDao.findById(hostId).getClusterId()).toString()));
}
}

Expand Down
8 changes: 8 additions & 0 deletions engine/schema/src/main/java/com/cloud/host/HostVO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,4 +782,12 @@ public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering){
public PartitionType partitionType() {
return PartitionType.Host;
}

public long getDom0MinMemory() {
return dom0MinMemory;
}

public void setDom0MinMemory(long dom0MinMemory) {
this.dom0MinMemory = dom0MinMemory;
}
}
28 changes: 26 additions & 2 deletions ...rvisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@
import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer;
import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.dc.Vlan;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host.Type;
Expand Down Expand Up @@ -1100,7 +1101,17 @@ public boolean configure(final String name, final Map<String, Object> params) th
videoRam = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_VIDEO_RAM);

// Reserve 1GB unless admin overrides
dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB));
long reservedMemory = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB);

value = (String)params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key());
long clusterReservedMemoryValue = NumbersUtil.parseInt(value, 1024);

if (clusterReservedMemoryValue != 1024) {
reservedMemory = clusterReservedMemoryValue;
}

dom0MinMem = ByteScaleUtils.mebibytesToBytes(reservedMemory);


dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT);

Expand Down Expand Up @@ -1442,6 +1453,15 @@ private void validateLocalStorageUUID(String localStorageUUID) throws Configurat
}
}

private void updateDom0MinMem(PropertiesStorage storage, final Map<String, String> params) {
long value = Long.parseLong(params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key()));
s_logger.info("Reserved memory for host is " + value + "MB");
dom0MinMem = ByteScaleUtils.mebibytesToBytes(value);
if (!StringUtils.isEmpty(String.valueOf(value))) {
storage.persist(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key(), String.valueOf(value));
}
}

public boolean configureHostParams(final Map<String, String> params) {
final File file = PropertiesUtil.findConfigFile("agent.properties");
if (file == null) {
Expand All @@ -1457,7 +1477,7 @@ public boolean configureHostParams(final Map<String, String> params) {
}

if (params.get(Config.MigrateWait.toString()) != null) {
String value = (String)params.get(Config.MigrateWait.toString());
String value = (String) params.get(Config.MigrateWait.toString());
Integer intValue = NumbersUtil.parseInt(value, -1);
storage.persist("vm.migrate.wait", String.valueOf(intValue));
migrateWait = intValue;
Expand All @@ -1467,6 +1487,10 @@ public boolean configureHostParams(final Map<String, String> params) {
isTungstenEnabled = Boolean.parseBoolean(params.get(NetworkOrchestrationService.TUNGSTEN_ENABLED.key()));
}

if (params.get(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.key()) != null) {
updateDom0MinMem(storage, params);
}

return true;
}

Expand Down
42 changes: 21 additions & 21 deletions server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@
import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO;
import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.query.QueryService;
import org.apache.cloudstack.region.PortableIp;
Expand Down Expand Up @@ -493,6 +492,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
public static final ConfigKey<Boolean> ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN = new ConfigKey<Boolean>(Boolean.class, "enable.domain.settings.for.child.domain", "Advanced", "false",
"Indicates whether the settings of parent domain should be applied for child domain. If true, the child domain will get value from parent domain if its not configured in child domain else global value is taken.",
true, ConfigKey.Scope.Global, null);
public static final ConfigKey<Integer> HOST_RESERVED_MEM_MB = new ConfigKey<>("Advanced", Integer.class, "host.reserved.mem.mb", "1024",
"Set an upper limit for memory in megabytes which will be reserved for host and not used for VM allocation.", true, ConfigKey.Scope.Cluster);

public static ConfigKey<Integer> VM_SERVICE_OFFERING_MAX_CPU_CORES = new ConfigKey<Integer>("Advanced", Integer.class, "vm.serviceoffering.cpu.cores.max", "0", "Maximum CPU cores "
+ "for vm service offering. If 0 - no limitation", true);
Expand Down Expand Up @@ -593,23 +594,22 @@ private void overProvisioningFactorsForValidation() {
}

private void initMessageBusListener() {
messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() {
@Override
public void onPublishMessage(String serderAddress, String subject, Object args) {
String globalSettingUpdated = (String) args;
if (StringUtils.isEmpty(globalSettingUpdated)) {
return;
}
if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) ||
globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) {
_indirectAgentLB.propagateMSListToAgents();
} else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())
|| globalSettingUpdated.equals(Config.MigrateWait.toString())) {
Map<String, String> params = new HashMap<String, String>();
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
_agentManager.propagateChangeToAgents(params);
}
messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, (serverAddress, subject, args) -> {
String globalSettingUpdated = (String) args;
if (StringUtils.isEmpty(globalSettingUpdated)) {
return;
}
if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) ||
globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) {
_indirectAgentLB.propagateMSListToAgents();
} else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())
|| globalSettingUpdated.equals(Config.MigrateWait.toString())) {
Map<String, String> params = new HashMap<String, String>();
params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
_agentManager.propagateChangeToAgents(params);
} else if (globalSettingUpdated.equalsIgnoreCase(HOST_RESERVED_MEM_MB.key())) {
_agentManager.updateCapacityOfHosts();
}
});
}
Expand Down Expand Up @@ -7802,9 +7802,9 @@ public String getConfigComponentName() {
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {SystemVMUseLocalStorage, IOPS_MAX_READ_LENGTH, IOPS_MAX_WRITE_LENGTH,
BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE,
VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS,
ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN, ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS
BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE, VM_SERVICE_OFFERING_MAX_CPU_CORES,
VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS, ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN,
ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN, ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, HOST_RESERVED_MEM_MB, AllowNonRFC1918CompliantIPs
};
}

Expand Down
6 changes: 6 additions & 0 deletions server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ObjectUtils;
Expand Down Expand Up @@ -100,6 +101,7 @@
import com.cloud.cluster.ClusterManager;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.ClusterDetailsVO;
import com.cloud.dc.ClusterVO;
Expand Down Expand Up @@ -2349,6 +2351,10 @@ protected HostVO createHostVO(final StartupCommand[] cmds, final ServerResource
} else {
hostTags = implicitHostTags;
}
// Update host memory reported by agent
if (ssCmd.getHypervisorType().equals(HypervisorType.KVM) ||
ssCmd.getHypervisorType().equals(HypervisorType.LXC)) {
host.setDom0MinMemory(ByteScaleUtils.mebibytesToBytes(ConfigurationManagerImpl.HOST_RESERVED_MEM_MB.valueIn(host.getClusterId())));
}
}

Expand Down

0 comments on commit b071723

Please sign in to comment.