Add support for global and test-local memory overrides

wrims-comparison-test/src/test/java/gov/ca/water/wrims/comparison/utils/ComputeTestUtils.java

@@ -13,13 +13,20 @@
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import gov.ca.water.wrims.engine.core.tools.MemoryParamParser;
 
 public class ComputeTestUtils {
 
     private static final Logger LOGGER = Logger.getLogger(ComputeTestUtils.class.getName());
 
-    public static final String XMX_PARAM = "-Xmx4096m";
-    public static final String XSS_PARAM = "-Xss1024K";
+	private static final String XMX_DEFAULT = "4096m";
+	private static final String XSS_DEFAULT = "1024K";
+
+	public static final String XMX_PARAM_KEY = "gov.ca.water.wrims.comparison.compute.memory.max";
+	public static final String XSS_PARAM_KEY = "gov.ca.water.wrims.comparison.compute.stack.size";
+
+	public static final String XMX_PARAM = MemoryParamParser.getMaxMemory(XMX_PARAM_KEY, XMX_DEFAULT);
+	public static final String XSS_PARAM = MemoryParamParser.getStackMemory(XSS_PARAM_KEY, XSS_DEFAULT);
 
     // Run the System Summary Report tool similarly to the Gradle testReport task
     // Overload with defaults: workingDir = build/testProjects, inputFile = build/(inputFilePath or testProjects/callite_version_check_dss6_6.inp)

wrims-core/src/main/java/gov/ca/water/wrims/engine/core/components/ControlData.java

@@ -25,21 +25,23 @@
 import com.sunsetsoft.xa.Optimizer;
 
 public class ControlData {
+	public static final String XMX_GLOBAL_PARAM_KEY = "gov.ca.water.wrims.comparison.compute.memory.max";
+	public static final String XSS_GLOBAL_PARAM_KEY = "gov.ca.water.wrims.comparison.compute.stack.size";
 	public static ArrayList<Integer> currTimeStep;
 	public static StudyDataSet currStudyDataSet=new StudyDataSet();
 	public static ModelDataSet currModelDataSet=new ModelDataSet();
 	public static LinkedHashMap<String, Svar> parameterMap=new LinkedHashMap<String, Svar>();
 	public static Map<String, Svar> currSvMap=new HashMap<String, Svar>() ;
 	public static Map<String, Svar> currSvFutMap=new HashMap<String, Svar>() ;
 	public static Map<String, Timeseries> currTsMap=new HashMap<String, Timeseries>();
 	public static Map<String, Dvar> currDvMap=new HashMap<String, Dvar>();
 	public static Map<String, Dvar> currDvSlackSurplusMap=new HashMap<String, Dvar>();
 	public static Map<String, Alias> currAliasMap=new HashMap<String, Alias>();
 	public static Map<String, Goal> currGoalMap=new HashMap<String, Goal>();
 	public static Map<String, External> currExMap = new HashMap<String, External> ();
 	public static Map<String, Timeseries> allTsMap=new HashMap<String, Timeseries>();
 	public static Map<String, String> allExternalFunction = new HashMap<String, String> ();
 	public static Map<String, ExternalFunction> allExternalFunctionMap = new HashMap<String, ExternalFunction>();  
 	public static ArrayList<String> allDll= new ArrayList<String>() ;
 	public static String currCycleName;
 	public static int currCycleIndex;
@@ -51,24 +53,24 @@
 	public static String defaultTimeStep="1MON";
 	public static String timeStep ="1MON"; 
 	public static String partE="1MON";
 	public static int startYear;       
 	public static int startMonth;
 	public static int startDay;
 	public static int endYear ;
 	public static int endMonth;
 	public static int endDay;
 	public static int cycleStartDay;
 	public static int cycleStartMonth;
 	public static int cycleStartYear;
 	public static int cycleEndDay;
 	public static int cycleEndMonth;
 	public static int cycleEndYear;
 	public static double cycleTimeStepPriority=0;
 	public static ArrayList<Integer> totalTimeStep;
 	public static Date monthlyStartTime;
 	public static Date dailyStartTime;
 	public static String simulationTimeFrame;
 	public static String partA="";
 	public static String svDvPartF="";
 	public static String initPartF;
 	//public static Group groupInit;
@@ -76,7 +78,7 @@
 	//public static Group groupSvar2;
 	public static CondensedReferenceCache cacheInit;
 	public static CondensedReferenceCache cacheSvar;
 	public static CondensedReferenceCache cacheSvar2;
 	//public static DSSDataWriter writer;
 	public static HecDss dvDss;
 	public static String solverName="cbc'";
@@ -86,39 +88,39 @@
 	public static Double clp_cbc_objective;
 	public static String clp_cbc_note="";
 	public static double lpsolve_objective;
 	public static double gurobi_objective;
 	public static boolean isPostProcessing=false;
 	public static boolean sendAliasToDvar=false;
 	public static boolean outputWreslCSV=false;
 	public static boolean showRunTimeMessage=false;
 	public static boolean showWreslLog=true;
 	public static boolean writeInitToDVOutput=true;
 	public static int solverType;
 	public static boolean isParseStudy=true;
 	public static boolean ignoreError=false;
 	public static boolean allowSvTsInit=false;
 	public static boolean allRestartFiles=false;
 	public static int numberRestartFiles=12;
 	public static int vHecLib=6;
 	public static int outputType=0;
 	public static boolean isOutputCycle=false;
 	public static boolean outputAllCycles=true;
 	public static String selectedCycleOutput="\'\'";
 	public static String[] selectedCycles=new String[0];
 	public static boolean initHDF5=false;
 	public static double solverTime_xa=0;
 	public static double solverTime_xa_this=0;
 	public static double solverTime_cbc=0;
 	public static double solverTime_cbc_this=0;
 	public static double solverCreationTime_cbc=0;
 	public static double solverCreationTime_xa=0;
 	public static double lpFileWritingTime=0;
 	public static double cdTime=0;	
 	public static double adTime=0;	
 	public static boolean useCplexLpString=false;
 	public static boolean saveCplexLpStringToFile=false;
 	public static boolean useCbcWarmStart=false;
 	public static boolean cbcCheckIntErr=true;	
 	public static boolean cbcLogNativeLp=false;
 	public static boolean writeCbcSolvingTime=false;
 	public static boolean isNameSorting = false;
@@ -127,7 +129,7 @@
 	public static ArrayList<Integer> cycWarmStart;
 	public static ArrayList<Integer> cycWarmStop;
 	public static ArrayList<Integer> cycWarmUse;
 	public static boolean cbc_debug_routeXA=false; //use xa solution	
 	public static boolean cbc_debug_routeCbc=false;	// use cbc solution
 	public static String[] watchList={};
 	public static double watchList_tolerance = 1e99;
@@ -135,9 +137,9 @@
 	public static double relationTolerance;
 	public static Double xaIntegerT =  null;
 	public static String xaSort = null;
 	public static boolean resimDate=false;
 	public static boolean resimGroundwater=false;
 	public static int resimYear;
 	public static int resimMonth;
 	public static int resimDay;
 	public static int cycleDataStartYear;
@@ -145,7 +147,7 @@
 	public static int cycleDataStartDay;
 
 	public static String USER;       
 	public static String PASS;  
 	public static String databaseURL;
 	public static String sqlGroup="calsim";
 	public static int ovOption=0;
@@ -153,7 +155,7 @@
 	public static Map<String, String> ovPartBC=new HashMap<String, String>();
 	public static boolean isSimOutput = true;
 
 	public static boolean enableProgressLog = false;
 
 	public static LocalDateTime ldt = LocalDateTime.now();
 	public static String dateTimeAppend = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm", Locale.ENGLISH).format(ldt);
@@ -161,55 +163,55 @@
 	public static boolean printGWFuncCalls=false;
 
 	public static int yearOutputSection=-1;
 	public static int monMemSection=-1;
 	public static int outputYear;
 	public static int outputMonth;
 	public static int outputDay;
 	public static int memStartYear;
 	public static int memStartMonth;
 	public static int memStartDay;
 	public static int prevMemYear;
 	public static int prevMemMonth;
 	public static int prevMemDay;
 	public static int prevOutputYear;
 	public static int prevOutputMonth;
 	public static int prevOutputDay;
 	public static Date prevMemDate;
 	public static Date memStartDate;
 	public static Date prevOutputDate;
 	public static int nThreads=1;
 	public static boolean unchangeGWRestart=false;
 	public static boolean genSVCatalog=true;
 	public static boolean showTimeUsage=true;
 	public static int t_cbc=0;
 	public static int t_xa=0;
 	public static int t_ts=0;
 	public static int t_svar=0;
 	public static int t_dvar=0;
 	public static int t_goal=0;
 	public static int t_wt=0;
 	public static int t_wtss=0;
 	public static int t_as=0;
 	public static int t_readTs=0;
 	public static int t_writeDss=0;
 	public static int t_cam=0;
 	public static int t_parse=0;
 	public static int t_ann=0;
 	public static int n_ann=0;
 	public static int t_annx2=0;
 	public static int n_annx2=0;
 	public static int t_annec=0;
 	public static int n_annec=0;
 	public static int t_annlinegen=0;
 	public static int n_annlinegen=0;
 	public static int t_anngetndo_x2=0;
 	public static int n_anngetndo_x2=0;
 	public static int t_anngetndo_x2_curmonndosplit=0;
 	public static int n_anngetndo_x2_curmonndosplit=0;
 	public static int t_annec_matchdsm2=0;
 	public static int n_annec_matchdsm2=0;
 
 	public static int pid=-1;
 
 	public ControlData(){
 	}

wrims-core/src/main/java/gov/ca/water/wrims/engine/core/solver/InitialXASolver.java

@@ -6,10 +6,15 @@
 import com.sunsetsoft.xa.XAException;
 
 import gov.ca.water.wrims.engine.core.components.Error;
+import gov.ca.water.wrims.engine.core.tools.MemoryParamParser;
 
 public class InitialXASolver {
+	public static final String XA_MAX_MEMORY_PARAM_KEY = "gov.ca.water.wrims.engine.core.solver.xa.memory.max";
+	private static final long MEMORY_DEFAULT = 25000L;
+
 	public InitialXASolver(){
-		ControlData.xasolver=new Optimizer(25000);
+		Long initialMemory = MemoryParamParser.getMaxMemoryInBytes(XA_MAX_MEMORY_PARAM_KEY, MEMORY_DEFAULT);
+		ControlData.xasolver=new Optimizer(initialMemory.intValue());
 		ControlData.xasolver.setActivationCodes( 234416483 , 19834525 ) ;
 		ControlData.xasolver.setXAMessageWindowOff();
 		try{

wrims-core/src/main/java/gov/ca/water/wrims/engine/core/tools/MemoryParamParser.java

@@ -0,0 +1,112 @@
+package gov.ca.water.wrims.engine.core.tools;
+
+import gov.ca.water.wrims.engine.core.components.ControlData;
+
+public final class MemoryParamParser {
+	public static final String MEMORY_DECISION_KEY = "gov.ca.water.wrims.engine.core.memory.decision";
+	private static final boolean USE_MIN_MEMORY = Boolean.parseBoolean(System.getProperty(MEMORY_DECISION_KEY, "true"));
+	private static final String XMX_GLOBAL = System.getProperty(ControlData.XMX_GLOBAL_PARAM_KEY);
+	private static final String XSS_GLOBAL = System.getProperty(ControlData.XSS_GLOBAL_PARAM_KEY);
+
+	private MemoryParamParser() {
+		throw new IllegalStateException("Utility class");
+	}
+
+	public static String getMaxMemory(String localMemoryKey, String defaultValue) {
+		String xmxLocal = null;
+		if (localMemoryKey != null && defaultValue != null) {
+			xmxLocal = System.getProperty(localMemoryKey, defaultValue);
+		} else if (localMemoryKey == null) {
+			xmxLocal = defaultValue;
+		}
+		return buildMaxParam(compareMemValues(XMX_GLOBAL, xmxLocal));
+	}
+
+	public static Long getMaxMemoryInBytes(String localMemoryKey, Long defaultValue) {
+		String xmxLocal = null;
+		if (localMemoryKey != null && defaultValue != null) {
+			xmxLocal = System.getProperty(localMemoryKey, defaultValue.toString());
+		} else if (localMemoryKey == null) {
+			xmxLocal = defaultValue.toString();
+		}
+		return parseParam(compareMemValues(XMX_GLOBAL, xmxLocal));
+	}
+
+	public static String getStackMemory(String localMemoryKey, String defaultValue) {
+		String xssLocal = null;
+		if (localMemoryKey != null && defaultValue != null) {
+			xssLocal = System.getProperty(localMemoryKey, defaultValue);
+		} else if (localMemoryKey == null) {
+			xssLocal = defaultValue;
+		}
+		return buildStackParam(compareMemValues(XSS_GLOBAL, xssLocal));
+	}
+
+	public static long getStackMemoryInBytes(String localMemoryKey, String defaultValue) {
+		String xssLocal = null;
+		if (localMemoryKey != null && defaultValue != null) {
+			xssLocal = System.getProperty(localMemoryKey, defaultValue);
+		} else if (localMemoryKey == null) {
+			xssLocal = defaultValue;
+		}
+		return parseParam(compareMemValues(XSS_GLOBAL, xssLocal));
+	}
+
+	private static String compareMemValues(String left, String right) {
+		long leftValue = parseParam(left);
+		long rightValue = parseParam(right);
+
+		if (USE_MIN_MEMORY) {
+			return (leftValue < rightValue) ? left : right;
+		} else {
+			return (leftValue > rightValue) ? left : right;
+		}
+	}
+
+	private static long parseParam(String value) {
+		long retVal;
+
+		if (value == null || value.isEmpty()) {
+			if (USE_MIN_MEMORY) {
+				return Long.MAX_VALUE;
+			} else {
+				return 0L;
+			}
+		}
+
+		if (value.startsWith("-")) {
+			value = value.substring(3);
+		}
+		if (value.endsWith("k") || value.endsWith("K")) {
+			value = value.substring(0, value.length() - 1);
+			retVal = Long.parseLong(value) * 1024L;
+		} else if (value.endsWith("m") || value.endsWith("M")) {
+			value = value.substring(0, value.length() - 1);
+			retVal = Long.parseLong(value) * 1024L * 1024L;
+		} else if (value.endsWith("g") || value.endsWith("G")) {
+			value = value.substring(0, value.length() - 1);
+			retVal = Long.parseLong(value) * 1024L * 1024L * 1024L;
+		} else {
+			retVal = Long.parseLong(value);
+		}
+		return retVal;
+	}
+
+	private static String buildMaxParam(String value) {
+		if (value == null || value.isEmpty()) {
+			return "";
+		} else if (value.startsWith("-")) {
+			return value;
+		}
+		return String.format("-Xmx%s", value);
+	}
+
+	private static String buildStackParam(String value) {
+		if (value == null || value.isEmpty()) {
+			return "";
+		} else if (value.startsWith("-")) {
+			return value;
+		}
+		return String.format("-Xss%s", value);
+	}
+}

wrims-core/src/test/java/gov/ca/water/wrims/engine/core/tools/MemoryParamParserTest.java

@@ -0,0 +1,63 @@
+package gov.ca.water.wrims.engine.core.tools;
+
+import gov.ca.water.wrims.engine.core.components.ControlData;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+final class MemoryParamParserTest {
+
+	@BeforeAll
+	static void setup() {
+		System.setProperty(ControlData.XSS_GLOBAL_PARAM_KEY, "512k");
+		System.setProperty(ControlData.XMX_GLOBAL_PARAM_KEY, "8192M");
+	}
+
+	@AfterAll
+	static void teardown() {
+		System.clearProperty(ControlData.XSS_GLOBAL_PARAM_KEY);
+		System.clearProperty(ControlData.XMX_GLOBAL_PARAM_KEY);
+	}
+
+	@Test
+	void testStackMemParse()
+	{
+		String xssLocal = "gov.ca.water.wrims.engine.core.tools.memory.minStackMem";
+		System.setProperty(xssLocal, "128k");
+
+		String memory = MemoryParamParser.getStackMemory(xssLocal, "128k");
+
+		assertEquals("-Xss128k", memory);
+
+		System.clearProperty(xssLocal);
+	}
+
+	@Test
+	void testMaxMemParse() {
+		String xmxLocal = "gov.ca.water.wrims.engine.core.tools.memory.maxMem";
+		System.setProperty(xmxLocal, "1024M");
+
+		String memory = MemoryParamParser.getMaxMemory(xmxLocal, "128k");
+
+		assertEquals("-Xmx1024M", memory);
+
+		System.clearProperty(xmxLocal);
+	}
+
+	@Test
+	void testStackMemParseGlobal() {
+		String memory = MemoryParamParser.getStackMemory(null, null);
+
+		assertEquals("-Xss512k", memory);
+	}
+
+	@Test
+	void testMaxMemParseGlobal() {
+		String memory = MemoryParamParser.getMaxMemory(null, null);
+
+		assertEquals("-Xmx8192M", memory);
+	}
+}