Parallel processing of rbank, zone heightmap, and zone lighter

nel/tools/build_gamedata/configuration/scripts.py

@@ -23,7 +23,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 
-import time, sys, os, shutil, subprocess, distutils.dir_util
+import time, sys, os, shutil, subprocess, distutils.dir_util, multiprocessing, math
 
 ActiveProjectDirectory = os.getenv("NELBUILDACTIVEPROJECT", "configuration/project")
 sys.path.append(ActiveProjectDirectory)
@@ -32,6 +32,27 @@ def printLog(log, text):
 	log.write(text + "\n")
 	print text
 
+pendingProcesses = []
+processLimit = math.ceil(multiprocessing.cpu_count() * 0.75)
+
+def callParallelProcess(command):
+	res = 0
+	if len(pendingProcesses) >= processLimit:
+		waitingProc = pendingProcesses.pop(0)
+		res = waitingProc.wait()
+	proc = subprocess.Popen(command)
+	pendingProcesses.append(proc)
+	return res
+
+def flushParallelProcesses():
+	res = 0
+	while (len(pendingProcesses) > 0):
+		waitingProc = pendingProcesses.pop(0)
+		procRes = waitingProc.wait()
+		if procRes != 0:
+			res = procRes
+	return res
+
 def mkPath(log, path):
 	printLog(log, "DIR " + path)
 	distutils.dir_util.mkpath(path)

nel/tools/build_gamedata/processes/rbank/2_build.py

@@ -24,7 +24,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 
-import time, sys, os, shutil, subprocess, distutils.dir_util, multiprocessing
+import time, sys, os, shutil, subprocess, distutils.dir_util
 sys.path.append("../../configuration")
 
 if os.path.isfile("log.log"):
@@ -226,8 +226,6 @@ else:
 		else:
 			printLog(log, "SKIP " + lr1)
 	while len(zonesToBuild) > 0:
-		procs = []
-		for i in range(0, multiprocessing.cpu_count()):
 		processCommand = [ ExecTimeout, str(RbankBuildTesselTimeout), BuildRbank, "-c", "-P", "-g" ]
 		processCommand.extend(zonesToBuild[:min(len(zonesToBuild), 64)])
 		if len(zonesToBuild) > 64:
@@ -235,11 +233,8 @@ else:
 		else:
 			zonesToBuild = []
 		print processCommand
-			print len(processCommand)
+		callParallelProcess(processCommand)
-			proc = subprocess.Popen(processCommand)
+	flushParallelProcesses()
-			procs.append(proc)
-		for proc in procs:
-			proc.wait()
 printLog(log, "")
 
 printLog(log, ">>> Detect modifications to rebuild lr <<<")

nel/tools/build_gamedata/processes/zone/2_build.py

@@ -147,7 +147,8 @@ else:
 		sourceFile = ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/" + file
 		destFile = ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/" + os.path.basename(file)[0:-len(".zonenhw")] + ".zonew"
 		if needUpdateLogRemoveDest(log, sourceFile, destFile):
-			subprocess.call([ ZoneElevation, sourceFile, destFile, "--land", land, "--heightmap", heightMap1, "--zfactor", LigoExportZFactor1, "--heightmap2", heightMap2, "--zfactor2", LigoExportZFactor2 ])
+			callParallelProcess([ ZoneElevation, sourceFile, destFile, "--land", land, "--heightmap", heightMap1, "--zfactor", LigoExportZFactor1, "--heightmap2", heightMap2, "--zfactor2", LigoExportZFactor2 ])
+	flushParallelProcesses()
 printLog(log, "")
 
 log.close()

nel/tools/build_gamedata/processes/zone_light/2_build.py

@@ -68,7 +68,8 @@ else:
 		destFile = destDir + "/" + file[0:-len(".zonew")] + ".zonel"
 		if (needUpdateLogRemoveDest(log, srcFile, destFile)):
 			dependFile = dependDir + "/" + file[0:-len(".zonew")] + ".depend"
-			subprocess.call([ ExecTimeout, str(ZoneLightBuildTimeout), ZoneLighter, srcFile, destFile, ActiveProjectDirectory + "/generated/properties.cfg", dependFile ])
+			callParallelProcess([ ExecTimeout, str(ZoneLightBuildTimeout), ZoneLighter, srcFile, destFile, ActiveProjectDirectory + "/generated/properties.cfg", dependFile ])
+	flushParallelProcesses()
 printLog(log, "")
 
 # For each zone_light ig