diff --git a/nel/tools/build_gamedata/configuration/scripts.py b/nel/tools/build_gamedata/configuration/scripts.py index 5bbed497f..6e885bdd3 100755 --- a/nel/tools/build_gamedata/configuration/scripts.py +++ b/nel/tools/build_gamedata/configuration/scripts.py @@ -23,7 +23,7 @@ # along with this program. If not, see . # -import time, sys, os, shutil, subprocess, distutils.dir_util +import time, sys, os, shutil, subprocess, distutils.dir_util, multiprocessing, math ActiveProjectDirectory = os.getenv("NELBUILDACTIVEPROJECT", "configuration/project") sys.path.append(ActiveProjectDirectory) @@ -32,6 +32,27 @@ def printLog(log, text): log.write(text + "\n") print text +pendingProcesses = [] +processLimit = math.ceil(multiprocessing.cpu_count() * 0.75) + +def callParallelProcess(command): + res = 0 + if len(pendingProcesses) >= processLimit: + waitingProc = pendingProcesses.pop(0) + res = waitingProc.wait() + proc = subprocess.Popen(command) + pendingProcesses.append(proc) + return res + +def flushParallelProcesses(): + res = 0 + while (len(pendingProcesses) > 0): + waitingProc = pendingProcesses.pop(0) + procRes = waitingProc.wait() + if procRes != 0: + res = procRes + return res + def mkPath(log, path): printLog(log, "DIR " + path) distutils.dir_util.mkpath(path) diff --git a/nel/tools/build_gamedata/processes/rbank/2_build.py b/nel/tools/build_gamedata/processes/rbank/2_build.py index ee41ede7f..6a55e94fc 100755 --- a/nel/tools/build_gamedata/processes/rbank/2_build.py +++ b/nel/tools/build_gamedata/processes/rbank/2_build.py @@ -24,7 +24,7 @@ # along with this program. If not, see . # -import time, sys, os, shutil, subprocess, distutils.dir_util, multiprocessing +import time, sys, os, shutil, subprocess, distutils.dir_util sys.path.append("../../configuration") if os.path.isfile("log.log"): @@ -226,20 +226,15 @@ else: else: printLog(log, "SKIP " + lr1) while len(zonesToBuild) > 0: - procs = [] - for i in range(0, multiprocessing.cpu_count()): - processCommand = [ ExecTimeout, str(RbankBuildTesselTimeout), BuildRbank, "-c", "-P", "-g" ] - processCommand.extend(zonesToBuild[:min(len(zonesToBuild), 64)]) - if len(zonesToBuild) > 64: - zonesToBuild = zonesToBuild[64:] - else: - zonesToBuild = [] - print processCommand - print len(processCommand) - proc = subprocess.Popen(processCommand) - procs.append(proc) - for proc in procs: - proc.wait() + processCommand = [ ExecTimeout, str(RbankBuildTesselTimeout), BuildRbank, "-c", "-P", "-g" ] + processCommand.extend(zonesToBuild[:min(len(zonesToBuild), 64)]) + if len(zonesToBuild) > 64: + zonesToBuild = zonesToBuild[64:] + else: + zonesToBuild = [] + print processCommand + callParallelProcess(processCommand) + flushParallelProcesses() printLog(log, "") printLog(log, ">>> Detect modifications to rebuild lr <<<") diff --git a/nel/tools/build_gamedata/processes/zone/2_build.py b/nel/tools/build_gamedata/processes/zone/2_build.py index f9a31e3c4..cb9287c36 100755 --- a/nel/tools/build_gamedata/processes/zone/2_build.py +++ b/nel/tools/build_gamedata/processes/zone/2_build.py @@ -147,7 +147,8 @@ else: sourceFile = ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/" + file destFile = ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/" + os.path.basename(file)[0:-len(".zonenhw")] + ".zonew" if needUpdateLogRemoveDest(log, sourceFile, destFile): - subprocess.call([ ZoneElevation, sourceFile, destFile, "--land", land, "--heightmap", heightMap1, "--zfactor", LigoExportZFactor1, "--heightmap2", heightMap2, "--zfactor2", LigoExportZFactor2 ]) + callParallelProcess([ ZoneElevation, sourceFile, destFile, "--land", land, "--heightmap", heightMap1, "--zfactor", LigoExportZFactor1, "--heightmap2", heightMap2, "--zfactor2", LigoExportZFactor2 ]) + flushParallelProcesses() printLog(log, "") log.close() diff --git a/nel/tools/build_gamedata/processes/zone_light/2_build.py b/nel/tools/build_gamedata/processes/zone_light/2_build.py index 84d2b26a4..53a926c29 100755 --- a/nel/tools/build_gamedata/processes/zone_light/2_build.py +++ b/nel/tools/build_gamedata/processes/zone_light/2_build.py @@ -68,7 +68,8 @@ else: destFile = destDir + "/" + file[0:-len(".zonew")] + ".zonel" if (needUpdateLogRemoveDest(log, srcFile, destFile)): dependFile = dependDir + "/" + file[0:-len(".zonew")] + ".depend" - subprocess.call([ ExecTimeout, str(ZoneLightBuildTimeout), ZoneLighter, srcFile, destFile, ActiveProjectDirectory + "/generated/properties.cfg", dependFile ]) + callParallelProcess([ ExecTimeout, str(ZoneLightBuildTimeout), ZoneLighter, srcFile, destFile, ActiveProjectDirectory + "/generated/properties.cfg", dependFile ]) + flushParallelProcesses() printLog(log, "") # For each zone_light ig