Update from core4 branch

code/nel/tools/build_gamedata/0_setup.py

@@ -30,6 +30,7 @@ sys.path.append("configuration")
 parser = argparse.ArgumentParser(description='Ryzom Core - Build Gamedata - Setup')
 parser.add_argument('--noconf', '-nc', action='store_true')
 parser.add_argument('--noverify', '-nv', action='store_true')
+parser.add_argument('--preset', '-p', action='store_true')
 # parser.add_argument('--haltonerror', '-eh', action='store_true')
 parser.add_argument('--includeproject', '-ipj', nargs='+')
 parser.add_argument('--excludeproject', '-epj', nargs='+')
@@ -64,129 +65,222 @@ if not args.noconf:
 	except NameError:
 		BuildQuality = 1
 	try:
+		if args.preset:
+			DummyUnknownName
+		RemapLocalFrom
+	except NameError:
+		RemapLocalFrom = 'R:'
+	try:
+		if args.preset:
+			DummyUnknownName
+		RemapLocalTo
+	except NameError:
+		RemapLocalTo = os.getenv('RC_ROOT').replace('\\', '/')
+		if (not RemapLocalTo) or (not ':' in RemapLocalTo):
+			RemapLocalTo = 'R:'
+	try:
+		if args.preset:
+			DummyUnknownName
 		ToolDirectories
 	except NameError:
-		ToolDirectories = [ 'R:/build/dev/bin/Release', 'D:/libraries/external/bin' ]
+		ToolDirectories = [ 'R:/distribution/nel_tools_win_x64', 'R:/distribution/ryzom_tools_win_x64' ]
 	try:
 		ToolSuffix
 	except NameError:
 		ToolSuffix = ".exe"
 	try:
+		if args.preset:
+			DummyUnknownName
 		ScriptDirectory
 	except NameError:
 		ScriptDirectory = "R:/code/nel/tools/build_gamedata"
 	try:
+		if args.preset:
+			DummyUnknownName
 		WorkspaceDirectory
 	except NameError:
-		WorkspaceDirectory = "L:/workspace"
+		WorkspaceDirectory = "R:/leveldesign/workspace"
 	try:
+		if args.preset:
+			DummyUnknownName
 		DatabaseDirectory
 	except NameError:
-		DatabaseDirectory = "W:/database"
+		DatabaseDirectory = "R:/graphics"
 	try:
+		if args.preset:
+			DummyUnknownName
 		SoundDirectory
 	except NameError:
-		SoundDirectory = "V:"
+		SoundDirectory = "R:/sound"
 	try:
+		if args.preset:
+			DummyUnknownName
 		SoundDfnDirectory
 	except NameError:
-		SoundDfnDirectory = "V:/DFN"
+		SoundDfnDirectory = "R:/sound/DFN"
 	try:
+		if args.preset:
+			DummyUnknownName
 		ExportBuildDirectory
 	except NameError:
-		ExportBuildDirectory = "T:/export"
+		ExportBuildDirectory = "R:/pipeline/export"
 	try:
+		if args.preset:
+			DummyUnknownName
 		InstallDirectory
 	except NameError:
-		InstallDirectory = "T:/install"
+		InstallDirectory = "R:/pipeline/install"
 	try:
+		if args.preset:
+			DummyUnknownName
 		ClientDevDirectory
 	except NameError:
-		ClientDevDirectory = "T:/client_dev"
+		ClientDevDirectory = "R:/pipeline/client_dev"
+	try:
+		if args.preset:
+			DummyUnknownName
+		ClientDevLiveDirectory
+	except NameError:
+		ClientDevLiveDirectory = "R:/pipeline/client_dev_live"
 	try:
+		if args.preset:
+			DummyUnknownName
 		ClientPatchDirectory
 	except NameError:
-		ClientPatchDirectory = "T:/client_patch"
+		ClientPatchDirectory = "R:/pipeline/client_patch"
 	try:
+		if args.preset:
+			DummyUnknownName
 		ClientInstallDirectory
 	except NameError:
-		ClientInstallDirectory = "T:/client_install"
+		ClientInstallDirectory = "R:/pipeline/client_install"
 	try:
+		if args.preset:
+			DummyUnknownName
 		ShardInstallDirectory
 	except NameError:
-		ShardInstallDirectory = "T:/shard"
+		ShardInstallDirectory = "R:/pipeline/shard"
+	try:
+		if args.preset:
+			DummyUnknownName
+		ShardDevDirectory
+	except NameError:
+		ShardDevDirectory = "R:/pipeline/shard_dev"
 	try:
+		if args.preset:
+			DummyUnknownName
 		WorldEditInstallDirectory
 	except NameError:
-		WorldEditInstallDirectory = "T:/worldedit"
+		WorldEditInstallDirectory = "R:/pipeline/worldedit"
+	try:
+		if args.preset:
+			DummyUnknownName
+		WorldEditorFilesDirectory
+	except NameError:
+		WorldEditorFilesDirectory = "R:/code/ryzom/common/data_leveldesign/leveldesign/world_editor_files"
 	try:
+		if args.preset:
+			DummyUnknownName
 		LeveldesignDirectory
 	except NameError:
-		LeveldesignDirectory = "L:/leveldesign"
+		LeveldesignDirectory = "R:/leveldesign"
 	try:
+		if args.preset:
+			DummyUnknownName
 		LeveldesignDfnDirectory
 	except NameError:
-		LeveldesignDfnDirectory = "L:/leveldesign/DFN"
+		LeveldesignDfnDirectory = "R:/leveldesign/DFN"
 	try:
+		if args.preset:
+			DummyUnknownName
 		LeveldesignWorldDirectory
 	except NameError:
-		LeveldesignWorldDirectory = "L:/leveldesign/world"
+		LeveldesignWorldDirectory = "R:/leveldesign/world"
 	try:
+		if args.preset:
+			DummyUnknownName
 		PrimitivesDirectory
 	except NameError:
-		PrimitivesDirectory = "L:/primitives"
-	try:
-		GamedevDirectory
-	except NameError:
-		GamedevDirectory = "R:/code/ryzom/client/data/gamedev"
+		PrimitivesDirectory = "R:/leveldesign/primitives"
 	try:
-		DataShardDirectory
+		if args.preset:
+			DummyUnknownName
+		LeveldesignDataCommonDirectory
 	except NameError:
-		DataShardDirectory = "R:/code/ryzom/server/data_shard"
+		LeveldesignDataCommonDirectory = "R:/leveldesign/common"
 	try:
-		DataCommonDirectory
+		if args.preset:
+			DummyUnknownName
+		LeveldesignDataShardDirectory
 	except NameError:
-		DataCommonDirectory = "R:/code/ryzom/common/data_common"
+		LeveldesignDataShardDirectory = "R:/leveldesign/shard"
 	try:
-		LeveldesignDataShardDirectory
+		if args.preset:
+			DummyUnknownName
+		TranslationDirectory
 	except NameError:
-		LeveldesignDataShardDirectory = "L:/shard"
+		TranslationDirectory = "R:/leveldesign/translation"
 	try:
-		LeveldesignDataCommonDirectory
+		if args.preset:
+			DummyUnknownName
+		GamedevDirectory
 	except NameError:
-		LeveldesignDataCommonDirectory = "L:/common"
+		GamedevDirectory = "R:/code/ryzom/client/data/gamedev"
 	try:
-		TranslationDirectory
+		if args.preset:
+			DummyUnknownName
+		DataCommonDirectory
 	except NameError:
-		TranslationDirectory = "L:/translation"
+		DataCommonDirectory = "R:/code/ryzom/common/data_common"
 	try:
-		WorldEditorFilesDirectory
+		if args.preset:
+			DummyUnknownName
+		DataShardDirectory
 	except NameError:
-		WorldEditorFilesDirectory = "R:/code/ryzom/common/data_leveldesign/leveldesign/world_editor_files"
+		DataShardDirectory = "R:/code/ryzom/server/data_shard"
 	try:
+		if args.preset:
+			DummyUnknownName
 		WindowsExeDllCfgDirectories
 	except NameError:
-		WindowsExeDllCfgDirectories = [ 'C:/Program Files (x86)/Microsoft Visual Studio 9.0/VC/redist/x86', 'D:/libraries/external/bin', 'R:/build/dev/bin/Release', 'R:/code/ryzom/client', 'R:/code/nel/lib', 'R:/code/ryzom/bin', 'R:/code/ryzom/tools/client/client_config/bin' ]
+		# TODO: Separate 64bit and 32bit
+		WindowsExeDllCfgDirectories = [ '', 'R:/build/fv_x64/bin/Release', 'R:/distribution/external_x64', 'R:/code/ryzom/client', '', '', '' ]
 	try:
+		if args.preset:
+			DummyUnknownName
 		LinuxServiceExecutableDirectory
 	except NameError:
-		LinuxServiceExecutableDirectory = "S:/devls_x64/bin"
+		LinuxServiceExecutableDirectory = "R:/build/server_gcc/bin"
 	try:
+		if args.preset:
+			DummyUnknownName
 		LinuxClientExecutableDirectory
 	except NameError:
-		LinuxClientExecutableDirectory = "S:/devl_x64/bin"
+		LinuxClientExecutableDirectory = "R:/build/client_gcc/bin"
 	try:
+		if args.preset:
+			DummyUnknownName
+		PatchmanDevDirectory
+	except NameError:
+		PatchmanDevDirectory = "R:/patchman/terminal_dev"
+	try:
+		if args.preset:
+			DummyUnknownName
 		PatchmanCfgAdminDirectory
 	except NameError:
-		PatchmanCfgAdminDirectory = "R:/code/ryzom/server/patchman_cfg/admin_install"
+		PatchmanCfgAdminDirectory = "R:/patchman/admin_install"
 	try:
+		if args.preset:
+			DummyUnknownName
 		PatchmanCfgDefaultDirectory
 	except NameError:
-		PatchmanCfgDefaultDirectory = "R:/code/ryzom/server/patchman_cfg/default"
+		PatchmanCfgDefaultDirectory = "R:/patchman/default"
 	try:
+		if args.preset:
+			DummyUnknownName
 		PatchmanBridgeServerDirectory
 	except NameError:
-		PatchmanBridgeServerDirectory = "T:/bridge_server"
+		PatchmanBridgeServerDirectory = "R:/pipeline/bridge_server"
 	try:
 		SignToolExecutable
 	except NameError:
@@ -232,6 +326,7 @@ if not args.noconf:
 	printLog(log, "Use -- if you need to insert an empty value.")
 	printLog(log, "")
 	BuildQuality = int(askVar(log, "Build Quality", str(BuildQuality)))
+	if not args.preset:
 		ToolDirectories[0] = askVar(log, "[IN] Primary Tool Directory", ToolDirectories[0]).replace("\\", "/")
 		ToolDirectories[1] = askVar(log, "[IN] Secondary Tool Directory", ToolDirectories[1]).replace("\\", "/")
 		ToolSuffix = askVar(log, "Tool Suffix", ToolSuffix)
@@ -243,9 +338,11 @@ if not args.noconf:
 		ExportBuildDirectory = askVar(log, "[OUT] Export Build Directory", ExportBuildDirectory).replace("\\", "/")
 		InstallDirectory = askVar(log, "[OUT] Install Directory", InstallDirectory).replace("\\", "/")
 		ClientDevDirectory = askVar(log, "[OUT] Client Dev Directory", ClientDevDirectory).replace("\\", "/")
+		ClientDevLiveDirectory = askVar(log, "[OUT] Client Dev Live Directory", ClientDevLiveDirectory).replace("\\", "/")
 		ClientPatchDirectory = askVar(log, "[OUT] Client Patch Directory", ClientPatchDirectory).replace("\\", "/")
 		ClientInstallDirectory = askVar(log, "[OUT] Client Install Directory", ClientInstallDirectory).replace("\\", "/")
 		ShardInstallDirectory = askVar(log, "[OUT] Shard Data Install Directory", ShardInstallDirectory).replace("\\", "/")
+		ShardDevDirectory = askVar(log, "[OUT] Shard Dev Directory", ShardDevDirectory).replace("\\", "/")
 		WorldEditInstallDirectory = askVar(log, "[OUT] World Edit Data Install Directory", WorldEditInstallDirectory).replace("\\", "/")
 		LeveldesignDirectory = askVar(log, "[IN] Leveldesign Directory", LeveldesignDirectory).replace("\\", "/")
 		LeveldesignDfnDirectory = askVar(log, "[IN] Leveldesign DFN Directory", LeveldesignDfnDirectory).replace("\\", "/")
@@ -267,6 +364,7 @@ if not args.noconf:
 		WindowsExeDllCfgDirectories[6] = askVar(log, "[IN] Septenary Windows exe/dll/cfg Directory", WindowsExeDllCfgDirectories[6]).replace("\\", "/")
 		LinuxServiceExecutableDirectory = askVar(log, "[IN] Linux Service Executable Directory", LinuxServiceExecutableDirectory).replace("\\", "/")
 		LinuxClientExecutableDirectory = askVar(log, "[IN] Linux Client Executable Directory", LinuxClientExecutableDirectory).replace("\\", "/")
+		PatchmanDevDirectory = askVar(log, "[IN] Patchman Directory", PatchmanDevDirectory).replace("\\", "/")
 		PatchmanCfgAdminDirectory = askVar(log, "[IN] Patchman Cfg Admin Directory", PatchmanCfgAdminDirectory).replace("\\", "/")
 		PatchmanCfgDefaultDirectory = askVar(log, "[IN] Patchman Cfg Default Directory", PatchmanCfgDefaultDirectory).replace("\\", "/")
 		PatchmanBridgeServerDirectory = askVar(log, "[OUT] Patchman Bridge Server Patch Directory", PatchmanBridgeServerDirectory).replace("\\", "/")
@@ -317,6 +415,9 @@ if not args.noconf:
 	sf.write("# Quality option for this site (1 for BEST, 0 for DRAFT)\n")
 	sf.write("BuildQuality = " + str(BuildQuality) + "\n")
 	sf.write("\n")
+	sf.write("RemapLocalFrom = \"" + str(RemapLocalFrom) + "\"\n")
+	sf.write("RemapLocalTo = \"" + str(RemapLocalTo) + "\"\n")
+	sf.write("\n")
 	sf.write("ToolDirectories = " + str(ToolDirectories) + "\n")
 	sf.write("ToolSuffix = \"" + str(ToolSuffix) + "\"\n")
 	sf.write("\n")
@@ -333,9 +434,11 @@ if not args.noconf:
 	sf.write("# Install directories\n")
 	sf.write("InstallDirectory = \"" + str(InstallDirectory) + "\"\n")
 	sf.write("ClientDevDirectory = \"" + str(ClientDevDirectory) + "\"\n")
+	sf.write("ClientDevLiveDirectory = \"" + str(ClientDevLiveDirectory) + "\"\n")
 	sf.write("ClientPatchDirectory = \"" + str(ClientPatchDirectory) + "\"\n")
 	sf.write("ClientInstallDirectory = \"" + str(ClientInstallDirectory) + "\"\n")
 	sf.write("ShardInstallDirectory = \"" + str(ShardInstallDirectory) + "\"\n")
+	sf.write("ShardDevDirectory = \"" + str(ShardDevDirectory) + "\"\n")
 	sf.write("WorldEditInstallDirectory = \"" + str(WorldEditInstallDirectory) + "\"\n")
 	sf.write("\n")
 	sf.write("# Utility directories\n")
@@ -357,6 +460,7 @@ if not args.noconf:
 	sf.write("WindowsExeDllCfgDirectories = " + str(WindowsExeDllCfgDirectories) + "\n")
 	sf.write("LinuxServiceExecutableDirectory = \"" + str(LinuxServiceExecutableDirectory) + "\"\n")
 	sf.write("LinuxClientExecutableDirectory = \"" + str(LinuxClientExecutableDirectory) + "\"\n")
+	sf.write("PatchmanDevDirectory = \"" + str(PatchmanDevDirectory) + "\"\n")
 	sf.write("PatchmanCfgAdminDirectory = \"" + str(PatchmanCfgAdminDirectory) + "\"\n")
 	sf.write("PatchmanCfgDefaultDirectory = \"" + str(PatchmanCfgDefaultDirectory) + "\"\n")
 	sf.write("PatchmanBridgeServerDirectory = \"" + str(PatchmanBridgeServerDirectory) + "\"\n")
@@ -374,7 +478,17 @@ if not args.noconf:
 	sf.write("\n")
 	sf.write("\n")
 	sf.write("# end of file\n")
+	sf.flush()
 	sf.close()
+	sf = open("configuration/buildsite_local.py", "w")
+	sfr = open("configuration/buildsite.py", "r")
+	for l in sfr:
+		sf.write(l.replace(RemapLocalFrom + '/', RemapLocalTo + '/'))
+	sf.flush()
+	sfr.close()
+	sf.close()
+
+from buildsite_local import *
 
 sys.path.append(WorkspaceDirectory)
 from projects import *
@@ -415,6 +529,7 @@ printLog(log, "")
 # Additional directories
 printLog(log, ">>> Setup additional directories <<<")
 mkPath(log, ClientDevDirectory)
+mkPath(log, ClientDevLiveDirectory)
 mkPath(log, ClientPatchDirectory)
 mkPath(log, ClientInstallDirectory)
 
@@ -470,6 +585,7 @@ if not args.noverify:
 	findTool(log, ToolDirectories, TranslationToolsTool, ToolSuffix)
 	findTool(log, ToolDirectories, BuildWorldPackedColTool, ToolSuffix)
 	findTool(log, ToolDirectories, R2IslandsTexturesTool, ToolSuffix)
+	findTool(log, ToolDirectories, PatchmanServiceTool, ToolSuffix)
 
 log.close()
 if os.path.isfile("0_setup.log"):

code/nel/tools/build_gamedata/1_export.py

@@ -47,7 +47,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)

code/nel/tools/build_gamedata/2_build.py

@@ -47,7 +47,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)

code/nel/tools/build_gamedata/3_install.py

@@ -47,7 +47,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)

code/nel/tools/build_gamedata/9_upload.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 try:

code/nel/tools/build_gamedata/a1_worldedit_data.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)

code/nel/tools/build_gamedata/b1_client_dev.py

@@ -24,14 +24,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 
-import time, sys, os, shutil, subprocess, distutils.dir_util
+import time, sys, os, shutil, subprocess, distutils.dir_util, socket
 sys.path.append("configuration")
 
 if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)
@@ -45,17 +45,37 @@ printLog(log, "-------")
 printLog(log, time.strftime("%Y-%m-%d %H:%MGMT", time.gmtime(time.time())))
 printLog(log, "")
 
-if not os.path.isfile(ClientDevDirectory + "/client.cfg"):
-	printLog(log, ">>> Generate client.cfg <<<")
-	cfg = open(ClientDevDirectory + "/client.cfg", "w")
+mkPath(log, ClientDevLiveDirectory)
+if not os.path.isfile(ClientDevLiveDirectory + "/client.cfg"):
+	printLog(log, ">>> Generate live dev client.cfg <<<")
+	cfg = open(ClientDevLiveDirectory + "/client.cfg", "w")
 	cfg.write("RootConfigFilename   = \"client_default.cfg\";\n")
 	cfg.write("PreDataPath          = {\n")
-	cfg.write("\t\"" + InstallDirectory + "\", \"user\", \"patch\", \"data\", \"examples\" \n")
+	cfg.write("\t\"user\", \"patch\", \"" + DataCommonDirectory + "\", \"" + GamedevDirectory + "\", \"" + LeveldesignDirectory + "/translation/translated\", \"" + InstallDirectory + "\", \"data\", \"examples\" \n")
 	cfg.write("};\n")
+	cfg.write("PreLoadPath          = \"" + InstallDirectory + "\";\n")
 	cfg.write("PatchWanted          = 0;\n")
 	cfg.write("DisplayLuaDebugInfo  = 1;\n")
 	cfg.write("AllowDebugLua        = 1;\n")
 	cfg.write("FullScreen           = 0;\n")
+	cfg.flush()
+	cfg.close()
+	printLog(log, "")
+
+mkPath(log, ClientDevDirectory)
+if not os.path.isfile(ClientDevDirectory + "/client.cfg"):
+	printLog(log, ">>> Generate local dev client.cfg <<<")
+	cfg = open(ClientDevDirectory + "/client.cfg", "w")
+	cfgr = open(ClientDevLiveDirectory + "/client.cfg", "r")
+	for l in cfgr:
+		cfg.write(l)
+	cfgr.close()
+	cfg.write("StartupHost          = \"http://" + socket.gethostname() + ":9042\";\n")
+	cfg.write("Application          = {\n")
+	cfg.write("	\"dev\", \"./client_ryzom_r.exe\", \"./\" \n")
+	cfg.write("};\n")
+	cfg.flush()
+	cfg.close()
 	printLog(log, "")
 
 printLog(log, ">>> Install data <<<")
@@ -63,13 +83,22 @@ for category in InstallClientData:
 	if (category["UnpackTo"] != None):
 		printLog(log, "CATEGORY " + category["Name"])
 		targetPath = ClientDevDirectory
+		targetPathLive = ClientDevLiveDirectory
 		if (category["UnpackTo"] != ""):
 			targetPath += "/" + category["UnpackTo"]
+			targetPathLive += "/" + category["UnpackTo"]
 		mkPath(log, targetPath)
+		mkPath(log, targetPathLive)
 		for package in category["Packages"]:
 			printLog(log, "PACKAGE " + package[0])
 			mkPath(log, InstallDirectory + "/" + package[0])
+			if "exedll" in package[0]:
+				if package[0] == "exedll": # or package[0] == platformExeDll # TODO: 64-bit and Linux separation of exedll, only include one
+					copyFileIfNeeded(log, InstallDirectory + "/" + package[0] + "/client_default.cfg", targetPath)
+					copyFileIfNeeded(log, InstallDirectory + "/" + package[0] + "/client_default.cfg", targetPathLive)
+			else:
 				copyFilesNoTreeIfNeeded(log, InstallDirectory + "/" + package[0], targetPath)
+				copyFilesNoTreeIfNeeded(log, InstallDirectory + "/" + package[0], targetPathLive)
 printLog(log, "")
 
 log.close()

code/nel/tools/build_gamedata/b2_shard_data.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)

code/nel/tools/build_gamedata/c1_shard_patch.py

@@ -35,7 +35,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)
@@ -49,6 +49,9 @@ printLog(log, "-------")
 printLog(log, time.strftime("%Y-%m-%d %H:%MGMT", time.gmtime(time.time())))
 printLog(log, "")
 
+# Find tools
+SevenZip = findTool(log, ToolDirectories, SevenZipTool, ToolSuffix)
+
 # List the directories that will be used
 archiveDirectories = [ ]
 for dir in InstallShardDataDirectories:
@@ -89,6 +92,9 @@ else:
 printLog(log, "")
 
 if not args.admininstall:
+	if SevenZip == "":
+		toolLogFail(log, SevenZipTool, ToolSuffix)
+	else:
 		printLog(log, ">>> Create new version <<<")
 		newVersion = 1
 		vstr = str(newVersion).zfill(6)
@@ -100,11 +106,14 @@ if not args.admininstall:
 		mkPath(log, vpath)
 		for dir in archiveDirectories:
 			mkPath(log, ShardInstallDirectory + "/" + dir)
-		tgzPath = vpath + "/" + dir + ".tgz"
-		printLog(log, "WRITE " + tgzPath)
-		tar = tarfile.open(tgzPath, "w:gz")
-		tar.add(ShardInstallDirectory + "/" + dir, arcname = dir)
-		tar.close()
+			# tgzPath = vpath + "/" + dir + ".tgz"
+			# printLog(log, "WRITE " + tgzPath)
+			# tar = tarfile.open(tgzPath, "w:gz")
+			# tar.add(ShardInstallDirectory + "/" + dir, arcname = dir)
+			# tar.close()
+			sevenZipPath = vpath + "/" + dir + ".7z"
+			printLog(log, "WRITE " + sevenZipPath)
+			subprocess.call([ SevenZip, "a", sevenZipPath, ShardInstallDirectory + "/" + dir ])
 		printLog(log, "")
 
 log.close()

code/nel/tools/build_gamedata/configuration/scripts.py

@@ -23,7 +23,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 
-import time, sys, os, shutil, subprocess, distutils.dir_util
+import time, sys, os, shutil, subprocess, distutils.dir_util, multiprocessing, math
 
 ActiveProjectDirectory = os.getenv("NELBUILDACTIVEPROJECT", "configuration/project")
 sys.path.append(ActiveProjectDirectory)
@@ -32,6 +32,27 @@ def printLog(log, text):
 	log.write(text + "\n")
 	print text
 
+pendingProcesses = []
+processLimit = math.ceil(multiprocessing.cpu_count() * 0.75)
+
+def callParallelProcess(command):
+	res = 0
+	if len(pendingProcesses) >= processLimit:
+		waitingProc = pendingProcesses.pop(0)
+		res = waitingProc.wait()
+	proc = subprocess.Popen(command)
+	pendingProcesses.append(proc)
+	return res
+
+def flushParallelProcesses():
+	res = 0
+	while (len(pendingProcesses) > 0):
+		waitingProc = pendingProcesses.pop(0)
+		procRes = waitingProc.wait()
+		if procRes != 0:
+			res = procRes
+	return res
+
 def mkPath(log, path):
 	printLog(log, "DIR " + path)
 	distutils.dir_util.mkpath(path)

code/nel/tools/build_gamedata/configuration/tools.py

@@ -98,3 +98,5 @@ PatchGenTool = "patch_gen"
 TranslationToolsTool = "translation_tools"
 BuildWorldPackedColTool = "build_world_packed_col"
 R2IslandsTexturesTool = "r2_islands_textures"
+PatchmanServiceTool = "ryzom_patchman_service"
+SevenZipTool = "7za"

code/nel/tools/build_gamedata/d1_client_patch.py

@@ -35,7 +35,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)
@@ -123,7 +123,9 @@ else:
 	printLog(log, "")
 	printLog(log, ">>> Make bnp <<<")
 	targetPath = ClientPatchDirectory + "/bnp"
+	tagPath = ClientPatchDirectory + "/bnp_tag"
 	mkPath(log, targetPath)
+	mkPath(log, tagPath)
 	for category in InstallClientData:
 		packExt = ".bnp"
 		if (category["StreamedPackages"]):
@@ -133,25 +135,32 @@ else:
 			sourcePath = InstallDirectory + "/" + package[0]
 			mkPath(log, sourcePath)
 			targetBnp = targetPath + "/" + package[0] + packExt
+			tagBnp = tagPath + "/" + package[0] + packExt + ".tag"
 			if (len(package[1]) > 0):
 				targetBnp = targetPath + "/" + package[1][0]
+				tagBnp = tagPath + "/" + package[1][0] + ".tag"
 				printLog(log, "TARGET " + package[1][0])
 			needUpdateBnp = 1
 			if (len(package) > 2):
-				needUpdateBnp = needUpdate(log, sourcePath + "/" + package[2], targetBnp)
+				needUpdateBnp = needUpdate(log, sourcePath + "/" + package[2], tagBnp)
 			else:
-				needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp)
+				needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, tagBnp)
 			if (needUpdateBnp):
+				subRet = 0
+				open(tagBnp, 'a').close()
+				os.utime(tagBnp, None)
 				if (category["StreamedPackages"]):
 					printLog(log, "SNP " + targetBnp)
 					# cwDir = os.getcwd().replace("\\", "/")
 					# toolDir = os.path.dirname(Lzma).replace("\\", "/")
 					# os.chdir(toolDir)
-					subprocess.call([ SnpMake, "-p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:])
+					subRet = subprocess.call([ SnpMake, "-p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:])
 					# os.chdir(cwDir)
 				else:
 					printLog(log, "BNP " + targetBnp)
-					subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:])
+					subRet = subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:])
+				if (subRet != 0):
+					os.remove(tagBnp)
 			else:
 				printLog(log, "SKIP " + targetBnp)
 	printLog(log, "")

code/nel/tools/build_gamedata/d2_client_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from tools import *
 
 sys.path.append(WorkspaceDirectory)

code/nel/tools/build_gamedata/processes/_dummy/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/ai_wmap/2_build.py

@@ -68,13 +68,15 @@ else:
 		printLog(log, ">>> Generate sized wmap <<<")
 		subprocess.call([ AiBuildWmap, "pacsBuildGabarit " + AiWmapContinentName ])
 		printLog(log, ">>> Generate cwmaps for each size <<<")
-		subprocess.call([ AiBuildWmap, "pacsBuildWmap " + AiWmapContinentName + "_0" ])
-		subprocess.call([ AiBuildWmap, "pacsBuildWmap " + AiWmapContinentName + "_1" ])
-		subprocess.call([ AiBuildWmap, "pacsBuildWmap " + AiWmapContinentName + "_2" ])
+		callParallelProcess([ AiBuildWmap, "pacsBuildWmap " + AiWmapContinentName + "_0" ])
+		callParallelProcess([ AiBuildWmap, "pacsBuildWmap " + AiWmapContinentName + "_1" ])
+		callParallelProcess([ AiBuildWmap, "pacsBuildWmap " + AiWmapContinentName + "_2" ])
+		flushParallelProcesses()
 		printLog(log, ">>> Generate bitmap for each size <<<")
-		subprocess.call([ AiBuildWmap, "pacsBuildBitmap " + AiWmapContinentName + "_0" ])
-		subprocess.call([ AiBuildWmap, "pacsBuildBitmap " + AiWmapContinentName + "_1" ])
-		subprocess.call([ AiBuildWmap, "pacsBuildBitmap " + AiWmapContinentName + "_2" ])
+		callParallelProcess([ AiBuildWmap, "pacsBuildBitmap " + AiWmapContinentName + "_0" ])
+		callParallelProcess([ AiBuildWmap, "pacsBuildBitmap " + AiWmapContinentName + "_1" ])
+		callParallelProcess([ AiBuildWmap, "pacsBuildBitmap " + AiWmapContinentName + "_2" ])
+		flushParallelProcesses()
 		printLog(log, ">>> Clear height maps for size 1 and 2 <<<")
 		subprocess.call([ AiBuildWmap, "pacsClearHeightmap " + AiWmapContinentName ])
 		printLog(log, ">>> Cut tga for world editor <<<")

code/nel/tools/build_gamedata/processes/ai_wmap/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/anim/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/cartographer/0_setup.py

@@ -46,6 +46,7 @@ printLog(log, "")
 # Setup build directories
 printLog(log, ">>> Setup build directories <<<")
 mkPath(log, ExportBuildDirectory + "/" + CartographerBuildDirectory)
+mkPath(log, ExportBuildDirectory + "/" + CartographerMapBuildDirectory)
 
 # Setup lookup directories
 printLog(log, ">>> Setup lookup directories <<<")
@@ -65,6 +66,7 @@ for dir in PropertiesExportBuildSearchPaths:
 # Setup client directories
 printLog(log, ">>> Setup install directories <<<")
 mkPath(log, InstallDirectory + "/" + CartographerInstallDirectory)
+mkPath(log, InstallDirectory + "/" + IslandsInstallDirectory)
 
 # Setup client directories
 printLog(log, ">>> Setup configuration <<<")
@@ -95,7 +97,11 @@ cfg.write("\t\"" + CartographerContinent + "\", \n")
 cfg.write("};\n")
 cfg.write("\n")
 cfg.write("SeasonSuffixes = {\n")
-for suffix in MultipleTilesPostfix:
+if CartographerSeasonSuffixes:
+	for suffix in CartographerSeasonSuffixes:
+		cfg.write("\t\"" + suffix + "\", \n")
+else:
+	for suffix in MultipleTilesPostfix:
 		cfg.write("\t\"" + suffix + "\", \n")
 cfg.write("};\n")
 cfg.write("\n")
@@ -103,7 +109,7 @@ cfg.write("InverseZTest = true;\n")
 cfg.write("Vegetation = true;\n")
 cfg.write("MeterPixelSize = 2;\n")
 cfg.write("\n")
-cfg.write("CompleteIslandsFile = \"r2_islands.xml\";\n")
+cfg.write("CompleteIslandsFile = \"" + ExportBuildDirectory + "/" + CartographerBuildDirectory + "/" + IslandsXmlFile + "\";\n")
 cfg.write("EntryPointsFile = \"r2_entry_points.txt\";\n")
 cfg.write("\n")
 cfg.close()

code/nel/tools/build_gamedata/processes/cartographer/2_build.py

@@ -45,6 +45,7 @@ printLog(log, "")
 
 # Find tools
 R2IslandsTextures = findTool(log, ToolDirectories, R2IslandsTexturesTool, ToolSuffix)
+TgaToDds = findTool(log, ToolDirectories, TgaToDdsTool, ToolSuffix)
 
 if R2IslandsTextures == "":
 	toolLogFail(log, R2IslandsTexturesTool, ToolSuffix)
@@ -53,9 +54,33 @@ else:
 	cfgPath = ActiveProjectDirectory + "/generated/island_screenshots.cfg"
 	shutil.copy(cfgPath, "island_screenshots.cfg")
 	printLog(log, ">>> Build cartographer <<<")
+	mkPath(log, ExportBuildDirectory + "/" + CartographerBuildDirectory)
 	subprocess.call([ R2IslandsTextures ])
 printLog(log, "")
 
+printLog(log, ">>> Compress cartographer maps to DDS <<<")
+if TgaToDds == "":
+	toolLogFail(log, TgaToDdsTool, ToolSuffix)
+else:
+	destPath = ExportBuildDirectory + "/" + CartographerMapBuildDirectory
+	mkPath(log, destPath)
+	sourcePath = ExportBuildDirectory + "/" + CartographerBuildDirectory
+	mkPath(log, sourcePath)
+	files = os.listdir(sourcePath)
+	len_tga_png = len(".tga")
+	len_dds = len(".dds")
+	for fileName in files:
+		if isLegalFileName(fileName):
+			sourceFile = sourcePath + "/" + fileName
+			if os.path.isfile(sourceFile):
+				if (fileName[-len_tga_png:].lower() == ".tga") or (fileName[-len_tga_png:].lower() == ".png"):
+					destFile = destPath + "/" + os.path.basename(fileName)[0:-len_tga_png] + ".dds"
+					if needUpdateLogRemoveDest(log, sourceFile, destFile):
+						subprocess.call([ TgaToDds, sourceFile, "-o", destFile, "-m" ])
+			elif not os.path.isdir(sourceFile):
+				printLog(log, "FAIL ?! file not dir or file ?! " + sourceFile)
+printLog(log, "")
+
 log.close()
 
 

code/nel/tools/build_gamedata/processes/cartographer/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *
@@ -43,12 +43,21 @@ printLog(log, "-------")
 printLog(log, time.strftime("%Y-%m-%d %H:%MGMT", time.gmtime(time.time())))
 printLog(log, "")
 
+
 installPath = InstallDirectory + "/" + CartographerInstallDirectory
-mkPath(log, installPath)
+islandsInstallPath = InstallDirectory + "/" + IslandsInstallDirectory
+
 
 printLog(log, ">>> Install cartographer <<<")
+
+mkPath(log, ExportBuildDirectory + "/" + CartographerMapBuildDirectory)
+mkPath(log, installPath)
+copyFilesExtNoTreeIfNeeded(log, ExportBuildDirectory + "/" + CartographerMapBuildDirectory, installPath, ".dds")
+
 mkPath(log, ExportBuildDirectory + "/" + CartographerBuildDirectory)
-copyFilesNoTreeIfNeeded(log, ExportBuildDirectory + "/" + CartographerBuildDirectory, installPath)
+mkPath(log, islandsInstallPath)
+copyFilesExtNoTreeIfNeeded(log, ExportBuildDirectory + "/" + CartographerBuildDirectory, islandsInstallPath, ".xml")
+
 
 printLog(log, "")
 log.close()

code/nel/tools/build_gamedata/processes/cegui/3_install.py

@@ -30,7 +30,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/clodbank/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/copy/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/displace/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/farbank/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *

code/nel/tools/build_gamedata/processes/font/1_export.py

@@ -49,6 +49,7 @@ mkPath(log, fontExportDir)
 for dir in FontSourceDirectories:
 	mkPath(log, DatabaseDirectory + "/" + dir)
 	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, fontExportDir, ".ttf")
+	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, fontExportDir, ".otf")
 	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, fontExportDir, ".afm")
 	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, fontExportDir, ".pfb")
 	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, fontExportDir, ".pfm")

code/nel/tools/build_gamedata/processes/font/3_install.py

@@ -31,7 +31,7 @@ if os.path.isfile("log.log"):
 	os.remove("log.log")
 log = open("log.log", "w")
 from scripts import *
-from buildsite import *
+from buildsite_local import *
 from process import *
 from tools import *
 from directories import *
@@ -50,6 +50,7 @@ mkPath(log, fontExportDir)
 
 printLog(log, ">>> Install font <<<")
 copyFilesExtNoTreeIfNeeded(log, fontExportDir, installPath, ".ttf")
+copyFilesExtNoTreeIfNeeded(log, fontExportDir, installPath, ".otf")
 copyFilesExtNoTreeIfNeeded(log, fontExportDir, installPath, ".afm")
 copyFilesExtNoTreeIfNeeded(log, fontExportDir, installPath, ".pfb")
 copyFilesExtNoTreeIfNeeded(log, fontExportDir, installPath, ".pfm")

code/nel/tools/build_gamedata/processes/ig/2_build.py

@@ -94,6 +94,7 @@ def igElevation(inputIgDir, outputIgDir):
 		cf.write("ZFactor1 = " + LigoExportZFactor1 + ";\n")
 		cf.write("HeightMapFile2 = \"" + DatabaseDirectory + "/" + LigoBaseSourceDirectory + "/" + LigoExportHeightmap2 + "\";\n")
 		cf.write("ZFactor2 = " + LigoExportZFactor2 + ";\n")
+		cf.write("ExtendCoords = " + str(LigoExportExtendCoords) + ";\n")
 		cf.write("\n")
 		cf.write("LandFile = \"" + DatabaseDirectory + "/" + LigoBaseSourceDirectory + "/" + LigoExportLand + "\";\n")
 		cf.write("\n")
@@ -264,7 +265,8 @@ printLog(log, ">>> Merge other IGs <<<") # (not true merge, since not necesserar
 mkPath(log, ExportBuildDirectory + "/" + IgStaticOtherExportDirectory)
 mkPath(log, ExportBuildDirectory + "/" + LigoIgOtherBuildDirectory)
 mkPath(log, ExportBuildDirectory + "/" + IgOtherBuildDirectory)
-copyFilesExtNoTreeIfNeeded(log, ExportBuildDirectory + "/" + IgStaticOtherExportDirectory, ExportBuildDirectory + "/" + IgOtherBuildDirectory, ".ig")
+# copyFilesExtNoTreeIfNeeded(log, ExportBuildDirectory + "/" + IgStaticOtherExportDirectory, ExportBuildDirectory + "/" + IgOtherBuildDirectory, ".ig")
+igElevation(ExportBuildDirectory + "/" + IgStaticOtherExportDirectory, ExportBuildDirectory + "/" + IgOtherBuildDirectory)
 copyFilesExtNoTreeIfNeeded(log, ExportBuildDirectory + "/" + LigoIgOtherBuildDirectory, ExportBuildDirectory + "/" + IgOtherBuildDirectory, ".ig")
 
 log.close()