Added: #1440 Configuration of multiple custom data directories

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/database_status.cpp

@@ -49,52 +49,75 @@ namespace PIPELINE {
 namespace {
 
 /// Input must be normalized path
-bool isInDatabaseDirectoryFast(const std::string &path)
+bool isInRootDirectoryFast(std::string &rootDirectoryName, std::string &rootDirectoryPath, const std::string &path)
 {
-	return path.find(g_DatabaseDirectory) == 0;
+	//return path.find(g_DatabaseDirectory) == 0;
+	CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
+	for (uint i = 0; i < rootDirectories.size(); ++i)
+	{
+		rootDirectoryName = rootDirectories.asString(i);
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
+		rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		if (path.find(rootDirectoryPath) == 0) return true;
+	}
+	return false;
 }
 
 /// Input must be normalized path
-bool isInPipelineDirectoryFast(const std::string &path)
+bool isInWorkspaceDirectoryFast(const std::string &path)
 {
-	return path.find(g_PipelineDirectory) == 0;
+	return path.find(g_WorkDir) == 0;
 }
 
 /// Input must be normalized path in database directory
-std::string dropDatabaseDirectoryFast(const std::string &path)
+std::string dropRootDirectoryFast(const std::string &path, const std::string &rootDirectoryPath)
 {
-	return path.substr(g_DatabaseDirectory.length());
+	return path.substr(rootDirectoryPath.length());
 }
 
 /// Input must be normalized path in pipeline directory
-std::string dropPipelineDirectoryFast(const std::string &path)
+std::string dropWorkspaceDirectoryFast(const std::string &path)
 {
-	return path.substr(g_PipelineDirectory.length());
+	return path.substr(g_WorkDir.length());
 }
 
-/// Create status file path
+} /* anonymous namespace */
-std::string getStatusFilePath(const std::string &path)
+
+std::string getMetaFilePath(const std::string &path, const std::string &dotSuffix)
 {
 	std::string stdPath = CPath::standardizePath(path, false);
-	if (isInDatabaseDirectoryFast(stdPath))
+	if (isInWorkspaceDirectoryFast(stdPath))
-	{
-		std::string relPath = dropDatabaseDirectoryFast(stdPath);
-		return g_PipelineDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + relPath + PIPELINE_DATABASE_STATUS_SUFFIX;
-	}
-	else if (isInPipelineDirectoryFast(stdPath))
 	{
 		// TODO_TEST
-		std::string relPath = dropPipelineDirectoryFast(stdPath);
+		std::string relPath = dropWorkspaceDirectoryFast(stdPath);
 		std::string::size_type slashPos = relPath.find_first_of('/');
 		std::string proProName = relPath.substr(0, slashPos);
 		std::string subPath = relPath.substr(slashPos);
-		return g_PipelineDirectory + proProName + PIPELINE_DATABASE_STATUS_SUFFIX + subPath + PIPELINE_DATABASE_STATUS_SUFFIX;
+		return g_WorkDir + proProName + PIPELINE_DATABASE_META_SUFFIX + subPath + dotSuffix;
 	}
 	else
 	{
-		nlerror("Path is not in database or pipeline");
+		std::string rootDirectoryName;
-		return path + PIPELINE_DATABASE_STATUS_SUFFIX;
+		std::string rootDirectoryPath;
+		if (isInRootDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath))
+		{
+			std::string relPath = dropRootDirectoryFast(stdPath, rootDirectoryPath);
+			return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_ROOT + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix;
 		}
+		else
+		{
+			nlerror("Path is not in database or pipeline (%s)", path.c_str());
+			return path + dotSuffix;
+		}
+	}
+}
+
+namespace {
+
+/// Create status file path
+std::string getStatusFilePath(const std::string &path)
+{
+	return getMetaFilePath(path, PIPELINE_DATABASE_STATUS_SUFFIX);
 }
 
 } /* anonymous namespace */
@@ -119,8 +142,8 @@ void CFileStatus::serial(NLMISC::IStream &stream)
 
 CDatabaseStatus::CDatabaseStatus()
 {
-	CFile::createDirectoryTree(g_PipelineDirectory + PIPELINE_DATABASE_STATUS_SUBDIR);
+	//CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR);
-	CFile::createDirectoryTree(g_PipelineDirectory + PIPELINE_DATABASE_ERRORS_SUBDIR);
+	//CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_ERRORS_SUBDIR);
 }
 
 CDatabaseStatus::~CDatabaseStatus()
@@ -186,7 +209,7 @@ public:
 			bool firstSeen = false;
 			uint32 time = CTime::getSecondsSince1970();
 			uint32 fmdt = CFile::getFileModificationDate(FilePath);
-			std::string statusPath = getStatusFilePath(FilePath); // g_PipelineDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status";
+			std::string statusPath = getStatusFilePath(FilePath); // g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status";
 			StatusMutex->lock_shared();
 			if (CFile::fileExists(statusPath))
 			{
@@ -402,7 +425,15 @@ void CDatabaseStatus::updateDatabaseStatus(const CCallback<void> &callback)
 	}
 
 	std::vector<std::string> paths;
-	paths.push_back(g_DatabaseDirectory);
+	//paths.push_back(g_DatabaseDirectory);
+	CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
+	for (uint i = 0; i < rootDirectories.size(); ++i)
+	{
+		std::string rootDirectoryName = rootDirectories.asString(i);
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
+		std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		paths.push_back(rootDirectoryPath);
+	}
 	updateDatabaseStatus(callback, paths, false, true);
 }
 

code/nel/tools/pipeline/service/database_status.h

@@ -48,13 +48,18 @@ namespace NLMISC {
 
 namespace PIPELINE {
 
-#define PIPELINE_DATABASE_STATUS_SUBDIR "database.status/"
+//#define PIPELINE_DATABASE_STATUS_SUBDIR "database.status/"
-#define PIPELINE_DATABASE_ERRORS_SUBDIR "database.errors/"
+//#define PIPELINE_DATABASE_ERRORS_SUBDIR "database.errors/"
-#define PIPELINE_DATABASE_DEPEND_SUBDIR "database.depend/"
+//#define PIPELINE_DATABASE_DEPEND_SUBDIR "database.depend/"
+
 #define PIPELINE_DATABASE_STATUS_SUFFIX ".status"
 #define PIPELINE_DATABASE_ERRORS_SUFFIX ".errors"
 #define PIPELINE_DATABASE_DEPEND_SUFFIX ".depend"
 
+#define PIPELINE_DATABASE_META_SUFFIX ".meta"
+
+std::string getMetaFilePath(const std::string &path, const std::string &dotSuffix);
+
 // Status is generated CRC32 for reference.
 // Errors are errors caused by using this file as an input or output file.
 

code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg

@@ -1,14 +1,41 @@
 
+// Intermediate working directory used by the pipeline.
+WorkspaceDirectory = "W:/workspace";
+
+// Workspace location
+WorkspaceDfnDirectory = "L:/leveldesign/dfn";
+WorkspaceSheetDirectory = "L:/leveldesign/pipeline";
 WorkspaceSheet = "ryzom_core.pipeline_workspace";
 
-DatabaseDirectory = "W:/database";
+// Lists all directories that can be accessed by the pipeline processes
-PipelineDirectory = "W:/pipeline";
+RootDirectories = 
+{
+	"DatabaseDirectory", 
+	"LeveldesignDfnDirectory", 
+	"LeveldesignWorldDirectory", 
+	"LeveldesignEcosystemsDirectory", 
+	"LeveldesignGameElemDirectory", 
+	"LeveldesignGameElementDirectory", 
+	"PrimitivesDirectory", 
+};
 
-LeveldesignDirectory = "L:/leveldesign";
+// Lists all directories that can be accessed by georges
+GeorgesDirectories = 
+{
+	"LeveldesignDfnDirectory", 
+	"LeveldesignWorldDirectory", 
+	"LeveldesignEcosystemsDirectory", 
+	"LeveldesignGameElemDirectory", 
+	"LeveldesignGameElementDirectory", 
+};
+
+DatabaseDirectory = "W:/database";
 LeveldesignDfnDirectory = "L:/leveldesign/dfn";
-LeveldesignPipelineDirectory = "L:/leveldesign/pipeline";
+LeveldesignWorldDirectory = "L:/leveldesign/World";
-LeveldesignWorldDirectory = "L:/leveldesign/world";
+LeveldesignEcosystemsDirectory = "L:/leveldesign/Ecosystems";
-// PrimitivesDirectory = "L:/primitives";
+LeveldesignGameElemDirectory = "L:/leveldesign/Game_elem";
+LeveldesignGameElementDirectory = "L:/leveldesign/game_element";
+PrimitivesDirectory = "L:/primitives";
 
 // ToolDirectories = { "R:/build/dev/bin/Release", "D:/libraries/external/bin" };
 // ToolSuffix = ".exe";

code/nel/tools/pipeline/service/module_pipeline_master.cpp

@@ -50,6 +50,8 @@ namespace PIPELINE {
 #define PIPELINE_INFO_MASTER_RELOAD_SHEETS "M_RELOAD_SHEETS"
 #define PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE "M_UPD_DB_FOR_S"
 
+#define PIPELINE_INFO_CODE_ERROR_UNMACRO "CODE_ERROR_UNMACRO"
+
 /**
  * \brief CModulePipelineMaster
  * \date 2012-03-03 16:26GMT
@@ -67,13 +69,15 @@ class CModulePipelineMaster :
 			: Master(master), 
 			Proxy(moduleProxy), 
 			ActiveTaskId(0),
-			SheetsOk(true) { }
+			SheetsOk(true),
+			SaneBehaviour(3) { }
 		CModulePipelineMaster *Master;
 		CModulePipelineSlaveProxy Proxy;
 		std::vector<std::string> Vector;
 		uint32 ActiveTaskId;
 		bool SheetsOk;
 		std::vector<uint32> PluginsAvailable;
+		sint SaneBehaviour;
 		
 		~CSlave()
 		{
@@ -92,7 +96,7 @@ class CModulePipelineMaster :
 
 		bool canAcceptTask()
 		{
-			return SheetsOk && (ActiveTaskId == 0);
+			return SheetsOk && (ActiveTaskId == 0) && SaneBehaviour > 0;
 		}
 	};
 	
@@ -201,7 +205,25 @@ public:
 	{
 		CSlave *slave = m_Slaves[sender];
 		CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE);
-		g_DatabaseStatus->updateDatabaseStatus(CCallback<void>(slave, &CSlave::cbUpdateDatabaseStatus), slave->Vector, false, false);
+
+		bool ok = true;
+		for (std::vector<std::string>::size_type i = 0; i < slave->Vector.size(); ++i)
+		{
+			// not real security, just an extra for catching coding errors
+			std::string &str = slave->Vector[i];
+			if ((str[0] != '[') || (str[1] != '$'))
+			{
+				CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_CODE_ERROR_UNMACRO); // permanent flag
+				CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE);
+				--slave->SaneBehaviour;
+				slave->Proxy.abortBuildTask(this);
+				ok = false;
+				break;
+			}
+		}
+		
+		if (ok) g_DatabaseStatus->updateDatabaseStatus(CCallback<void>(slave, &CSlave::cbUpdateDatabaseStatus), slave->Vector, false, false);
+		
 		slave->Vector.clear();
 	}
 

code/nel/tools/pipeline/service/module_pipeline_slave.cpp

@@ -140,11 +140,18 @@ public:
 		}
 	}
 
-	virtual void startBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, const std::string &projectName, const std::string &processHandler)
+	virtual void startBuildTask(NLNET::IModuleProxy *sender, const std::string &projectName, const uint32 &pluginId)
 	{
+		// TODO
+
 		//this->queueModuleTask
 		CModulePipelineMasterProxy master(sender);
-		master.slaveRefusedBuildTask(this, taskId);
+		master.slaveRefusedBuildTask(this, 999999999); // NO MORE TASK ID
+	}
+
+	virtual void abortBuildTask(NLNET::IModuleProxy *sender)
+	{
+		// TODO
 	}
 
 	virtual void masterUpdatedDatabaseStatus(NLNET::IModuleProxy *sender)

code/nel/tools/pipeline/service/module_pipeline_slave_itf.xml

@@ -7,9 +7,12 @@
 			<method name="startBuildTask" msg="GO_BT">
 				<doc line=""/>
 
-				<param type="uint32"			name="taskId"	/>
 				<param type="std::string"		name="projectName"		byref="true"/>
-				<param type="std::string"		name="processHandler"		byref="true"/>
+				<param type="uint32"		name="pluginId"		byref="true"/>
+			</method>
+
+			<method name="abortBuildTask" msg="ABORT_BT">
+				<doc line=""/>
 			</method>
 
 			<method name="masterUpdatedDatabaseStatus" msg="RE_UPD_DB_ST">

code/nel/tools/pipeline/service/pipeline_process_impl.cpp

@@ -81,7 +81,8 @@ std::string CPipelineProcessImpl::getTempDirectory()
 	{
 		nlwarning("(m_ActiveProject == NULL)");
 		std::stringstream ss;
-		ss << g_PipelineDirectory;
+		ss << g_WorkDir;
+		ss << PIPELINE_DIRECTORY_UNKNOWN_PREFIX;
 		ss << NLMISC::CTime::getSecondsSince1970();
 		ss << ".";
 		ss << rand();

code/nel/tools/pipeline/service/pipeline_project.cpp

@@ -206,7 +206,7 @@ std::string CPipelineProject::getName()
 
 std::string CPipelineProject::getOutputDirectory()
 {
-	return g_PipelineDirectory + getName() + "/";
+	return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_PROJECT + getName() + "/";
 }
 
 std::string CPipelineProject::getTempDirectory()
@@ -214,7 +214,8 @@ std::string CPipelineProject::getTempDirectory()
 	if (m_TempDirectory.empty())
 	{
 		std::stringstream ss;
-		ss << g_PipelineDirectory;
+		ss << g_WorkDir;
+		ss << PIPELINE_DIRECTORY_PREFIX_PROJECT;
 		ss << getName();
 		ss << ".";
 		ss << NLMISC::CTime::getSecondsSince1970();

code/nel/tools/pipeline/service/pipeline_service.cpp

@@ -64,8 +64,7 @@ using namespace NLGEORGES;
 
 namespace PIPELINE {
 
-std::string g_DatabaseDirectory;
+std::string g_WorkDir;
-std::string g_PipelineDirectory;
 bool g_IsExiting = false;
 bool g_IsMaster = false;
 
@@ -74,19 +73,46 @@ CPipelineWorkspace *g_PipelineWorkspace = NULL;
 CDatabaseStatus *g_DatabaseStatus = NULL;
 CPipelineInterfaceImpl *g_PipelineInterfaceImpl = NULL;
 
+#define PIPELINE_MACRO_WORKSPACE_DIRECTORY "[$WorkspaceDirectory]"
+
 std::string unMacroPath(const std::string &path)
 {
 	std::string result = path;
-	strFindReplace(result, PIPELINE_MACRO_DATABASE_DIRECTORY, g_DatabaseDirectory);
+
-	strFindReplace(result, PIPELINE_MACRO_PIPELINE_DIRECTORY, g_PipelineDirectory);
+	strFindReplace(result, PIPELINE_MACRO_WORKSPACE_DIRECTORY, g_WorkDir);
+
+	CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
+	for (uint i = 0; i < rootDirectories.size(); ++i)
+	{
+		std::string rootDirectoryName = rootDirectories.asString(i);
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
+		std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		std::string macroName = "[$" + rootDirectoryName + "]";
+		strFindReplace(result, macroName, rootDirectoryPath);
+	}
+
 	return CPath::standardizePath(result, false);
 }
 
 std::string macroPath(const std::string &path)
 {
 	std::string result = CPath::standardizePath(path, false);
-	strFindReplace(result, g_DatabaseDirectory, PIPELINE_MACRO_DATABASE_DIRECTORY "/");
+
-	strFindReplace(result, g_PipelineDirectory, PIPELINE_MACRO_PIPELINE_DIRECTORY "/");
+	strFindReplace(result, g_WorkDir, PIPELINE_MACRO_WORKSPACE_DIRECTORY "/");
+	
+	CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
+	for (uint i = 0; i < rootDirectories.size(); ++i)
+	{
+		std::string rootDirectoryName = rootDirectories.asString(i);
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
+		std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		std::string macroName = "[$" + rootDirectoryName + "]";
+		strFindReplace(result, rootDirectoryPath,  macroName);
+	}
+
+	if (result[0] != '[' || result[1] != '$')
+		nlerror("Invalid macro path %s", result.c_str());
+
 	return result;
 }
 
@@ -237,14 +263,26 @@ namespace {
 
 void initSheets()
 {
-	std::string leveldesignDfnDirectory = IService::getInstance()->ConfigFile.getVar("LeveldesignDfnDirectory").asString();
+	std::string dfnDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString();
-	std::string leveldesignPipelineDirectory = IService::getInstance()->ConfigFile.getVar("LeveldesignPipelineDirectory").asString();
+	std::string sheetDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString();
 	
-	nlinfo("Adding 'LeveldesignDfnDirectory' to search path (%s)", leveldesignDfnDirectory.c_str());
+	if (!CFile::isDirectory(dfnDirectory)) nlerror("'WorkspaceDfnDirectory' does not exist! (%s)", dfnDirectory.c_str());
-	CPath::addSearchPath(leveldesignDfnDirectory, true, false);
+	nlinfo("Adding 'WorkspaceDfnDirectory' to search path (%s)", dfnDirectory.c_str());
+	CPath::addSearchPath(dfnDirectory, true, false);
 	
-	nlinfo("Adding 'LeveldesignPipelineDirectory' to search path (%s)", leveldesignPipelineDirectory.c_str());
+	if (!CFile::isDirectory(sheetDirectory)) nlerror("'WorkspaceSheetDirectory' does not exist! (%s)", sheetDirectory.c_str());
-	CPath::addSearchPath(leveldesignPipelineDirectory, true, false);
+	nlinfo("Adding 'WorkspaceSheetDirectory' to search path (%s)", sheetDirectory.c_str());
+	CPath::addSearchPath(sheetDirectory, true, false);
+
+	CConfigFile::CVar &georgesDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("GeorgesDirectories");
+	for (uint i = 0; i < georgesDirectories.size(); ++i)
+	{
+		std::string rootName = georgesDirectories.asString(i);
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootName);
+		std::string dirName = CPath::standardizePath(dir.asString(), true);
+		if (!CFile::isDirectory(dirName)) nlerror("'%s' does not exist! (%s)", rootName.c_str(), dirName.c_str());
+		CPath::addSearchPath(dirName, true, false);
+	}
 	
 	g_FormLoader = UFormLoader::createLoader();
 	
@@ -377,12 +415,23 @@ public:
 
 		s_InfoFlags->addFlag("INIT");
 
-		g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
+		//g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
-		if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str());
+		//if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str());
-		ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory);
+		//ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory);
-		g_PipelineDirectory = CPath::standardizePath(ConfigFile.getVar("PipelineDirectory").asString(), true);
+		g_WorkDir = CPath::standardizePath(ConfigFile.getVar("WorkspaceDirectory").asString(), true);
-		if (!CFile::isDirectory(g_PipelineDirectory)) nlwarning("'PipelineDirectory' does not exist! (%s)", g_PipelineDirectory.c_str());
+		if (!CFile::isDirectory(g_WorkDir)) nlerror("'WorkspaceDirectory' does not exist! (%s)", g_WorkDir.c_str());
-		ConfigFile.getVar("PipelineDirectory").setAsString(g_PipelineDirectory);
+		ConfigFile.getVar("WorkspaceDirectory").setAsString(g_WorkDir);
+
+		// Cleanup root directory names
+		CConfigFile::CVar &rootDirectories = ConfigFile.getVar("RootDirectories");
+		for (uint i = 0; i < rootDirectories.size(); ++i)
+		{
+			std::string rootName = rootDirectories.asString(i);
+			CConfigFile::CVar &dir = ConfigFile.getVar(rootName);
+			std::string dirName = CPath::standardizePath(dir.asString(), true);
+			if (!CFile::isDirectory(dirName)) nlerror("'%s' does not exist! (%s)", rootName.c_str(), dirName.c_str());
+			dir.setAsString(dirName);
+		}
 
 		s_TaskManager = new CTaskManager();
 		

code/nel/tools/pipeline/service/pipeline_service.h

@@ -54,12 +54,11 @@ namespace PIPELINE {
 extern bool g_IsExiting;
 extern bool g_IsMaster;
 
-extern std::string g_DatabaseDirectory;
+extern std::string g_WorkDir;
-extern std::string g_PipelineDirectory;
-
-#define PIPELINE_MACRO_DATABASE_DIRECTORY "[$DatabaseDirectory]"
-#define PIPELINE_MACRO_PIPELINE_DIRECTORY "[$PipelineDirectory]"
 
+#define PIPELINE_DIRECTORY_UNKNOWN_PREFIX "unknown."
+#define PIPELINE_DIRECTORY_PREFIX_PROJECT "project."
+#define PIPELINE_DIRECTORY_PREFIX_ROOT "root."
 #define PIPELINE_DIRECTORY_TEMP_SUFFIX ".temp"
 
 /// Unmacros a path, and standardizes it as well.