Ryzom
/
ryzom-core
1
1
Fork 0
Code Issues 38 Pull Requests Projects 1 Releases 1 Activity

Added: #1440 Configuration of multiple custom data directories

Browse Source 
--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 13 years ago
parent e1d4f16939
commit 1837054f5f
12 changed files with 212 additions and 67 deletions
Show Stats Download Patch File Download Diff File

79
code/nel/tools/pipeline/service/database_status.cpp
Unescape Escape View File

@ -49,54 +49,77 @@ namespace PIPELINE {
namespace {
/// Input must be normalized path
bool isInDatabaseDirectoryFast(const std::string &path)
bool isInRootDirectoryFast(std::string &rootDirectoryName, std::string &rootDirectoryPath, const std::string &path)
{
return path.find(g_DatabaseDirectory) == 0;
//return path.find(g_DatabaseDirectory) == 0;
CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
for (uint i = 0; i < rootDirectories.size(); ++i)
{
rootDirectoryName = rootDirectories.asString(i);
CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
if (path.find(rootDirectoryPath) == 0) return true;
}
return false;
}
/// Input must be normalized path
bool isInPipelineDirectoryFast(const std::string &path)
bool isInWorkspaceDirectoryFast(const std::string &path)
{
return path.find(g_PipelineDirectory) == 0;
return path.find(g_WorkDir) == 0;
}
/// Input must be normalized path in database directory
std::string dropDatabaseDirectoryFast(const std::string &path)
std::string dropRootDirectoryFast(const std::string &path, const std::string &rootDirectoryPath)
{
return path.substr(g_DatabaseDirectory.length());
return path.substr(rootDirectoryPath.length());
}
/// Input must be normalized path in pipeline directory
std::string dropPipelineDirectoryFast(const std::string &path)
std::string dropWorkspaceDirectoryFast(const std::string &path)
{
return path.substr(g_PipelineDirectory.length());
return path.substr(g_WorkDir.length());
}
/// Create status file path
std::string getStatusFilePath(const std::string &path)
} /* anonymous namespace */
std::string getMetaFilePath(const std::string &path, const std::string &dotSuffix)
{
std::string stdPath = CPath::standardizePath(path, false);
if (isInDatabaseDirectoryFast(stdPath))
{
std::string relPath = dropDatabaseDirectoryFast(stdPath);
return g_PipelineDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + relPath + PIPELINE_DATABASE_STATUS_SUFFIX;
}
else if (isInPipelineDirectoryFast(stdPath))
if (isInWorkspaceDirectoryFast(stdPath))
{
// TODO_TEST
std::string relPath = dropPipelineDirectoryFast(stdPath);
std::string relPath = dropWorkspaceDirectoryFast(stdPath);
std::string::size_type slashPos = relPath.find_first_of('/');
std::string proProName = relPath.substr(0, slashPos);
std::string subPath = relPath.substr(slashPos);
return g_PipelineDirectory + proProName + PIPELINE_DATABASE_STATUS_SUFFIX + subPath + PIPELINE_DATABASE_STATUS_SUFFIX;
return g_WorkDir + proProName + PIPELINE_DATABASE_META_SUFFIX + subPath + dotSuffix;
}
else
{
nlerror("Path is not in database or pipeline");
return path + PIPELINE_DATABASE_STATUS_SUFFIX;
std::string rootDirectoryName;
std::string rootDirectoryPath;
if (isInRootDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath))
{
std::string relPath = dropRootDirectoryFast(stdPath, rootDirectoryPath);
return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_ROOT + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix;
}
else
{
nlerror("Path is not in database or pipeline (%s)", path.c_str());
return path + dotSuffix;
}
}
}
namespace {
/// Create status file path
std::string getStatusFilePath(const std::string &path)
{
return getMetaFilePath(path, PIPELINE_DATABASE_STATUS_SUFFIX);
}
} /* anonymous namespace */
void CFileError::serial(NLMISC::IStream &stream)
@ -119,8 +142,8 @@ void CFileStatus::serial(NLMISC::IStream &stream)
CDatabaseStatus::CDatabaseStatus()
{
CFile::createDirectoryTree(g_PipelineDirectory + PIPELINE_DATABASE_STATUS_SUBDIR);
CFile::createDirectoryTree(g_PipelineDirectory + PIPELINE_DATABASE_ERRORS_SUBDIR);
//CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR);
//CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_ERRORS_SUBDIR);
}
CDatabaseStatus::~CDatabaseStatus()
@ -186,7 +209,7 @@ public:
bool firstSeen = false;
uint32 time = CTime::getSecondsSince1970();
uint32 fmdt = CFile::getFileModificationDate(FilePath);
std::string statusPath = getStatusFilePath(FilePath); // g_PipelineDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status";
std::string statusPath = getStatusFilePath(FilePath); // g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status";
StatusMutex->lock_shared();
if (CFile::fileExists(statusPath))
{
@ -402,7 +425,15 @@ void CDatabaseStatus::updateDatabaseStatus(const CCallback<void> &callback)
}
std::vector<std::string> paths;
paths.push_back(g_DatabaseDirectory);
//paths.push_back(g_DatabaseDirectory);
CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
for (uint i = 0; i < rootDirectories.size(); ++i)
{
std::string rootDirectoryName = rootDirectories.asString(i);
CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
paths.push_back(rootDirectoryPath);
}
updateDatabaseStatus(callback, paths, false, true);
}

11
code/nel/tools/pipeline/service/database_status.h
Unescape Escape View File

@ -48,13 +48,18 @@ namespace NLMISC {
namespace PIPELINE {
#define PIPELINE_DATABASE_STATUS_SUBDIR "database.status/"
#define PIPELINE_DATABASE_ERRORS_SUBDIR "database.errors/"
#define PIPELINE_DATABASE_DEPEND_SUBDIR "database.depend/"
//#define PIPELINE_DATABASE_STATUS_SUBDIR "database.status/"
//#define PIPELINE_DATABASE_ERRORS_SUBDIR "database.errors/"
//#define PIPELINE_DATABASE_DEPEND_SUBDIR "database.depend/"
#define PIPELINE_DATABASE_STATUS_SUFFIX ".status"
#define PIPELINE_DATABASE_ERRORS_SUFFIX ".errors"
#define PIPELINE_DATABASE_DEPEND_SUFFIX ".depend"
#define PIPELINE_DATABASE_META_SUFFIX ".meta"
std::string getMetaFilePath(const std::string &path, const std::string &dotSuffix);
// Status is generated CRC32 for reference.
// Errors are errors caused by using this file as an input or output file.

39
code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg
Unescape Escape View File

@ -1,14 +1,41 @@
// Intermediate working directory used by the pipeline.
WorkspaceDirectory = "W:/workspace";
// Workspace location
WorkspaceDfnDirectory = "L:/leveldesign/dfn";
WorkspaceSheetDirectory = "L:/leveldesign/pipeline";
WorkspaceSheet = "ryzom_core.pipeline_workspace";
DatabaseDirectory = "W:/database";
PipelineDirectory = "W:/pipeline";
// Lists all directories that can be accessed by the pipeline processes
RootDirectories =
{
"DatabaseDirectory",
"LeveldesignDfnDirectory",
"LeveldesignWorldDirectory",
"LeveldesignEcosystemsDirectory",
"LeveldesignGameElemDirectory",
"LeveldesignGameElementDirectory",
"PrimitivesDirectory",
};
LeveldesignDirectory = "L:/leveldesign";
// Lists all directories that can be accessed by georges
GeorgesDirectories =
{
"LeveldesignDfnDirectory",
"LeveldesignWorldDirectory",
"LeveldesignEcosystemsDirectory",
"LeveldesignGameElemDirectory",
"LeveldesignGameElementDirectory",
};
DatabaseDirectory = "W:/database";
LeveldesignDfnDirectory = "L:/leveldesign/dfn";
LeveldesignPipelineDirectory = "L:/leveldesign/pipeline";
LeveldesignWorldDirectory = "L:/leveldesign/world";
// PrimitivesDirectory = "L:/primitives";
LeveldesignWorldDirectory = "L:/leveldesign/World";
LeveldesignEcosystemsDirectory = "L:/leveldesign/Ecosystems";
LeveldesignGameElemDirectory = "L:/leveldesign/Game_elem";
LeveldesignGameElementDirectory = "L:/leveldesign/game_element";
PrimitivesDirectory = "L:/primitives";
// ToolDirectories = { "R:/build/dev/bin/Release", "D:/libraries/external/bin" };
// ToolSuffix = ".exe";

28
code/nel/tools/pipeline/service/module_pipeline_master.cpp
Unescape Escape View File

@ -50,6 +50,8 @@ namespace PIPELINE {
#define PIPELINE_INFO_MASTER_RELOAD_SHEETS "M_RELOAD_SHEETS"
#define PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE "M_UPD_DB_FOR_S"
#define PIPELINE_INFO_CODE_ERROR_UNMACRO "CODE_ERROR_UNMACRO"
/**
* \brief CModulePipelineMaster
* \date 2012-03-03 16:26GMT
@ -67,13 +69,15 @@ class CModulePipelineMaster :
: Master(master),
Proxy(moduleProxy),
ActiveTaskId(0),
SheetsOk(true) { }
SheetsOk(true),
SaneBehaviour(3) { }
CModulePipelineMaster *Master;
CModulePipelineSlaveProxy Proxy;
std::vector<std::string> Vector;
uint32 ActiveTaskId;
bool SheetsOk;
std::vector<uint32> PluginsAvailable;
sint SaneBehaviour;
~CSlave()
{
@ -92,7 +96,7 @@ class CModulePipelineMaster :
bool canAcceptTask()
{
return SheetsOk && (ActiveTaskId == 0);
return SheetsOk && (ActiveTaskId == 0) && SaneBehaviour > 0;
}
};
@ -201,7 +205,25 @@ public:
{
CSlave *slave = m_Slaves[sender];
CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE);
g_DatabaseStatus->updateDatabaseStatus(CCallback<void>(slave, &CSlave::cbUpdateDatabaseStatus), slave->Vector, false, false);
bool ok = true;
for (std::vector<std::string>::size_type i = 0; i < slave->Vector.size(); ++i)
{
// not real security, just an extra for catching coding errors
std::string &str = slave->Vector[i];
if ((str[0] != '[') || (str[1] != '$'))
{
CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_CODE_ERROR_UNMACRO); // permanent flag
CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE);
--slave->SaneBehaviour;
slave->Proxy.abortBuildTask(this);
ok = false;
break;
}
}
if (ok) g_DatabaseStatus->updateDatabaseStatus(CCallback<void>(slave, &CSlave::cbUpdateDatabaseStatus), slave->Vector, false, false);
slave->Vector.clear();
}

11
code/nel/tools/pipeline/service/module_pipeline_slave.cpp
Unescape Escape View File

@ -140,11 +140,18 @@ public:
}
}
virtual void startBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, const std::string &projectName, const std::string &processHandler)
virtual void startBuildTask(NLNET::IModuleProxy *sender, const std::string &projectName, const uint32 &pluginId)
{
// TODO
//this->queueModuleTask
CModulePipelineMasterProxy master(sender);
master.slaveRefusedBuildTask(this, taskId);
master.slaveRefusedBuildTask(this, 999999999); // NO MORE TASK ID
}
virtual void abortBuildTask(NLNET::IModuleProxy *sender)
{
// TODO
}
virtual void masterUpdatedDatabaseStatus(NLNET::IModuleProxy *sender)

BIN
code/nel/tools/pipeline/service/module_pipeline_slave_itf.cpp
View File

Binary file not shown.

BIN
code/nel/tools/pipeline/service/module_pipeline_slave_itf.h
View File

Binary file not shown.

7
code/nel/tools/pipeline/service/module_pipeline_slave_itf.xml
Unescape Escape View File

@ -7,9 +7,12 @@
<method name="startBuildTask" msg="GO_BT">
<doc line=""/>
<param type="uint32" name="taskId" />
<param type="std::string" name="projectName" byref="true"/>
<param type="std::string" name="processHandler" byref="true"/>
<param type="uint32" name="pluginId" byref="true"/>
</method>
<method name="abortBuildTask" msg="ABORT_BT">
<doc line=""/>
</method>
<method name="masterUpdatedDatabaseStatus" msg="RE_UPD_DB_ST">

3
code/nel/tools/pipeline/service/pipeline_process_impl.cpp
Unescape Escape View File

@ -81,7 +81,8 @@ std::string CPipelineProcessImpl::getTempDirectory()
{
nlwarning("(m_ActiveProject == NULL)");
std::stringstream ss;
ss << g_PipelineDirectory;
ss << g_WorkDir;
ss << PIPELINE_DIRECTORY_UNKNOWN_PREFIX;
ss << NLMISC::CTime::getSecondsSince1970();
ss << ".";
ss << rand();

5
code/nel/tools/pipeline/service/pipeline_project.cpp
Unescape Escape View File

@ -206,7 +206,7 @@ std::string CPipelineProject::getName()
std::string CPipelineProject::getOutputDirectory()
{
return g_PipelineDirectory + getName() + "/";
return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_PROJECT + getName() + "/";
}
std::string CPipelineProject::getTempDirectory()
@ -214,7 +214,8 @@ std::string CPipelineProject::getTempDirectory()
if (m_TempDirectory.empty())
{
std::stringstream ss;
ss << g_PipelineDirectory;
ss << g_WorkDir;
ss << PIPELINE_DIRECTORY_PREFIX_PROJECT;
ss << getName();
ss << ".";
ss << NLMISC::CTime::getSecondsSince1970();

85
code/nel/tools/pipeline/service/pipeline_service.cpp
Unescape Escape View File

@ -64,8 +64,7 @@ using namespace NLGEORGES;
namespace PIPELINE {
std::string g_DatabaseDirectory;
std::string g_PipelineDirectory;
std::string g_WorkDir;
bool g_IsExiting = false;
bool g_IsMaster = false;
@ -74,19 +73,46 @@ CPipelineWorkspace *g_PipelineWorkspace = NULL;
CDatabaseStatus *g_DatabaseStatus = NULL;
CPipelineInterfaceImpl *g_PipelineInterfaceImpl = NULL;
#define PIPELINE_MACRO_WORKSPACE_DIRECTORY "[$WorkspaceDirectory]"
std::string unMacroPath(const std::string &path)
{
std::string result = path;
strFindReplace(result, PIPELINE_MACRO_DATABASE_DIRECTORY, g_DatabaseDirectory);
strFindReplace(result, PIPELINE_MACRO_PIPELINE_DIRECTORY, g_PipelineDirectory);
strFindReplace(result, PIPELINE_MACRO_WORKSPACE_DIRECTORY, g_WorkDir);
CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
for (uint i = 0; i < rootDirectories.size(); ++i)
{
std::string rootDirectoryName = rootDirectories.asString(i);
CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
std::string macroName = "[$" + rootDirectoryName + "]";
strFindReplace(result, macroName, rootDirectoryPath);
}
return CPath::standardizePath(result, false);
}
std::string macroPath(const std::string &path)
{
std::string result = CPath::standardizePath(path, false);
strFindReplace(result, g_DatabaseDirectory, PIPELINE_MACRO_DATABASE_DIRECTORY "/");
strFindReplace(result, g_PipelineDirectory, PIPELINE_MACRO_PIPELINE_DIRECTORY "/");
strFindReplace(result, g_WorkDir, PIPELINE_MACRO_WORKSPACE_DIRECTORY "/");
CConfigFile::CVar &rootDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("RootDirectories");
for (uint i = 0; i < rootDirectories.size(); ++i)
{
std::string rootDirectoryName = rootDirectories.asString(i);
CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
std::string macroName = "[$" + rootDirectoryName + "]";
strFindReplace(result, rootDirectoryPath, macroName);
}
if (result[0] != '[' || result[1] != '$')
nlerror("Invalid macro path %s", result.c_str());
return result;
}
@ -237,14 +263,26 @@ namespace {
void initSheets()
{
std::string leveldesignDfnDirectory = IService::getInstance()->ConfigFile.getVar("LeveldesignDfnDirectory").asString();
std::string leveldesignPipelineDirectory = IService::getInstance()->ConfigFile.getVar("LeveldesignPipelineDirectory").asString();
std::string dfnDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString();
std::string sheetDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString();
nlinfo("Adding 'LeveldesignDfnDirectory' to search path (%s)", leveldesignDfnDirectory.c_str());
CPath::addSearchPath(leveldesignDfnDirectory, true, false);
if (!CFile::isDirectory(dfnDirectory)) nlerror("'WorkspaceDfnDirectory' does not exist! (%s)", dfnDirectory.c_str());
nlinfo("Adding 'WorkspaceDfnDirectory' to search path (%s)", dfnDirectory.c_str());
CPath::addSearchPath(dfnDirectory, true, false);
nlinfo("Adding 'LeveldesignPipelineDirectory' to search path (%s)", leveldesignPipelineDirectory.c_str());
CPath::addSearchPath(leveldesignPipelineDirectory, true, false);
if (!CFile::isDirectory(sheetDirectory)) nlerror("'WorkspaceSheetDirectory' does not exist! (%s)", sheetDirectory.c_str());
nlinfo("Adding 'WorkspaceSheetDirectory' to search path (%s)", sheetDirectory.c_str());
CPath::addSearchPath(sheetDirectory, true, false);
CConfigFile::CVar &georgesDirectories = NLNET::IService::getInstance()->ConfigFile.getVar("GeorgesDirectories");
for (uint i = 0; i < georgesDirectories.size(); ++i)
{
std::string rootName = georgesDirectories.asString(i);
CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootName);
std::string dirName = CPath::standardizePath(dir.asString(), true);
if (!CFile::isDirectory(dirName)) nlerror("'%s' does not exist! (%s)", rootName.c_str(), dirName.c_str());
CPath::addSearchPath(dirName, true, false);
}
g_FormLoader = UFormLoader::createLoader();
@ -377,12 +415,23 @@ public:
s_InfoFlags->addFlag("INIT");
g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str());
ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory);
g_PipelineDirectory = CPath::standardizePath(ConfigFile.getVar("PipelineDirectory").asString(), true);
if (!CFile::isDirectory(g_PipelineDirectory)) nlwarning("'PipelineDirectory' does not exist! (%s)", g_PipelineDirectory.c_str());
ConfigFile.getVar("PipelineDirectory").setAsString(g_PipelineDirectory);
//g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
//if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str());
//ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory);
g_WorkDir = CPath::standardizePath(ConfigFile.getVar("WorkspaceDirectory").asString(), true);
if (!CFile::isDirectory(g_WorkDir)) nlerror("'WorkspaceDirectory' does not exist! (%s)", g_WorkDir.c_str());
ConfigFile.getVar("WorkspaceDirectory").setAsString(g_WorkDir);
// Cleanup root directory names
CConfigFile::CVar &rootDirectories = ConfigFile.getVar("RootDirectories");
for (uint i = 0; i < rootDirectories.size(); ++i)
{
std::string rootName = rootDirectories.asString(i);
CConfigFile::CVar &dir = ConfigFile.getVar(rootName);
std::string dirName = CPath::standardizePath(dir.asString(), true);
if (!CFile::isDirectory(dirName)) nlerror("'%s' does not exist! (%s)", rootName.c_str(), dirName.c_str());
dir.setAsString(dirName);
}
s_TaskManager = new CTaskManager();

9
code/nel/tools/pipeline/service/pipeline_service.h
Unescape Escape View File

@ -54,12 +54,11 @@ namespace PIPELINE {
extern bool g_IsExiting;
extern bool g_IsMaster;
extern std::string g_DatabaseDirectory;
extern std::string g_PipelineDirectory;
#define PIPELINE_MACRO_DATABASE_DIRECTORY "[$DatabaseDirectory]"
#define PIPELINE_MACRO_PIPELINE_DIRECTORY "[$PipelineDirectory]"
extern std::string g_WorkDir;
#define PIPELINE_DIRECTORY_UNKNOWN_PREFIX "unknown."
#define PIPELINE_DIRECTORY_PREFIX_PROJECT "project."
#define PIPELINE_DIRECTORY_PREFIX_ROOT "root."
#define PIPELINE_DIRECTORY_TEMP_SUFFIX ".temp"
/// Unmacros a path, and standardizes it as well.

Write Preview
Loading…
Cancel
Save