Ryzom
/
ryzom-core
1
1
Fork 0
Code Issues 38 Pull Requests Projects 1 Releases 1 Activity

Added: #1440 Send tool errors from slave to master

Browse Source 
--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 12 years ago
parent 5e168b210a
commit 7292685545
8 changed files with 221 additions and 25 deletions
Show Stats Download Patch File Download Diff File

3
code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg
Unescape Escape View File

@ -54,6 +54,9 @@ ToolBnpMake = "bnp_make";
// MasterIgnoreProcessPlugins = { }; // Only used by the master service. NOT USED, USE WORKSPACE INTEAD!
// MasterTerminalPassword = "MASTERCLIENTPASSWORD"; // Only used by the master service.
// Log containing the stuff that would be sent from the master to the user terminal
MasterLog = SharedWork + "/master.log";
// MasterSlavePassword = "MASTERSLAVEPASSWORD";
MasterAddress = "localhost";
MasterPort = 50123;

88
code/nel/tools/pipeline/service/module_pipeline_master.cpp
Unescape Escape View File

@ -36,6 +36,7 @@
#include <nel/misc/debug.h>
#include <nel/net/service.h>
#include <nel/misc/task_manager.h>
#include <nel/misc/tool_logger.h>
// Project includes
#include "info_flags.h"
@ -157,7 +158,7 @@ protected:
bool m_VerifyOnly;
public:
CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks")
CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks"), m_MasterLog(NULL)
{
g_IsMaster = true;
m_TaskManager = new NLMISC::CTaskManager();
@ -397,6 +398,7 @@ public:
CProcessPluginInfo pluginInfo;
g_PipelineWorkspace->getProcessPlugin(pluginInfo, taskInfo->ProcessPluginId);
nlinfo("Dispatching task '%i' ('%s': '%s') to slave '%s'", taskInfo->Id.Global, taskInfo->ProjectName.c_str(), pluginInfo.Handler.c_str(), it->second->Proxy.getModuleProxy()->getModuleName().c_str());
notifyTerminalTaskBegin(taskInfo->Id.Global);
}
}
}
@ -431,6 +433,8 @@ public:
PIPELINE::endedBuildReadyMaster();
notifyTerminalBuildEnd();
nlinfo("#####################################");
nlinfo("#####################################");
nlinfo("! DONE !");
@ -503,6 +507,15 @@ public:
CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_REJECTED);
}
virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
{
TSlaveMap::iterator slaveIt = m_Slaves.find(sender);
if (slaveIt == m_Slaves.end()) { nlerror("Received 'slaveLoggedToolError' from unknown slave at '%s'", sender->getModuleName().c_str()); m_Slaves.erase(sender); /*m_SlavesMutex.unlock();*/ return; }
CSlave *slave = slaveIt->second;
notifyTerminalTaskMessage(slave->ActiveTaskId, (TError)type, macroPath, time, error);
}
virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender)
{
//m_SlavesMutex.lock();
@ -551,10 +564,68 @@ public:
///////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////
FILE *m_MasterLog;
void notifyTerminalBuildStart()
{
nlinfo("");
// TODO NOTIFY TERMINAL
if (m_MasterLog)
{
fflush(m_MasterLog);
fclose(m_MasterLog);
m_MasterLog = NULL;
}
m_MasterLog = fopen(IService::getInstance()->ConfigFile.getVar("MasterLog").asString(0).c_str(), "at");
fprintf(m_MasterLog, "[BUILD_START]\n");
fflush(m_MasterLog);
}
// task info
void notifyTerminalTaskInfo(uint32 taskId, const std::string &projectName, const std::string &processHandler)
{
nlinfo("taskId: %i, projectName: %s, processHandler: %s", taskId, projectName.c_str(), processHandler.c_str());
// TODO NOTIFY TERMINAL
fprintf(m_MasterLog, "[TASK_INFO] taskId: %i, projectName: %s, processHandler: %s\n", taskId, projectName.c_str(), processHandler.c_str());
fflush(m_MasterLog);
}
// task begin
void notifyTerminalTaskBegin(uint32 taskId)
{
nlinfo("taskId: %i", taskId);
// TODO NOTIFY TERMINAL
fprintf(m_MasterLog, "[TASK_BEGIN] taskId: %i\n", taskId);
fflush(m_MasterLog);
}
// task issues
void notifyTerminalTaskMessage(uint32 taskId, TError type, const std::string &macroPath, const std::string &time, const std::string &error)
{
nlinfo("taskId: %i, type: %i, macroPath: %s, time: %s, error: %s", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str());
// TODO NOTIFY TERMINAL (send type as uint8 as usual)
fprintf(m_MasterLog, "[TASK_MESSAGE] taskId: %i, type: %i, macroPath: %s, time: %s, error: %s\n", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str());
fflush(m_MasterLog);
}
// task end
void notifyTerminalTaskState(uint32 taskId, TProcessResult errorLevel, const std::string &errorMessage)
{
nlinfo("taskId: %i, errorLevel: %i, errorMessage: %s", taskId, (uint32)errorLevel, errorMessage.c_str());
// TODO NOTIFY TERMINAL (send errorlevel as uint8 as usual)
fprintf(m_MasterLog, "[TASK_STATE] taskId: %i, errorLevel: %i, errorMessage: %s\n", taskId, (uint32)errorLevel, errorMessage.c_str());
fflush(m_MasterLog);
}
// build end (any remaining tasks are considered aborted)
void notifyTerminalBuildEnd()
{
nlinfo("");
// TODO NOTIFY TERMINAL
fprintf(m_MasterLog, "[BUILD_END]\n");
fflush(m_MasterLog);
fclose(m_MasterLog);
m_MasterLog = NULL;
}
///////////////////////////////////////////////////////////////////
@ -711,11 +782,26 @@ public:
{
if (PIPELINE::tryBuildReadyMaster())
{
notifyTerminalBuildStart();
m_BuildWorking = true;
m_BypassErrors = bypassEros;
m_VerifyOnly = verifyOnly;
m_BuildTaskQueue.resetQueue();
m_BuildTaskQueue.loadQueue(g_PipelineWorkspace, bypassEros);
// notify terminal task info
{
std::vector<PIPELINE::CBuildTaskInfo *> tasks;
m_BuildTaskQueue.listTaskQueueByMostDependents(tasks);
for (std::vector<PIPELINE::CBuildTaskInfo *>::iterator it = tasks.begin(), end = tasks.end(); it != end; ++it)
{
PIPELINE::CBuildTaskInfo *task = *it;
PIPELINE::CProcessPluginInfo pluginInfo;
PIPELINE::g_PipelineWorkspace->getProcessPlugin(pluginInfo, task->ProcessPluginId);
notifyTerminalTaskInfo(task->Id.Global, task->ProjectName, pluginInfo.Handler);
}
}
return true;
}
return false;

49
code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp
Unescape Escape View File

@ -34,6 +34,10 @@ namespace PIPELINE
// if this assert, you have a doubly message name in your interface definition !
nlassert(res.second);
res = handlers.insert(std::make_pair(std::string("TL_ERR_LOG"), &CModulePipelineMasterSkel::slaveLoggedToolError_skel));
// if this assert, you have a doubly message name in your interface definition !
nlassert(res.second);
res = handlers.insert(std::make_pair(std::string("RE_SHEETS_OK"), &CModulePipelineMasterSkel::slaveReloadedSheets_skel));
// if this assert, you have a doubly message name in your interface definition !
nlassert(res.second);
@ -103,6 +107,20 @@ namespace PIPELINE
slaveRefusedBuildTask(sender);
}
void CModulePipelineMasterSkel::slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message)
{
H_AUTO(CModulePipelineMasterSkel_slaveLoggedToolError_TL_ERR_LOG);
uint8 type;
nlRead(__message, serial, type);
std::string macroPath;
nlRead(__message, serial, macroPath);
std::string time;
nlRead(__message, serial, time);
std::string error;
nlRead(__message, serial, error);
slaveLoggedToolError(sender, type, macroPath, time, error);
}
void CModulePipelineMasterSkel::slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message)
{
H_AUTO(CModulePipelineMasterSkel_slaveReloadedSheets_RE_SHEETS_OK);
@ -197,6 +215,24 @@ namespace PIPELINE
}
}
//
void CModulePipelineMasterProxy::slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
{
if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported())
{
// immediate local synchronous dispatching
_LocalModuleSkel->slaveLoggedToolError(_ModuleProxy->getModuleGateway()->getPluggedModuleProxy(sender), type, macroPath, time, error);
}
else
{
// send the message for remote dispatching and execution or local queing
NLNET::CMessage __message;
buildMessageFor_slaveLoggedToolError(__message, type, macroPath, time, error);
_ModuleProxy->sendModuleMessage(sender, __message);
}
}
//
void CModulePipelineMasterProxy::slaveReloadedSheets(NLNET::IModule *sender)
{
if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported())
@ -334,6 +370,19 @@ namespace PIPELINE
return __message;
}
// Message serializer. Return the message received in reference for easier integration
const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
{
__message.setType("TL_ERR_LOG");
nlWrite(__message, serial, type);
nlWrite(__message, serial, const_cast < std::string& > (macroPath));
nlWrite(__message, serial, const_cast < std::string& > (time));
nlWrite(__message, serial, const_cast < std::string& > (error));
return __message;
}
// Message serializer. Return the message received in reference for easier integration
const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message)
{

9
code/nel/tools/pipeline/service/module_pipeline_master_itf.h
Unescape Escape View File

@ -66,6 +66,8 @@ namespace PIPELINE
void slaveRefusedBuildTask_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
void slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
void slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
void slaveBuildReadySuccess_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
@ -95,6 +97,8 @@ namespace PIPELINE
//
virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender) =0;
//
virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error) =0;
//
virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender) =0;
//
virtual void slaveBuildReadySuccess(NLNET::IModuleProxy *sender) =0;
@ -162,6 +166,8 @@ namespace PIPELINE
//
void slaveRefusedBuildTask(NLNET::IModule *sender);
//
void slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error);
//
void slaveReloadedSheets(NLNET::IModule *sender);
//
void slaveBuildReadySuccess(NLNET::IModule *sender);
@ -183,6 +189,9 @@ namespace PIPELINE
// Message serializer. Return the message received in reference for easier integration
static const NLNET::CMessage &buildMessageFor_slaveRefusedBuildTask(NLNET::CMessage &__message);
// Message serializer. Return the message received in reference for easier integration
static const NLNET::CMessage &buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error);
// Message serializer. Return the message received in reference for easier integration
static const NLNET::CMessage &buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message);

9
code/nel/tools/pipeline/service/module_pipeline_master_itf.xml
Unescape Escape View File

@ -19,6 +19,15 @@
<doc line=""/>
</method>
<method name="slaveLoggedToolError" msg="TL_ERR_LOG">
<doc line=""/>
<param type="uint8" name="type" />
<param type="std::string" name="macroPath" byref="true" />
<param type="std::string" name="time" byref="true" />
<param type="std::string" name="error" byref="true" />
</method>
<method name="slaveReloadedSheets" msg="RE_SHEETS_OK">
<doc line=""/>
</method>

48
code/nel/tools/pipeline/service/module_pipeline_slave.cpp
Unescape Escape View File

@ -114,8 +114,18 @@ public:
bool m_PluginBuildDone;
struct CErrorLogData
{
TError Type;
std::string Path;
std::string Time;
std::string Error;
};
NLMISC::CSynchronized<std::vector<CErrorLogData> > m_ErrorLogWaiting;
public:
CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false)
CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false), m_ErrorLogWaiting("ErrorLogWaiting")
{
NLMISC::CSynchronized<bool>::CAccessor(&m_StatusUpdateMasterDone).value() = false;
NLMISC::CSynchronized<bool>::CAccessor(&m_StatusUpdateSlaveDone).value() = false;
@ -215,13 +225,35 @@ public:
abortBuildTask(NULL);
leaveBuildReadyState(NULL); // leave state if building
clearErrorLogQueue();
delete m_Master;
m_Master = NULL;
}
}
void clearErrorLogQueue()
{
NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
errorLogQueue.value().clear();
}
void sendErrorLogQueue()
{
NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
for (std::vector<CErrorLogData>::iterator it = errorLogQueue.value().begin(), end = errorLogQueue.value().end(); it != end; ++it)
{
const CErrorLogData &erdt = *it;
nlassert(m_Master);
m_Master->slaveLoggedToolError(this, (uint8)erdt.Type, macroPath(erdt.Path), erdt.Time, erdt.Error);
}
errorLogQueue.value().clear();
}
virtual void onModuleUpdate()
{
sendErrorLogQueue();
if (m_ReloadSheetsState == REQUEST_MADE)
{
if (PIPELINE::reloadSheets())
@ -313,6 +345,7 @@ public:
{
m_SlaveTaskState = SOMEWHERE_INBETWEEN;
CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_PLUGIN_WORKING);
sendErrorLogQueue();
if (m_AbortRequested)
{
nlinfo("Aborted slave task while plugin was working");
@ -525,6 +558,18 @@ public:
///////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////
void notifyErrorLog(TError type, const std::string &path, const std::string &time, const std::string &error)
{
// Received an error log from a plugin tool
NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
CErrorLogData erdt;
erdt.Type = type;
erdt.Path = path;
erdt.Time = time;
erdt.Error = error;
errorLogQueue.value().push_back(erdt);
}
class CPluginBuildTask : public IRunnable
{
public:
@ -537,6 +582,7 @@ public:
case PIPELINE::PLUGIN_REGISTERED_CLASS:
{
PIPELINE::IProcessHandler *processHandler = static_cast<PIPELINE::IProcessHandler *>(NLMISC::CClassRegistry::create(m_Slave->m_ActiveProcess->m_ActivePlugin.Handler));
m_Slave->m_ActiveProcess->m_ErrorLogCallback = TErrorLogCallback(m_Slave, &CModulePipelineSlave::notifyErrorLog);
processHandler->setPipelineProcess(m_Slave->m_ActiveProcess);
m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_SUCCESS;
processHandler->build();

5
code/nel/tools/pipeline/service/pipeline_process_impl.h
Unescape Escape View File

@ -32,15 +32,19 @@
// STL includes
// NeL includes
#include <nel/misc/tool_logger.h>
// Project includes
#include "../plugin_library/pipeline_process.h"
#include "pipeline_workspace.h"
#include "metadata_storage.h"
#include "callback.h"
namespace PIPELINE {
class CPipelineProject;
typedef CCallback<void, TError /* type */, const std::string & /* path */, const std::string & /* time */ , const std::string & /* error */ > TErrorLogCallback;
/**
* \brief CPipelineProcessImpl
* \date 2012-03-03 09:33GMT
@ -108,6 +112,7 @@ private:
std::set<std::string> m_ListDependentFiles;
bool m_Aborting;
TErrorLogCallback m_ErrorLogCallback;
private:
bool getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath);

35
code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace
Unescape Escape View File

@ -3,17 +3,7 @@
<STRUCT>
<ATOM Name="Description" Value="Ryzom Core"/>
<ARRAY Name="Projects">
<ATOM Value="gamedev.pipeline_package"/>
<ATOM Value="data_common.pipeline_package"/>
<ATOM Value="common_interface.pipeline_project"/>
<ATOM Value="interfaces.pipeline_package"/>
<ATOM Value="common_objects_generique.pipeline_project"/>
<ATOM Value="common_objects_fyros.pipeline_project"/>
<ATOM Value="common_objects_tryker.pipeline_project"/>
<ATOM Value="common_objects_matis.pipeline_project"/>
<ATOM Value="common_objects_zorai.pipeline_project"/>
<ATOM Value="common_objects_caravan.pipeline_project"/>
<ATOM Value="objects.pipeline_package"/>
</ARRAY>
<ARRAY Name="Plugins">
<ATOM Value="plugin_nel.pipeline_plugin"/>
@ -24,17 +14,16 @@
<STRUCT/>
<STRUCT/>
<STRUCT/>
<LOG>Sat Feb 18 13:48:48 2012 (Kaetemi) .Name = Ryzom Core
Sat Feb 18 13:48:48 2012 (Kaetemi) formName Resized = 1
Sat Feb 18 14:44:25 2012 (Kaetemi) .Description = Ryzom Core
Sat Feb 18 14:44:37 2012 (Kaetemi) .Projects[0] = common_interface.pipeline_project
Sat Feb 18 23:23:17 2012 (Kaetemi) .Description = Ryzom Core Test
Sat Feb 18 23:23:26 2012 (Kaetemi) .Description = Ryzom Core
Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[0] = plugin_nel.pipeline_plugin
Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[1] = plugin_max.pipeline_plugin
Sat Mar 03 10:58:02 2012 (Kaetemi) formName Resized = 2
Sat Aug 04 21:04:25 2012 (kaetemi) .Projects[1] = interfaces.pipeline_package
Sat Aug 04 21:04:25 2012 (kaetemi) formName Resized = 2
Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[0] = interfaces.pipeline_package
Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[1] = common_interface.pipeline_project</LOG>
<COMMENTS> &lt;ATOM Value="gamedev.pipeline_package"/&gt;
&lt;ATOM Value="data_common.pipeline_package"/&gt;
&lt;ATOM Value="common_interface.pipeline_project"/&gt;
&lt;ATOM Value="interfaces.pipeline_package"/&gt;
&lt;ATOM Value="common_objects_generique.pipeline_project"/&gt;
&lt;ATOM Value="common_objects_fyros.pipeline_project"/&gt;
&lt;ATOM Value="common_objects_tryker.pipeline_project"/&gt;
&lt;ATOM Value="common_objects_matis.pipeline_project"/&gt;
&lt;ATOM Value="common_objects_zorai.pipeline_project"/&gt;
&lt;ATOM Value="common_objects_caravan.pipeline_project"/&gt;
&lt;ATOM Value="objects.pipeline_package"/&gt;</COMMENTS>
<LOG></LOG>
</FORM>

Write Preview
Loading…
Cancel
Save