Added: #1440 Build and abort commands

--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 13 years ago
parent eb7a1b8ce5
commit 64723b3234

@ -227,17 +227,18 @@ uint CBuildTaskQueue::countWorkingTasks()
return nb;
}
uint CBuildTaskQueue::countRemainingBuildableTasksAndWorkingTasks()
void CBuildTaskQueue::countRemainingBuildableTasksAndWorkingTasks(uint &buildable, uint &working)
{
m_Mutex.lock();
std::vector<CBuildTaskInfo *> availableTasks;
createBuildableTaskList(availableTasks, m_BypassDependencyError);
uint nb = availableTasks.size();
buildable = availableTasks.size();
uint nb = 0;
for (std::vector<CBuildTaskInfo *>::iterator it = m_Tasks.begin(), end = m_Tasks.end(); it != end; ++it)
if ((*it)->State == TASK_WORKING)
++nb;
working = nb;
m_Mutex.unlock();
return nb;
}
void CBuildTaskQueue::listTaskQueueByMostDependents(std::vector<CBuildTaskInfo *> &result)

@ -113,7 +113,7 @@ public:
uint countRemainingBuildableTasks();
uint countWorkingTasks();
// when next are 0 the build should stop
uint countRemainingBuildableTasksAndWorkingTasks();
void countRemainingBuildableTasksAndWorkingTasks(uint &buildable, uint &working);
// informational listing for sending initial task listing to terminals
void listTaskQueueByMostDependents(std::vector<CBuildTaskInfo *> &result);

@ -40,6 +40,7 @@
#include "module_pipeline_slave_itf.h"
#include "pipeline_service.h"
#include "database_status.h"
#include "build_task_queue.h"
using namespace std;
using namespace NLMISC;
@ -47,10 +48,14 @@ using namespace NLNET;
namespace PIPELINE {
// temporary flags
#define PIPELINE_INFO_MASTER_RELOAD_SHEETS "M_RELOAD_SHEETS"
#define PIPELINE_INFO_MASTER_UPDATE_DATABASE_FOR_SLAVE "M_UPD_DB_FOR_S"
// permanent flags
#define PIPELINE_INFO_CODE_ERROR_UNMACRO "CODE_ERROR_UNMACRO"
#define PIPELINE_INFO_SLAVE_REJECTED "SLAVE_REJECT"
#define PIPELINE_INFO_SLAVE_CRASHED "SLAVE_CRASH"
/**
* \brief CModulePipelineMaster
@ -70,7 +75,8 @@ class CModulePipelineMaster :
Proxy(moduleProxy),
ActiveTaskId(0),
SheetsOk(true),
SaneBehaviour(3) { }
SaneBehaviour(3),
BuildReadyState(0) { }
CModulePipelineMaster *Master;
CModulePipelineSlaveProxy Proxy;
std::vector<std::string> Vector;
@ -78,6 +84,7 @@ class CModulePipelineMaster :
bool SheetsOk;
std::vector<uint32> PluginsAvailable;
sint SaneBehaviour;
uint BuildReadyState;
~CSlave()
{
@ -96,7 +103,7 @@ class CModulePipelineMaster :
bool canAcceptTask()
{
return SheetsOk && (ActiveTaskId == 0) && SaneBehaviour > 0;
return SheetsOk && (ActiveTaskId == 0) && SaneBehaviour > 0 && BuildReadyState == 2;
}
};
@ -104,9 +111,15 @@ protected:
typedef std::map<IModuleProxy *, CSlave *> TSlaveMap;
TSlaveMap m_Slaves;
mutable boost::mutex m_SlavesMutex;
CBuildTaskQueue m_BuildTaskQueue;
bool m_BuildWorking;
// build command
bool m_BypassErrors;
bool m_VerifyOnly;
public:
CModulePipelineMaster()
CModulePipelineMaster() : m_BuildWorking(false)
{
g_IsMaster = true;
}
@ -163,7 +176,9 @@ public:
if (slave->ActiveTaskId)
{
// ...
// if it goes down while busy on a task it crashed (or was poorly stopped by user...)
CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_CRASHED);
// ... TODO ...
}
m_Slaves.erase(slaveIt);
@ -176,16 +191,83 @@ public:
virtual void onModuleUpdate()
{
// if state build, iterate trough all slaves to see if any is free, and check if there's any waiting tasks
if (m_BuildWorking)
{
m_SlavesMutex.lock();
// iterate trough all slaves to tell them the enter build_ready state.
for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
{
if (it->second->BuildReadyState == 0 && it->second->SaneBehaviour > 0 && it->second->SheetsOk)
{
it->second->BuildReadyState = 1;
it->second->Proxy.enterBuildReadyState(this);
}
// wait for confirmation, set BuildReadyState = 2 in that callback!
}
m_SlavesMutex.unlock();
virtual void slaveFinishedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, uint8 errorLevel)
uint nbBuildable, nbWorking;
m_BuildTaskQueue.countRemainingBuildableTasksAndWorkingTasks(nbBuildable, nbWorking);
if (nbBuildable > 0)
{
m_SlavesMutex.lock();
// Iterate trough all slaves to see if any is free for a building task.
for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
{
if (it->second->canAcceptTask())
{
CBuildTaskInfo *taskInfo = m_BuildTaskQueue.getTaskForSlave(it->second->PluginsAvailable);
if (taskInfo != NULL)
{
it->second->ActiveTaskId = taskInfo->Id.Global;
it->second->Proxy.startBuildTask(this, taskInfo->ProjectName, taskInfo->ProcessPluginId);
// the slave may either reject; or not answer until it's finished with this task
}
}
}
m_SlavesMutex.unlock();
}
else if (nbWorking == 0)
{
// done (or stuck)
m_BuildWorking = false;
m_SlavesMutex.lock();
// Iterate trough all slaves to tell them to end build_ready state.
for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
{
it->second->BuildReadyState = 0;
it->second->Proxy.leaveBuildReadyState(this);
nlassert(it->second->ActiveTaskId == 0);
}
m_SlavesMutex.unlock();
PIPELINE::endedBuildReady();
}
}
}
virtual void slaveFinishedBuildTask(NLNET::IModuleProxy *sender, uint8 errorLevel)
{
// TODO
}
virtual void slaveAbortedBuildTask(NLNET::IModuleProxy *sender)
{
// TODO
}
virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId)
// in fact slaves are not allowed to refuse tasks, but they may do this if the user is toying around with the slave service
virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender)
{
// TODO
CSlave *slave = m_Slaves[sender];
m_BuildTaskQueue.rejectedTask(slave->ActiveTaskId);
slave->ActiveTaskId = 0;
--slave->SaneBehaviour;
CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_REJECTED);
// TODO
}
virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender)
@ -195,6 +277,20 @@ public:
CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS);
}
virtual void slaveBuildReadySuccess(NLNET::IModuleProxy *sender)
{
CSlave *slave = m_Slaves[sender];
slave->BuildReadyState = 2;
}
virtual void slaveBuildReadyFail(NLNET::IModuleProxy *sender)
{
CSlave *slave = m_Slaves[sender];
slave->BuildReadyState = 0;
--slave->SaneBehaviour;
CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_REJECTED);
}
virtual void vectorPushString(NLNET::IModuleProxy *sender, const std::string &str)
{
CSlave *slave = m_Slaves[sender];
@ -233,9 +329,41 @@ public:
slave->PluginsAvailable = pluginsAvailable;
}
bool build(bool bypassEros, bool verifyOnly)
{
if (PIPELINE::tryBuildReady())
{
m_BuildWorking = true;
m_BypassErrors = bypassEros;
m_VerifyOnly = verifyOnly;
m_BuildTaskQueue.resetQueue();
m_BuildTaskQueue.loadQueue(g_PipelineWorkspace, bypassEros);
return true;
}
return false;
}
bool abort()
{
m_BuildTaskQueue.abortQueue();
m_SlavesMutex.lock();
// Abort all slaves.
for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
{
if (it->second->ActiveTaskId != 0)
it->second->Proxy.abortBuildTask(this);
}
m_SlavesMutex.unlock();
return true;
}
protected:
NLMISC_COMMAND_HANDLER_TABLE_EXTEND_BEGIN(CModulePipelineMaster, CModuleBase)
NLMISC_COMMAND_HANDLER_ADD(CModulePipelineMaster, reloadSheets, "Reload sheets across all services", "")
NLMISC_COMMAND_HANDLER_ADD(CModulePipelineMaster, build, "Build", "")
NLMISC_COMMAND_HANDLER_ADD(CModulePipelineMaster, abort, "Abort", "")
NLMISC_COMMAND_HANDLER_TABLE_END
NLMISC_CLASS_COMMAND_DECL(reloadSheets)
@ -268,6 +396,49 @@ protected:
}
}
NLMISC_CLASS_COMMAND_DECL(build)
{
bool bypassErrors = false;
bool verifyOnly = false;
std::vector<std::string> unknownVec;
std::vector<std::string> &workingVec = unknownVec;
for (std::vector<std::string>::const_iterator it = args.begin(), end = args.end(); it != end; ++it)
{
if ((*it)[0] == '-')
{
workingVec = unknownVec;
if (((*it) == "--bypassErrors") || ((*it) == "-be"))
bypassErrors = true;
else if (((*it) == "--verifyOnly") || ((*it) == "-vo"))
bypassErrors = true;
else
unknownVec.push_back(*it);
}
else
{
workingVec.push_back(*it);
}
}
if (unknownVec.size() > 0)
{
for (std::vector<std::string>::iterator it = unknownVec.begin(), end = unknownVec.end(); it != end; ++it)
log.displayNL("Unknown build parameter: %s", (*it).c_str());
return false;
}
else
{
return build(bypassErrors, verifyOnly);
}
}
NLMISC_CLASS_COMMAND_DECL(abort)
{
if (args.size() != 0) return false;
return this->abort();
}
}; /* class CModulePipelineMaster */
void module_pipeline_master_forceLink() { }

@ -7,20 +7,29 @@
<method name="slaveFinishedBuildTask" msg="RE_BT_DONE">
<doc line=""/>
<param type="uint32" name="taskId" />
<param type="uint8" name="errorLevel" />
</method>
<method name="slaveRefusedBuildTask" msg="RE_BT_REFUSED">
<method name="slaveAbortedBuildTask" msg="RE_BT_ABORTED">
<doc line=""/>
</method>
<param type="uint32" name="taskId" />
<method name="slaveRefusedBuildTask" msg="RE_BT_REFUSED">
<doc line=""/>
</method>
<method name="slaveReloadedSheets" msg="RE_SHEETS_OK">
<doc line=""/>
</method>
<method name="slaveBuildReadySuccess" msg="RE_BRT_OK">
<doc line=""/>
</method>
<method name="slaveBuildReadyFail" msg="RE_BRT_FAIL">
<doc line=""/>
</method>
<method name="vectorPushString" msg="VEC_PUSH_STR">
<doc line=""/>

@ -146,7 +146,7 @@ public:
//this->queueModuleTask
CModulePipelineMasterProxy master(sender);
master.slaveRefusedBuildTask(this, 999999999); // NO MORE TASK ID
master.slaveRefusedBuildTask(this); // NO MORE TASK ID
}
virtual void abortBuildTask(NLNET::IModuleProxy *sender)
@ -173,6 +173,23 @@ public:
else m_ReloadSheetsState = REQUEST_MADE;
}
virtual void enterBuildReadyState(NLNET::IModuleProxy *sender)
{
if (PIPELINE::tryBuildReady())
{
m_Master->slaveBuildReadySuccess(this);
}
else
{
m_Master->slaveBuildReadyFail(this);
}
}
virtual void leaveBuildReadyState(NLNET::IModuleProxy *sender)
{
PIPELINE::endedBuildReady();
}
void sendMasterAvailablePlugins()
{
std::vector<uint32> availablePlugins;

@ -22,6 +22,14 @@
<method name="reloadSheets" msg="RELOAD_SHEETS">
</method>
<method name="enterBuildReadyState" msg="BRT_ENTER">
<doc line=""/>
</method>
<method name="leaveBuildReadyState" msg="BRT_LEAVE">
<doc line=""/>
</method>
</module_interface>
</namespace>

@ -142,6 +142,8 @@ enum EState
STATE_RUNNABLE_TASK,
STATE_BUSY_TEST,
STATE_DIRECT_CODE,
STATE_BUILD_READY,
STATE_BUILD_PROCESS,
};
/// Data
@ -152,6 +154,7 @@ CPipelineProcessImpl *s_PipelineProcessImpl = NULL;
EState s_State = STATE_IDLE;
std::string s_StateRunnableTaskName = "";
CMutex s_StateMutex;
uint s_BuildReadyRecursive = 0;
std::vector<NLMISC::CLibrary *> s_LoadedLibraries;
@ -258,6 +261,63 @@ void endedDirectTask()
endedRunnableTask(STATE_DIRECT_CODE);
}
bool tryBuildReady()
{
bool result = false;
s_StateMutex.enter();
result = (s_State == STATE_IDLE)
|| (s_State == STATE_BUILD_READY);
if (result)
{
s_State = STATE_BUILD_READY;
++s_BuildReadyRecursive;
}
s_StateMutex.leave();
if (!result) return false;
nlassert(s_State == STATE_BUILD_READY && s_BuildReadyRecursive > 0);
return true;
}
void endedBuildReady()
{
nlassert(s_State == STATE_BUILD_READY && s_BuildReadyRecursive > 0);
s_StateMutex.enter();
--s_BuildReadyRecursive;
if (s_BuildReadyRecursive == 0)
s_State = STATE_IDLE;
s_StateMutex.leave();
}
bool tryBuildProcess(const std::string &stateName)
{
bool result = false;
s_StateMutex.enter();
result = (s_State == STATE_BUILD_READY);
if (result)
{
s_State = STATE_BUILD_PROCESS;
s_StateRunnableTaskName = stateName;
}
s_StateMutex.leave();
if (!result) return false;
nlassert(s_State == STATE_BUILD_PROCESS);
return true;
}
void endedBuildProcess()
{
nlassert(s_State == STATE_BUILD_PROCESS);
s_StateMutex.enter();
s_State = STATE_BUILD_READY;
s_StateMutex.leave();
}
// ******************************************************************
namespace {
@ -472,14 +532,14 @@ public:
s_InfoFlags->addFlag("RELEASE");
NLNET::IModuleManager::releaseInstance();
while (NLMISC::CAsyncFileManager::getInstance().getNumWaitingTasks() > 0)
{
nlSleep(10);
}
NLMISC::CAsyncFileManager::terminate();
NLNET::IModuleManager::releaseInstance();
for (std::vector<NLMISC::CLibrary *>::iterator it = s_LoadedLibraries.begin(), end = s_LoadedLibraries.end(); it != end; ++it)
{
(*it)->freeLibrary();
@ -540,6 +600,17 @@ NLMISC_DYNVARIABLE(std::string, pipelineServiceState, "State of the pipeline ser
case PIPELINE::STATE_BUSY_TEST:
*pointer = "BUSY_TEST";
break;
case PIPELINE::STATE_BUILD_READY:
if (PIPELINE::s_BuildReadyRecursive == 1)
*pointer = "BUILD_READY (S)";
else if (PIPELINE::s_BuildReadyRecursive == 2)
*pointer = "BUILD_READY (M)";
else
*pointer = "BUILD_READY (???)";
break;
case PIPELINE::STATE_BUILD_PROCESS:
*pointer = "BP: " + PIPELINE::s_StateRunnableTaskName;
break;
}
}
}

@ -73,6 +73,12 @@ void endedRunnableTask();
bool tryDirectTask(const std::string &stateName);
void endedDirectTask();
bool tryBuildReady();
void endedBuildReady();
bool tryBuildProcess(const std::string &stateName);
void endedBuildProcess();
/// Only use for informational purposes, service may not be idle after calling this. Not for thread safe usage.
bool isServiceStateIdle();

@ -149,7 +149,7 @@ void CPipelineWorkspace::getProcessPlugins(std::vector<CProcessPluginInfo> &resu
processPlugin.Id.Sub.Handler = i;
resultAppend.push_back(processPlugin);
nldebug("Found '%s': '%s', '%s'", process.c_str(), processPlugin.Handler.c_str(), processPlugin.Info.c_str());
// nldebug("Found '%s': '%s', '%s'", process.c_str(), processPlugin.Handler.c_str(), processPlugin.Info.c_str());
}
else
{

@ -2,6 +2,7 @@
<DFN Revision="$Revision$" State="modified">
<ELEMENT Name="Description" Type="Type" Filename="string.typ"/>
<ELEMENT Name="Process" Type="Type" Filename="string.typ"/>
<ELEMENT Name="ProcessDependencies" Type="Type" Filename="string.typ" Array="true"/>
<ELEMENT Name="HandlerType" Type="Type" Filename="pipeline_plugin_process_type.typ"/>
<ELEMENT Name="Handler" Type="Type" Filename="string.typ"/>
<ELEMENT Name="InfoType" Type="Type" Filename="pipeline_plugin_process_type.typ"/>
@ -9,5 +10,6 @@
<LOG>Sat Feb 18 13:34:33 2012 (Kaetemi) Dfn Structure =
Sat Feb 18 13:34:44 2012 (Kaetemi) Dfn Structure =
Sat Feb 18 13:44:42 2012 (Kaetemi) Dfn Structure =
Sat Mar 03 11:06:35 2012 (Kaetemi) Dfn Structure = </LOG>
Sat Mar 03 11:06:35 2012 (Kaetemi) Dfn Structure =
Fri Mar 09 19:46:05 2012 (Kaetemi) Dfn Structure = </LOG>
</DFN>

@ -14,6 +14,7 @@
</ARRAY>
<ARRAY Name="Processes">
<ATOM Value="Interface"/>
<ATOM Value="Dummy"/>
</ARRAY>
<STRUCT Name="Interface">
<ARRAY Name="Atlas">
@ -92,5 +93,7 @@ Sat Mar 03 13:34:02 2012 (Kaetemi) .Interface.Atlas[2].DstFile = [&amp;DstInterf
Sat Mar 03 13:34:02 2012 (Kaetemi) .Macros[1].Name = DstInterfaceAtlasDxtc
Sat Mar 03 13:34:02 2012 (Kaetemi) .Macros[1].Value = [!OutputDirectory]/interface_atlas_dxtc/
Sat Mar 03 13:34:02 2012 (Kaetemi) formName Resized = 2
Sat Mar 03 13:34:32 2012 (Kaetemi) .Interface.AtlasDxtc[0].DstFile = [&amp;DstInterfaceAtlasDxtc]/texture_interfaces_dxtc.tga</LOG>
Sat Mar 03 13:34:32 2012 (Kaetemi) .Interface.AtlasDxtc[0].DstFile = [&amp;DstInterfaceAtlasDxtc]/texture_interfaces_dxtc.tga
Fri Mar 09 19:49:23 2012 (Kaetemi) .Processes[1] = Dummy
Fri Mar 09 19:49:23 2012 (Kaetemi) formName Resized = 2</LOG>
</FORM>

@ -4,7 +4,17 @@
<ATOM Name="Description" Value="Pipeline Plugin NeL"/>
<ARRAY Name="ProcessHandlers">
<STRUCT>
<ATOM Name="Description" Value="Invalid process for testing"/>
<ATOM Name="Process" Value="Dummy"/>
<ATOM Name="Handler" Value="CProcessDummy"/>
<ATOM Name="Info" Value="CProcessDummy"/>
</STRUCT>
<STRUCT>
<ATOM Name="Description" Value="Builds interface textures"/>
<ATOM Name="Process" Value="Interface"/>
<ARRAY Name="ProcessDependencies">
<ATOM Value="Dummy"/>
</ARRAY>
<ATOM Name="HandlerType" Value="RegisteredClass"/>
<ATOM Name="Handler" Value="CProcessInterface"/>
<ATOM Name="Info" Value="CProcessInterfaceInfo"/>
@ -22,5 +32,15 @@ Sat Mar 03 10:55:24 2012 (Kaetemi) formName Resized = 1
Sat Mar 03 10:56:08 2012 (Kaetemi) .ProcessHandlers[0].Handler = CProcessInterface
Sat Mar 03 10:56:08 2012 (Kaetemi) .ProcessHandlers[0].Process = Interface
Sat Mar 03 11:13:55 2012 (Kaetemi) .ProcessHandlers[0].Info = CProcessInterfaceInfo
Sat Mar 03 12:10:28 2012 (Kaetemi) .Description = Pipeline Plugin NeL</LOG>
Sat Mar 03 12:10:28 2012 (Kaetemi) .Description = Pipeline Plugin NeL
Fri Mar 09 19:48:26 2012 (Kaetemi) .ProcessHandlers[0].Description = Invalid process for testing
Fri Mar 09 19:48:26 2012 (Kaetemi) .ProcessHandlers[0].Handler = CProcessDummy
Fri Mar 09 19:48:26 2012 (Kaetemi) .ProcessHandlers[0].Info = CProcessDummy
Fri Mar 09 19:48:26 2012 (Kaetemi) .ProcessHandlers[0].Process = Dummy
Fri Mar 09 19:48:26 2012 (Kaetemi) .ProcessHandlers[1].Description = Builds interface textures
Fri Mar 09 19:48:26 2012 (Kaetemi) Array Insert = 0
Fri Mar 09 19:48:26 2012 (Kaetemi) formName Deleted =
Fri Mar 09 19:48:26 2012 (Kaetemi) formName Resized = 2
Fri Mar 09 19:48:39 2012 (Kaetemi) .ProcessHandlers[1].ProcessDependencies[0] = Dummy
Fri Mar 09 19:48:39 2012 (Kaetemi) formName Resized = 1</LOG>
</FORM>

Loading…
Cancel
Save