diff --git a/code/nel/tools/3d/build_interface/main.cpp b/code/nel/tools/3d/build_interface/main.cpp index f2e42b80c..25bd35df9 100644 --- a/code/nel/tools/3d/build_interface/main.cpp +++ b/code/nel/tools/3d/build_interface/main.cpp @@ -280,6 +280,9 @@ int main(int nNbArg, char **ppArgs) inFile.open( AllMapNames[i] ); pBtmp->load(inFile); AllMaps.push_back(pBtmp); + + if (pBtmp->getWidth() == 0 || pBtmp->getHeight() == 0) + ToolLogger.writeError(PIPELINE::WARNING, AllMapNames[i], "Bitmap of width or height 0, corrupt data"); } catch (const NLMISC::Exception &e) { diff --git a/code/nel/tools/pipeline/plugin_library/pipeline_interface.h b/code/nel/tools/pipeline/plugin_library/pipeline_interface.h index 3a4bd8880..b1f34ae8a 100644 --- a/code/nel/tools/pipeline/plugin_library/pipeline_interface.h +++ b/code/nel/tools/pipeline/plugin_library/pipeline_interface.h @@ -60,6 +60,7 @@ public: // ***************** PLUGIN UTILITY FUNCTIONS ***************** // ************* DO NOT USE FROM INSIDE A PROCESS ************* + // ************ USE ONLY FOR SPECIAL TOOL COMMANDS ************ /// Get the configuration file of the pipeline service. Must only be used for configuration values that may be different on different services, such as tool paths. virtual NLMISC::CConfigFile &getConfigFile() = 0; @@ -78,6 +79,9 @@ public: /// Call when a runnable task has ended to reset to STATE_IDLE. virtual void endedDirectCode() = 0; + + /// Check if service is exiting + virtual bool isExiting() = 0; }; /* class IPipelineInterface */ diff --git a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp index 4cc110b05..c61cefb2d 100644 --- a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp +++ b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp @@ -36,6 +36,9 @@ // NeL includes #include "nel/misc/dynloadlib.h" #include "nel/misc/debug.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/file.h" // Project includes #include "../plugin_library/pipeline_interface.h" @@ -83,6 +86,463 @@ CPipelinePluginMax::~CPipelinePluginMax() }*/ +namespace { + +std::set MissingFiles; +std::map KnownFileCache; + +// COPY FROM PIPELINE_SERVICE.CPP WITH BACKSLASHES INSTEAD OF FORWARD SLASHES +std::string standardizePath(const std::string &path, bool addFinalSlash) +{ + // check empty path + if (path.empty()) + return ""; + + std::string newPath; + newPath.resize(path.size() + 1); + + std::string::size_type j = 0; + for (std::string::size_type i = 0; i < path.size(); ++i) + { + if (path[i] == '\\' || path[i] == '/') + { + if (j <= 1 && path[i] == '\\') + { + // for windows network + newPath[j] = '\\'; + ++j; + } + else if (j == 0 || newPath[j - 1] != '\\') + { + newPath[j] = '\\'; + ++j; + } + } + else + { + newPath[j] = path[i]; + ++j; + } + } + newPath[j] = 0; + newPath.resize(j); + + // add terminal slash + if (addFinalSlash && newPath[newPath.size()-1] != '\\') + newPath += '\\'; + + return newPath; +} + +inline bool isCharacter(char c) +{ + return (32 <= c /*&& c <= 127) || (161 <= c*/ && c <= 255); +} + +inline char stripFrenchLocale(char c) +{ + if (192 <= c && c <= 197) return 'a'; + if (200 <= c && c <= 203) return 'e'; + if (204 <= c && c <= 207) return 'i'; + if (210 <= c && c <= 214) return 'o'; + if (217 <= c && c <= 220) return 'u'; + if (c == 221) return 'y'; + if (224 <= c && c <= 229) return 'a'; + if (232 <= c && c <= 235) return 'e'; + if (236 <= c && c <= 239) return 'i'; + if (242 <= c && c <= 246) return 'o'; + if (249 <= c && c <= 252) return 'u'; + if (c == 253 || c == 255) return 'y'; + return c; +} + +// maxRewritePaths W:/database/interfaces/anims_max + +std::string rewritePath(const std::string &path, const std::string &databaseDirectory) +{ + static std::set fileNameCache; + + std::string stdPath = standardizePath(path, false); + for (std::string::size_type i = 0; i < stdPath.size(); ++i) + stdPath[i] = stripFrenchLocale(stdPath[i]); + stdPath = NLMISC::toLower(stdPath); + + // TODO: remove ./stuff/caravan/agents/_textures/actors/trame.png + + NLMISC::strFindReplace(stdPath, "w:\\database\\", databaseDirectory); + NLMISC::strFindReplace(stdPath, "\\\\amiga\\3d\\database\\", databaseDirectory); + NLMISC::strFindReplace(stdPath, "ma_hom_armor_01", "ma_hom_armor01"); + NLMISC::strFindReplace(stdPath, "ma_hof_armor_01", "ma_hof_armor01"); + NLMISC::strFindReplace(stdPath, "ma_hom_armor_00", "ma_hom_armor00"); + NLMISC::strFindReplace(stdPath, "ma_hof_armor_00", "ma_hof_armor00"); + NLMISC::strFindReplace(stdPath, "zo_hom_armor_00", "zo_hom_armor00"); + NLMISC::strFindReplace(stdPath, "zo_hof_armor_00", "zo_hof_armor00"); + NLMISC::strFindReplace(stdPath, "fy_hof_cheveux_shave01", "fy_hof_cheveux_shave"); + + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_avtbras", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_avtbras"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_epaule", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_epaule"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_hand-downside", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_hand-downside"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_hand-upside", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_hand-upside"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_molet", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_molet"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hof_underwear_hand-downside", "tryker\\agents\\_textures\\actors\\tr_hof_underwear_hand-downside"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hof_underwear_hand-upside", "tryker\\agents\\_textures\\actors\\tr_hof_underwear_hand-upside"); + + NLMISC::strFindReplace(stdPath, "tr_hof_underwear_torso", "tr_hof_underwear_torse"); + NLMISC::strFindReplace(stdPath, "tr_hof_underwear_hand-", "tr_hof_underwear_hand_"); + + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_avbras.", "fy_hom_armor00_avbras_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_bottes.", "fy_hom_armor00_bottes_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_bras.", "fy_hom_armor00_bras_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_cuissear.", "fy_hom_armor00_cuissear_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_cuisseav.", "fy_hom_armor00_cuisseav_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_dos.", "fy_hom_armor00_dos_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_hand-downside.", "fy_hom_armor00_hand-downside_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_hand-upside.", "fy_hom_armor00_hand-upside_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_pieds.", "fy_hom_armor00_pieds_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_torse.", "fy_hom_armor00_torse_c1."); + + NLMISC::strFindReplace(stdPath, "interfaces_visage", "visage_interface"); + + if (stdPath.find("\\trame.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/lod_actors/texture_lod/trame.png", false); + if (stdPath.find("\\tr_hof_visage_c1.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c1.png", false); + if (stdPath.find("\\tr_hof_visage_c2.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c2.png", false); + if (stdPath.find("\\tr_hof_visage_c3.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c3.png", false); + if (stdPath.find("\\ma_hof_cheveux_medium02.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/matis/agents/_textures/actors/ma_hof_cheveux_medium02.png", false); + + /*if (stdPath.size() > path.size()) + { + nlwarning("Path size becomes too large: '%s' -> '%s'", path.c_str(), stdPath.c_str()); + return path; + }*/ + + if (NLMISC::CFile::getFilename(stdPath) == stdPath) + { + breakable + { + if (fileNameCache.find(stdPath) == fileNameCache.end() || KnownFileCache.find(stdPath) == KnownFileCache.end()) + { + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { + stdPath[stdPath.size() - 3] = 'p'; + stdPath[stdPath.size() - 2] = 'n'; + stdPath[stdPath.size() - 1] = 'g'; + if (fileNameCache.find(stdPath) != fileNameCache.end()) + break; + } + nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str()); + return stdPath; + } + } + return stdPath; + } + else + { + if (stdPath.find(databaseDirectory) == std::string::npos) + { + if (stdPath[1] == ':' || (stdPath[0] == '\\' && stdPath[1] == '\\')) + { + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { stdPath[stdPath.size() - 3] = 'p'; stdPath[stdPath.size() - 2] = 'n'; stdPath[stdPath.size() - 1] = 'g'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 'p' && stdPath[stdPath.size() - 2] == 'n' && stdPath[stdPath.size() - 1] == 'g') + { stdPath[stdPath.size() - 3] = 't'; stdPath[stdPath.size() - 2] = 'g'; stdPath[stdPath.size() - 1] = 'a'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str()); + MissingFiles.insert(path); + return stdPath; + } + else + { + // invalid path, don't care too much + return stdPath; + } + } + + breakable + { + if (!NLMISC::CFile::fileExists(stdPath)) + { + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { + stdPath[stdPath.size() - 3] = 'p'; + stdPath[stdPath.size() - 2] = 'n'; + stdPath[stdPath.size() - 1] = 'g'; + if (NLMISC::CFile::fileExists(stdPath)) + break; + } + { + std::string stdPathVv2 = standardizePath(NLMISC::CFile::getPath(stdPath) + "/vv2/" + NLMISC::CFile::getFilename(stdPath), false); + bool vv2works = false; + if (NLMISC::CFile::fileExists(stdPathVv2)) + { + vv2works = true; + } + else + { + if (stdPathVv2[stdPathVv2.size() - 3] == 'p' && stdPathVv2[stdPathVv2.size() - 2] == 'n' && stdPathVv2[stdPathVv2.size() - 1] == 'g') + { + stdPathVv2[stdPathVv2.size() - 3] = 't'; + stdPathVv2[stdPathVv2.size() - 2] = 'g'; + stdPathVv2[stdPathVv2.size() - 1] = 'a'; + if (NLMISC::CFile::fileExists(stdPathVv2)) + { + vv2works = true; + } + } + } + if (vv2works) + { + // try with vv2 + /*if (stdPathVv2.size() > path.size()) + { + nlwarning("Path with vv2 size becomes too large: '%s' -> '%s'", path.c_str(), stdPathVv2.c_str()); + return stdPath; + } + else + {*/ + stdPath = stdPathVv2; + break; + //} + } + } + // try find + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { stdPath[stdPath.size() - 3] = 'p'; stdPath[stdPath.size() - 2] = 'n'; stdPath[stdPath.size() - 1] = 'g'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 'p' && stdPath[stdPath.size() - 2] == 'n' && stdPath[stdPath.size() - 1] == 'g') + { stdPath[stdPath.size() - 3] = 't'; stdPath[stdPath.size() - 2] = 'g'; stdPath[stdPath.size() - 1] = 'a'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str()); + MissingFiles.insert(path); + return stdPath; + } + } + } + + fileNameCache.insert(NLMISC::CFile::getFilename(stdPath)); + return stdPath; +} + +class CMaxRewritePathsCommand : public NLMISC::IRunnable +{ +public: + NLMISC::CLog *Log; + std::string SrcDirectoryRecursive; + std::string DatabaseDirectory; + + virtual void getName(std::string &result) const + { result = "CMaxRewritePathsCommand"; } + + void doFile(const std::string &filePath) + { + if (filePath[filePath.size() - 3] == 'm' && filePath[filePath.size() - 2] == 'a' && filePath[filePath.size() - 1] == 'x') + { + nldebug("File: '%s'", filePath.c_str()); + + std::vector buffer; + buffer.resize(NLMISC::CFile::getFileSize(filePath)); + + // read + { + NLMISC::CIFile ifile; + ifile.open(filePath, false); + ifile.serialBuffer((uint8 *)(&buffer[0]), buffer.size()); + ifile.close(); + } + + // find paths + for (std::vector::size_type i = 256; i < buffer.size(); ++i) // skip the first 256 lol :) + { + if (((NLMISC::toLower(buffer[i - 1]) == 'x' && NLMISC::toLower(buffer [i - 2]) == 'a' && NLMISC::toLower(buffer[i - 3]) == 'm') + || (NLMISC::toLower(buffer[i - 1]) == 'a' && NLMISC::toLower(buffer [i - 2]) == 'g' && NLMISC::toLower(buffer[i - 3]) == 't') + || (NLMISC::toLower(buffer[i - 1]) == 'g' && NLMISC::toLower(buffer [i - 2]) == 'n' && NLMISC::toLower(buffer[i - 3]) == 'p')) + && (NLMISC::toLower(buffer[i - 4]) == '.')) + { + // buffer[i] is the character after the path! :) + std::vector::size_type beginPath = 0; + for (std::vector::size_type j = i - 4; j > 0; --j) + { + if (!isCharacter(buffer[j])) + { + if (buffer[j + 1] == '\\' && buffer[j + 2] == '\\' || buffer[j] == 0) + { + beginPath = j + 1; + break; + } + // nlwarning("Invalid characters '%i' in path at %i, len %i!", (uint32)buffer[j], (uint32)j, (uint32)(i - j)); + // beginPath = j + 1; // test + break; + } + if (buffer[j] == ':') + { + beginPath = j - 1; + break; + } + } + if (beginPath != 0) + { + std::vector::size_type sizePath = i - beginPath; + std::string foundPath = std::string(&buffer[beginPath], sizePath); //std::string(buffer.at(beginPath), buffer.at(i)); + //nldebug("Found path: '%s' from %i to %i", foundPath.c_str(), (uint32)beginPath, (uint32)i); + std::string fixedPath = rewritePath(foundPath, DatabaseDirectory); + //nldebug("Rewrite to: '%s'", fixedPath.c_str()); + } + } + } + + //NLMISC::CFile::re + + // ... + } + } + + // maxRewritePaths W:/database/interfaces/anims_max + + void doDirectory(const std::string &directoryPath) + { + nldebug("Directory: '%s'", directoryPath.c_str()); + + std::string dirPath = standardizePath(directoryPath, true); + std::vector dirContents; + + NLMISC::CPath::getPathContent(dirPath, false, true, true, dirContents); + + for (std::vector::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it) + { + const std::string &subPath = standardizePath(*it, false); + + if (NLMISC::CFile::isDirectory(subPath)) + { + if (subPath.find("\\.") == std::string.npos) + doDirectory(subPath); + } + else + doFile(subPath); + + if (PIPELINE::IPipelineInterface::getInstance()->isExiting()) + return; + } + } + + virtual void run() + { + std::string tempDirectory = PIPELINE::IPipelineProcess::getInstance()->getTempDirectory(); + DatabaseDirectory = standardizePath(PIPELINE::IPipelineInterface::getInstance()->getConfigFile().getVar("DatabaseDirectory").asString(0), true); + nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory.c_str()); + + doDirectory(SrcDirectoryRecursive); + + for (std::set::iterator it = MissingFiles.begin(), end = MissingFiles.end(); it != end; ++it) + nlinfo("Missing: '%s'", (*it).c_str()); + + PIPELINE::IPipelineInterface::getInstance()->endedRunnableTask(); + } +}; +CMaxRewritePathsCommand s_MaxRewritePathsCommand; + +class CMaxRewriteInitCacheCommand : public NLMISC::IRunnable +{ +public: + NLMISC::CLog *Log; + std::string SrcDirectoryRecursive; + std::string DatabaseDirectory; + + virtual void getName(std::string &result) const + { result = "CMaxRewriteInitCacheCommand"; } + + void doFile(const std::string &filePath) + { + KnownFileCache[NLMISC::CFile::getFilename(filePath)] = standardizePath(filePath, true); + } + + // maxRewritePaths W:/database/interfaces/anims_max + + void doDirectory(const std::string &directoryPath) + { + nldebug("Directory: '%s'", directoryPath.c_str()); + + std::string dirPath = standardizePath(directoryPath, true); + std::vector dirContents; + + NLMISC::CPath::getPathContent(dirPath, false, true, true, dirContents); + + for (std::vector::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it) + { + const std::string &subPath = standardizePath(*it, false); + + if (NLMISC::CFile::isDirectory(subPath)) + { + if (subPath.find("\\.") == std::string.npos) + doDirectory(subPath); + } + else + doFile(subPath); + + if (PIPELINE::IPipelineInterface::getInstance()->isExiting()) + return; + } + } + + virtual void run() + { + std::string tempDirectory = PIPELINE::IPipelineProcess::getInstance()->getTempDirectory(); + DatabaseDirectory = standardizePath(PIPELINE::IPipelineInterface::getInstance()->getConfigFile().getVar("DatabaseDirectory").asString(0), true); + nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory.c_str()); + + doDirectory(SrcDirectoryRecursive); + + PIPELINE::IPipelineInterface::getInstance()->endedRunnableTask(); + } +}; +CMaxRewriteInitCacheCommand s_MaxRewriteInitCacheCommand; + +} /* anonymous namespace */ + } /* namespace PIPELINE */ +NLMISC_CATEGORISED_COMMAND(max, maxRewriteInitCache, "Find all .tga, .png and .max files", "") +{ + if(args.size() != 1) return false; + PIPELINE::s_MaxRewriteInitCacheCommand.Log = &log; + PIPELINE::s_MaxRewriteInitCacheCommand.SrcDirectoryRecursive = args[0]; + if (!PIPELINE::IPipelineInterface::getInstance()->tryRunnableTask("COMMAND_MAX_REWRITE_INIT_CACHE", &PIPELINE::s_MaxRewriteInitCacheCommand)) + { + log.displayNL("Busy."); + return false; + } + return true; +} + +NLMISC_CATEGORISED_COMMAND(max, maxRewritePaths, "Rewrite all paths to .tga, .png and .max files found in a max file", "") +{ + if(args.size() != 1) return false; + PIPELINE::s_MaxRewritePathsCommand.Log = &log; + PIPELINE::s_MaxRewritePathsCommand.SrcDirectoryRecursive = args[0]; + if (!PIPELINE::IPipelineInterface::getInstance()->tryRunnableTask("COMMAND_MAX_REWRITE_PATHS", &PIPELINE::s_MaxRewritePathsCommand)) + { + log.displayNL("Busy."); + return false; + } + return true; +} + /* end of file */ diff --git a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h index dd45afd80..784cdb915 100644 --- a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h +++ b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h @@ -32,6 +32,8 @@ // STL includes // NeL includes +#include +#include // Project includes diff --git a/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg b/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg index b3f8a1caa..9aaff3b2c 100644 --- a/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg +++ b/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg @@ -54,6 +54,9 @@ ToolBnpMake = "bnp_make"; // MasterIgnoreProcessPlugins = { }; // Only used by the master service. NOT USED, USE WORKSPACE INTEAD! // MasterTerminalPassword = "MASTERCLIENTPASSWORD"; // Only used by the master service. +// Log containing the stuff that would be sent from the master to the user terminal +MasterLog = SharedWork + "/master.log"; + // MasterSlavePassword = "MASTERSLAVEPASSWORD"; MasterAddress = "localhost"; MasterPort = 50123; diff --git a/code/nel/tools/pipeline/service/metadata_storage.cpp b/code/nel/tools/pipeline/service/metadata_storage.cpp index 80fd20399..cf467f363 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.cpp +++ b/code/nel/tools/pipeline/service/metadata_storage.cpp @@ -50,8 +50,10 @@ namespace PIPELINE { void CFileError::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) { uint version = stream.serialVersion(1); + stream.serial((uint8 &)MasterTime); // does this work? stream.serial(Project); - stream.serial(Process); + stream.serial(Plugin); + stream.serial(Time); stream.serial(Message); } diff --git a/code/nel/tools/pipeline/service/metadata_storage.h b/code/nel/tools/pipeline/service/metadata_storage.h index 205db4508..d41c53b86 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.h +++ b/code/nel/tools/pipeline/service/metadata_storage.h @@ -33,6 +33,7 @@ // NeL includes #include +#include // Project includes #include "workspace_storage.h" @@ -76,11 +77,11 @@ public: struct CFileError { public: - uint32 Time; // The time when this error occured. - // TFileState Level; // Success, Warning, Error, Removal + uint32 MasterTime; // The time when this error occured. + TError Level; std::string Project; - std::string Process; - // std::string Plugin; + std::string Plugin; + std::string Time; std::string Message; void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); @@ -180,6 +181,9 @@ public: // static void createRemove(const CFileRemove &remove, const std::string &path); // Remove cannot be modified after creation, only erased. // static void eraseRemove(const std::string &path); + static std::string getErrorPath(const std::string &file) { return "[TODO]"; } + static void appendError(const CFileError &error, const std::string &metaPath) { /* TODO */ } + static std::string getDependPath(const std::string &file); static bool readDepend(CFileDepend &depend, const std::string &metaPath); static void writeDepend(const CFileDepend &depend, const std::string &metaPath); diff --git a/code/nel/tools/pipeline/service/module_pipeline_master.cpp b/code/nel/tools/pipeline/service/module_pipeline_master.cpp index 73c963a90..183099a29 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_master.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_master.cpp @@ -36,6 +36,7 @@ #include #include #include +#include // Project includes #include "info_flags.h" @@ -157,7 +158,7 @@ protected: bool m_VerifyOnly; public: - CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks") + CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks"), m_MasterLog(NULL) { g_IsMaster = true; m_TaskManager = new NLMISC::CTaskManager(); @@ -397,6 +398,7 @@ public: CProcessPluginInfo pluginInfo; g_PipelineWorkspace->getProcessPlugin(pluginInfo, taskInfo->ProcessPluginId); nlinfo("Dispatching task '%i' ('%s': '%s') to slave '%s'", taskInfo->Id.Global, taskInfo->ProjectName.c_str(), pluginInfo.Handler.c_str(), it->second->Proxy.getModuleProxy()->getModuleName().c_str()); + notifyTerminalTaskBegin(taskInfo->Id.Global); } } } @@ -431,6 +433,8 @@ public: PIPELINE::endedBuildReadyMaster(); + notifyTerminalBuildEnd(); + nlinfo("#####################################"); nlinfo("#####################################"); nlinfo("! DONE !"); @@ -503,6 +507,31 @@ public: CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_REJECTED); } + virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string ¯oPath, const std::string &time, const std::string &error) + { + TSlaveMap::iterator slaveIt = m_Slaves.find(sender); + if (slaveIt == m_Slaves.end()) { nlerror("Received 'slaveLoggedToolError' from unknown slave at '%s'", sender->getModuleName().c_str()); m_Slaves.erase(sender); /*m_SlavesMutex.unlock();*/ return; } + CSlave *slave = slaveIt->second; + + if (!macroPath.empty()) + { + CBuildTaskInfo *task = m_BuildTaskQueue.getTaskInfo(slave->ActiveTaskId); + CProcessPluginInfo pluginInfo; + g_PipelineWorkspace->getProcessPlugin(pluginInfo, task->ProcessPluginId); + + CFileError fe; + fe.MasterTime = CTime::getSecondsSince1970(); + fe.Level = (TError)type; + fe.Message = error; + fe.Time = time; + fe.Project = task->ProjectName;; + fe.Plugin = pluginInfo.Handler; + CMetadataStorage::appendError(fe, CMetadataStorage::getErrorPath(unMacroPath(macroPath))); + } + + notifyTerminalTaskMessage(slave->ActiveTaskId, (TError)type, macroPath, time, error); + } + virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender) { //m_SlavesMutex.lock(); @@ -550,11 +579,69 @@ public: /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// + + FILE *m_MasterLog; + + void notifyTerminalBuildStart() + { + nlinfo(""); + // TODO NOTIFY TERMINAL + if (m_MasterLog) + { + fflush(m_MasterLog); + fclose(m_MasterLog); + m_MasterLog = NULL; + } + m_MasterLog = fopen(IService::getInstance()->ConfigFile.getVar("MasterLog").asString(0).c_str(), "at"); + fprintf(m_MasterLog, "[BUILD_START]\n"); + fflush(m_MasterLog); + } + + // task info + void notifyTerminalTaskInfo(uint32 taskId, const std::string &projectName, const std::string &processHandler) + { + nlinfo("taskId: %i, projectName: %s, processHandler: %s", taskId, projectName.c_str(), processHandler.c_str()); + // TODO NOTIFY TERMINAL + fprintf(m_MasterLog, "[TASK_INFO] taskId: %i, projectName: %s, processHandler: %s\n", taskId, projectName.c_str(), processHandler.c_str()); + fflush(m_MasterLog); + } + + // task begin + void notifyTerminalTaskBegin(uint32 taskId) + { + nlinfo("taskId: %i", taskId); + // TODO NOTIFY TERMINAL + fprintf(m_MasterLog, "[TASK_BEGIN] taskId: %i\n", taskId); + fflush(m_MasterLog); + } + // task issues + void notifyTerminalTaskMessage(uint32 taskId, TError type, const std::string ¯oPath, const std::string &time, const std::string &error) + { + nlinfo("taskId: %i, type: %i, macroPath: %s, time: %s, error: %s", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str()); + // TODO NOTIFY TERMINAL (send type as uint8 as usual) + fprintf(m_MasterLog, "[TASK_MESSAGE] taskId: %i, type: %i, macroPath: %s, time: %s, error: %s\n", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str()); + fflush(m_MasterLog); + } + + // task end void notifyTerminalTaskState(uint32 taskId, TProcessResult errorLevel, const std::string &errorMessage) { nlinfo("taskId: %i, errorLevel: %i, errorMessage: %s", taskId, (uint32)errorLevel, errorMessage.c_str()); // TODO NOTIFY TERMINAL (send errorlevel as uint8 as usual) + fprintf(m_MasterLog, "[TASK_STATE] taskId: %i, errorLevel: %i, errorMessage: %s\n", taskId, (uint32)errorLevel, errorMessage.c_str()); + fflush(m_MasterLog); + } + + // build end (any remaining tasks are considered aborted) + void notifyTerminalBuildEnd() + { + nlinfo(""); + // TODO NOTIFY TERMINAL + fprintf(m_MasterLog, "[BUILD_END]\n"); + fflush(m_MasterLog); + fclose(m_MasterLog); + m_MasterLog = NULL; } /////////////////////////////////////////////////////////////////// @@ -711,11 +798,26 @@ public: { if (PIPELINE::tryBuildReadyMaster()) { + notifyTerminalBuildStart(); m_BuildWorking = true; m_BypassErrors = bypassEros; m_VerifyOnly = verifyOnly; m_BuildTaskQueue.resetQueue(); m_BuildTaskQueue.loadQueue(g_PipelineWorkspace, bypassEros); + + // notify terminal task info + { + std::vector tasks; + m_BuildTaskQueue.listTaskQueueByMostDependents(tasks); + for (std::vector::iterator it = tasks.begin(), end = tasks.end(); it != end; ++it) + { + PIPELINE::CBuildTaskInfo *task = *it; + PIPELINE::CProcessPluginInfo pluginInfo; + PIPELINE::g_PipelineWorkspace->getProcessPlugin(pluginInfo, task->ProcessPluginId); + notifyTerminalTaskInfo(task->Id.Global, task->ProjectName, pluginInfo.Handler); + } + } + return true; } return false; diff --git a/code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp b/code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp index 542c7c07d..ba00c10a3 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp @@ -34,6 +34,10 @@ namespace PIPELINE // if this assert, you have a doubly message name in your interface definition ! nlassert(res.second); + res = handlers.insert(std::make_pair(std::string("TL_ERR_LOG"), &CModulePipelineMasterSkel::slaveLoggedToolError_skel)); + // if this assert, you have a doubly message name in your interface definition ! + nlassert(res.second); + res = handlers.insert(std::make_pair(std::string("RE_SHEETS_OK"), &CModulePipelineMasterSkel::slaveReloadedSheets_skel)); // if this assert, you have a doubly message name in your interface definition ! nlassert(res.second); @@ -103,6 +107,20 @@ namespace PIPELINE slaveRefusedBuildTask(sender); } + void CModulePipelineMasterSkel::slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message) + { + H_AUTO(CModulePipelineMasterSkel_slaveLoggedToolError_TL_ERR_LOG); + uint8 type; + nlRead(__message, serial, type); + std::string macroPath; + nlRead(__message, serial, macroPath); + std::string time; + nlRead(__message, serial, time); + std::string error; + nlRead(__message, serial, error); + slaveLoggedToolError(sender, type, macroPath, time, error); + } + void CModulePipelineMasterSkel::slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message) { H_AUTO(CModulePipelineMasterSkel_slaveReloadedSheets_RE_SHEETS_OK); @@ -197,6 +215,24 @@ namespace PIPELINE } } // + void CModulePipelineMasterProxy::slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string ¯oPath, const std::string &time, const std::string &error) + { + if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported()) + { + // immediate local synchronous dispatching + _LocalModuleSkel->slaveLoggedToolError(_ModuleProxy->getModuleGateway()->getPluggedModuleProxy(sender), type, macroPath, time, error); + } + else + { + // send the message for remote dispatching and execution or local queing + NLNET::CMessage __message; + + buildMessageFor_slaveLoggedToolError(__message, type, macroPath, time, error); + + _ModuleProxy->sendModuleMessage(sender, __message); + } + } + // void CModulePipelineMasterProxy::slaveReloadedSheets(NLNET::IModule *sender) { if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported()) @@ -334,6 +370,19 @@ namespace PIPELINE return __message; } + // Message serializer. Return the message received in reference for easier integration + const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string ¯oPath, const std::string &time, const std::string &error) + { + __message.setType("TL_ERR_LOG"); + nlWrite(__message, serial, type); + nlWrite(__message, serial, const_cast < std::string& > (macroPath)); + nlWrite(__message, serial, const_cast < std::string& > (time)); + nlWrite(__message, serial, const_cast < std::string& > (error)); + + + return __message; + } + // Message serializer. Return the message received in reference for easier integration const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message) { diff --git a/code/nel/tools/pipeline/service/module_pipeline_master_itf.h b/code/nel/tools/pipeline/service/module_pipeline_master_itf.h index b2d5c9578..4e368208e 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_master_itf.h +++ b/code/nel/tools/pipeline/service/module_pipeline_master_itf.h @@ -66,6 +66,8 @@ namespace PIPELINE void slaveRefusedBuildTask_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message); + void slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message); + void slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message); void slaveBuildReadySuccess_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message); @@ -95,6 +97,8 @@ namespace PIPELINE // virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender) =0; // + virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string ¯oPath, const std::string &time, const std::string &error) =0; + // virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender) =0; // virtual void slaveBuildReadySuccess(NLNET::IModuleProxy *sender) =0; @@ -162,6 +166,8 @@ namespace PIPELINE // void slaveRefusedBuildTask(NLNET::IModule *sender); // + void slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string ¯oPath, const std::string &time, const std::string &error); + // void slaveReloadedSheets(NLNET::IModule *sender); // void slaveBuildReadySuccess(NLNET::IModule *sender); @@ -183,6 +189,9 @@ namespace PIPELINE // Message serializer. Return the message received in reference for easier integration static const NLNET::CMessage &buildMessageFor_slaveRefusedBuildTask(NLNET::CMessage &__message); + // Message serializer. Return the message received in reference for easier integration + static const NLNET::CMessage &buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string ¯oPath, const std::string &time, const std::string &error); + // Message serializer. Return the message received in reference for easier integration static const NLNET::CMessage &buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message); diff --git a/code/nel/tools/pipeline/service/module_pipeline_master_itf.xml b/code/nel/tools/pipeline/service/module_pipeline_master_itf.xml index 1a1f7a2ff..66f9bb5f8 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_master_itf.xml +++ b/code/nel/tools/pipeline/service/module_pipeline_master_itf.xml @@ -19,6 +19,15 @@ + + + + + + + + + diff --git a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp index ea8db6fd0..80e9af6fb 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp @@ -114,8 +114,18 @@ public: bool m_PluginBuildDone; + struct CErrorLogData + { + TError Type; + std::string Path; + std::string Time; + std::string Error; + }; + + NLMISC::CSynchronized > m_ErrorLogWaiting; + public: - CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false) + CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false), m_ErrorLogWaiting("ErrorLogWaiting") { NLMISC::CSynchronized::CAccessor(&m_StatusUpdateMasterDone).value() = false; NLMISC::CSynchronized::CAccessor(&m_StatusUpdateSlaveDone).value() = false; @@ -215,13 +225,35 @@ public: abortBuildTask(NULL); leaveBuildReadyState(NULL); // leave state if building + clearErrorLogQueue(); + delete m_Master; m_Master = NULL; } } + void clearErrorLogQueue() + { + NLMISC::CSynchronized >::CAccessor errorLogQueue(&m_ErrorLogWaiting); + errorLogQueue.value().clear(); + } + + void sendErrorLogQueue() + { + NLMISC::CSynchronized >::CAccessor errorLogQueue(&m_ErrorLogWaiting); + for (std::vector::iterator it = errorLogQueue.value().begin(), end = errorLogQueue.value().end(); it != end; ++it) + { + const CErrorLogData &erdt = *it; + nlassert(m_Master); + m_Master->slaveLoggedToolError(this, (uint8)erdt.Type, macroPath(erdt.Path), erdt.Time, erdt.Error); + } + errorLogQueue.value().clear(); + } + virtual void onModuleUpdate() { + sendErrorLogQueue(); + if (m_ReloadSheetsState == REQUEST_MADE) { if (PIPELINE::reloadSheets()) @@ -313,6 +345,7 @@ public: { m_SlaveTaskState = SOMEWHERE_INBETWEEN; CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_PLUGIN_WORKING); + sendErrorLogQueue(); if (m_AbortRequested) { nlinfo("Aborted slave task while plugin was working"); @@ -525,6 +558,18 @@ public: /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// + void notifyErrorLog(TError type, const std::string &path, const std::string &time, const std::string &error) + { + // Received an error log from a plugin tool + NLMISC::CSynchronized >::CAccessor errorLogQueue(&m_ErrorLogWaiting); + CErrorLogData erdt; + erdt.Type = type; + erdt.Path = path; + erdt.Time = time; + erdt.Error = error; + errorLogQueue.value().push_back(erdt); + } + class CPluginBuildTask : public IRunnable { public: @@ -537,6 +582,7 @@ public: case PIPELINE::PLUGIN_REGISTERED_CLASS: { PIPELINE::IProcessHandler *processHandler = static_cast(NLMISC::CClassRegistry::create(m_Slave->m_ActiveProcess->m_ActivePlugin.Handler)); + m_Slave->m_ActiveProcess->m_ErrorLogCallback = TErrorLogCallback(m_Slave, &CModulePipelineSlave::notifyErrorLog); processHandler->setPipelineProcess(m_Slave->m_ActiveProcess); m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_SUCCESS; processHandler->build(); @@ -639,6 +685,14 @@ public: void finishedTask(TProcessResult errorLevel, std::string errorMessage) { nlinfo("errorLevel: %i, errorMessage: %s", (uint32)errorLevel, errorMessage.c_str()); + stringstream ss; + ss << "Build: " << m_ActiveProcess->m_StatsBuild << ", Skip: " << m_ActiveProcess->m_StatsSkip << ", Invalid: " << m_ActiveProcess->m_StatsInvalid << ", Total: " + << (m_ActiveProcess->m_StatsBuild + m_ActiveProcess->m_StatsSkip + m_ActiveProcess->m_StatsInvalid); + nlinfo("%s", ss.str().c_str()); + stringstream ss2; + ss2 << NLMISC::CTime::getSecondsSince1970(); + if (m_Master) // TODO: Maybe send this as part of the finished build task message, would be cleaner ;) + m_Master->slaveLoggedToolError(this, (uint8)MESSAGE, "", ss2.str(), ss.str()); clearActiveProcess(); m_SlaveTaskState = IDLE_WAIT_MASTER; if (m_Master) // else was disconnect diff --git a/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp b/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp index 4d473b85a..0ce85f8b3 100644 --- a/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp +++ b/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp @@ -88,6 +88,11 @@ void CPipelineInterfaceImpl::endedDirectCode() PIPELINE::endedDirectTask(); } +bool CPipelineInterfaceImpl::isExiting() +{ + return g_IsExiting; +} + } /* namespace PIPELINE */ /* end of file */ diff --git a/code/nel/tools/pipeline/service/pipeline_interface_impl.h b/code/nel/tools/pipeline/service/pipeline_interface_impl.h index 9a7dbabcb..79fb09823 100644 --- a/code/nel/tools/pipeline/service/pipeline_interface_impl.h +++ b/code/nel/tools/pipeline/service/pipeline_interface_impl.h @@ -61,6 +61,7 @@ public: virtual void endedRunnableTask(); virtual bool tryDirectCode(const std::string &stateName); virtual void endedDirectCode(); + virtual bool isExiting(); }; /* class CPipelineInterfaceImpl */ } /* namespace PIPELINE */ diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl.cpp index b410217e9..1f9fc0e2b 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.cpp @@ -50,7 +50,7 @@ using namespace std; namespace PIPELINE { -CPipelineProcessImpl::CPipelineProcessImpl(CPipelineProject *activeProject) : m_ActiveProject(activeProject), m_SubTaskResult(FINISH_NOT), m_Aborting(false) +CPipelineProcessImpl::CPipelineProcessImpl(CPipelineProject *activeProject) : m_ActiveProject(activeProject), m_SubTaskResult(FINISH_NOT), m_Aborting(false), m_StatsBuild(0), m_StatsSkip(0), m_StatsInvalid(0) { if (activeProject == NULL) { diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.h b/code/nel/tools/pipeline/service/pipeline_process_impl.h index 5bfc2975b..c4ec657c0 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.h +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.h @@ -32,15 +32,19 @@ // STL includes // NeL includes +#include // Project includes #include "../plugin_library/pipeline_process.h" #include "pipeline_workspace.h" #include "metadata_storage.h" +#include "callback.h" namespace PIPELINE { class CPipelineProject; +typedef CCallback TErrorLogCallback; + /** * \brief CPipelineProcessImpl * \date 2012-03-03 09:33GMT @@ -108,6 +112,11 @@ private: std::set m_ListDependentFiles; bool m_Aborting; + TErrorLogCallback m_ErrorLogCallback; + + uint32 m_StatsBuild; + uint32 m_StatsSkip; + uint32 m_StatsInvalid; private: bool getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath); diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp index 9f212a572..242220c40 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp @@ -183,7 +183,10 @@ bool CPipelineProcessImpl::hasFileBeenAddedSince(const std::string &inputFile, u bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) + { + ++m_StatsInvalid; return false; // Cannot continue on previous failure. + } m_SubTaskResult = FINISH_NOT; @@ -195,12 +198,14 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { m_SubTaskResult = FINISH_ERROR; m_SubTaskErrorMessage = std::string("Input file '") + path + "' cannot be a directory"; + ++m_StatsInvalid; return false; // Error, cannot rebuild. } if (!isFileDependency(path)) { m_SubTaskResult = FINISH_ERROR; m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; + ++m_StatsInvalid; return false; // Error, cannot rebuild. } } @@ -233,6 +238,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { nldebug("No output files were tampered with since last successful build, rebuild not needed"); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsSkip; return false; // No rebuild required. } } @@ -262,6 +268,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { nldebug("Not all input files have an .output files, rebuild"); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild } else @@ -277,7 +284,10 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) + { + ++m_StatsInvalid; return false; // Cannot continue on previous failure. + } m_SubTaskResult = FINISH_NOT; @@ -311,6 +321,7 @@ bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector &inputPaths, const std::vector &outputPaths, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) + { + ++m_StatsInvalid; return false; // Cannot continue on previous failure. + } m_SubTaskResult = FINISH_NOT; @@ -348,6 +362,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { m_SubTaskResult = FINISH_ERROR; m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies"; + ++m_StatsInvalid; return false; // Error, cannot rebuild. } } @@ -358,6 +373,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { m_SubTaskResult = FINISH_ERROR; m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; + ++m_StatsInvalid; return false; // Error, cannot rebuild. } } @@ -407,6 +423,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { nldebug("No output files were tampered with since last successful build, rebuild not needed"); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsSkip; return false; // No rebuild required. } else @@ -421,7 +438,10 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) + { + ++m_StatsInvalid; return false; // Cannot continue on previous failure. + } m_SubTaskResult = FINISH_NOT; @@ -434,6 +454,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { m_SubTaskResult = FINISH_ERROR; m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory"; + ++m_StatsInvalid; return false; // Error, cannot rebuild. } } @@ -447,6 +468,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i // If so, rebuild nldebug("Output file '%s' has been removed, rebuild", path.c_str()); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } } @@ -461,6 +483,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i // If so, rebuild nldebug("Output file '%s' does not exist, rebuild", path.c_str()); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } } @@ -498,6 +521,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str()); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } else @@ -513,6 +537,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i // FIXME: Is it really necessary to recalculate the crc32 if the status file is broken for an output file? Useful though for some rare cases. m_SubTaskResult = FINISH_ERROR; m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "', this should never happen at all, coding error"; + ++m_StatsInvalid; return false; // Error, cannot rebuild. } else @@ -521,6 +546,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nlwarning("Status checksum %i for output file '%s' does match depend checksum %i, output file was modified, this should not happen, rebuild", metaStatus.CRC32, path.c_str(), metaDepend.CRC32); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } } @@ -537,6 +563,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nlwarning("Output file '%s' depends on unknown file '%s', rebuild", path.c_str(), dependencyFile.c_str()); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } else @@ -545,6 +572,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nldebug("Status checksum %i for input file '%s' does match depend checksum %i, input file was modified, rebuild", metaStatus.CRC32, dependencyFile.c_str(), dependency.CRC32); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } } @@ -574,6 +602,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nldebug("Found a file added after last build start in a dependency directory that is not known by the depend files, rebuild"); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } } @@ -585,6 +614,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nldebug("Found a dependency file added after last build start that is not known by the depend files, rebuild"); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsBuild; return true; // Rebuild. } } @@ -595,6 +625,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i // (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed) nldebug("No differences found, no rebuild needed"); m_SubTaskResult = FINISH_SUCCESS; + ++m_StatsSkip; return false; // Rebuild not necessary. } diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp index 81493840d..c3c288a55 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp @@ -160,7 +160,8 @@ void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const } std::string path = standardizePath(tabbedLine[1], false); nlwarning("Read error log line: %s, %s, %s, %s", tabbedLine[0].c_str(), path.c_str(), tabbedLine[2].c_str(), tabbedLine[3].c_str()); - // TODO: Notify the master to write to the .errors meta file and update any connected terminals + // Notify the master through the slave to write to the .errors meta file and update any connected terminals + m_ErrorLogCallback(type, path, tabbedLine[2], tabbedLine[3]); if (type == ERROR) ++nbErrors; } diff --git a/code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace b/code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace index 7f5117ee7..4bbd181a0 100644 --- a/code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace +++ b/code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace @@ -3,17 +3,7 @@ - - - - - - - - - - @@ -24,17 +14,16 @@ - Sat Feb 18 13:48:48 2012 (Kaetemi) .Name = Ryzom Core -Sat Feb 18 13:48:48 2012 (Kaetemi) formName Resized = 1 -Sat Feb 18 14:44:25 2012 (Kaetemi) .Description = Ryzom Core -Sat Feb 18 14:44:37 2012 (Kaetemi) .Projects[0] = common_interface.pipeline_project -Sat Feb 18 23:23:17 2012 (Kaetemi) .Description = Ryzom Core Test -Sat Feb 18 23:23:26 2012 (Kaetemi) .Description = Ryzom Core -Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[0] = plugin_nel.pipeline_plugin -Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[1] = plugin_max.pipeline_plugin -Sat Mar 03 10:58:02 2012 (Kaetemi) formName Resized = 2 -Sat Aug 04 21:04:25 2012 (kaetemi) .Projects[1] = interfaces.pipeline_package -Sat Aug 04 21:04:25 2012 (kaetemi) formName Resized = 2 -Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[0] = interfaces.pipeline_package -Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[1] = common_interface.pipeline_project + <ATOM Value="gamedev.pipeline_package"/> + <ATOM Value="data_common.pipeline_package"/> + <ATOM Value="common_interface.pipeline_project"/> + <ATOM Value="interfaces.pipeline_package"/> + <ATOM Value="common_objects_generique.pipeline_project"/> + <ATOM Value="common_objects_fyros.pipeline_project"/> + <ATOM Value="common_objects_tryker.pipeline_project"/> + <ATOM Value="common_objects_matis.pipeline_project"/> + <ATOM Value="common_objects_zorai.pipeline_project"/> + <ATOM Value="common_objects_caravan.pipeline_project"/> + <ATOM Value="objects.pipeline_package"/> +