From 44218214bb7dcf50de368afbc29d22f198059349 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 8 Aug 2012 16:46:37 +0200 Subject: [PATCH] Added: #1440 Experimental tool for finding missing assets refered to in max files --HG-- branch : build_pipeline_v3 --- .../plugin_library/pipeline_interface.h | 4 + .../plugin_max/pipeline_plugin_max.cpp | 326 ++++++++++++++++++ .../pipeline/plugin_max/pipeline_plugin_max.h | 2 + .../service/pipeline_interface_impl.cpp | 5 + .../service/pipeline_interface_impl.h | 1 + 5 files changed, 338 insertions(+) diff --git a/code/nel/tools/pipeline/plugin_library/pipeline_interface.h b/code/nel/tools/pipeline/plugin_library/pipeline_interface.h index 3a4bd8880..b1f34ae8a 100644 --- a/code/nel/tools/pipeline/plugin_library/pipeline_interface.h +++ b/code/nel/tools/pipeline/plugin_library/pipeline_interface.h @@ -60,6 +60,7 @@ public: // ***************** PLUGIN UTILITY FUNCTIONS ***************** // ************* DO NOT USE FROM INSIDE A PROCESS ************* + // ************ USE ONLY FOR SPECIAL TOOL COMMANDS ************ /// Get the configuration file of the pipeline service. Must only be used for configuration values that may be different on different services, such as tool paths. virtual NLMISC::CConfigFile &getConfigFile() = 0; @@ -78,6 +79,9 @@ public: /// Call when a runnable task has ended to reset to STATE_IDLE. virtual void endedDirectCode() = 0; + + /// Check if service is exiting + virtual bool isExiting() = 0; }; /* class IPipelineInterface */ diff --git a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp index 4cc110b05..331512cfe 100644 --- a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp +++ b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp @@ -36,6 +36,9 @@ // NeL includes #include "nel/misc/dynloadlib.h" #include "nel/misc/debug.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/file.h" // Project includes #include "../plugin_library/pipeline_interface.h" @@ -83,6 +86,329 @@ CPipelinePluginMax::~CPipelinePluginMax() }*/ +namespace { + +std::set MissingFiles; + +class CMaxRewritePathsCommand : public NLMISC::IRunnable +{ +public: + NLMISC::CLog *Log; + std::string SrcDirectoryRecursive; + std::string DatabaseDirectory; + + virtual void getName(std::string &result) const + { result = "CMaxRewritePathsCommand"; } + + // COPY FROM PIPELINE_SERVICE.CPP WITH BACKSLASHES INSTEAD OF FORWARD SLASHES + static std::string standardizePath(const std::string &path, bool addFinalSlash) + { + // check empty path + if (path.empty()) + return ""; + + std::string newPath; + newPath.resize(path.size() + 1); + + std::string::size_type j = 0; + for (std::string::size_type i = 0; i < path.size(); ++i) + { + if (path[i] == '\\' || path[i] == '/') + { + if (j <= 1 && path[i] == '\\') + { + // for windows network + newPath[j] = '\\'; + ++j; + } + else if (j == 0 || newPath[j - 1] != '\\') + { + newPath[j] = '\\'; + ++j; + } + } + else + { + newPath[j] = path[i]; + ++j; + } + } + newPath[j] = 0; + newPath.resize(j); + + // add terminal slash + if (addFinalSlash && newPath[newPath.size()-1] != '\\') + newPath += '\\'; + + return newPath; + } + + static inline bool isCharacter(char c) + { + return (32 <= c /*&& c <= 127) || (161 <= c*/ && c <= 255); + } + + static inline char stripFrenchLocale(char c) + { + if (192 <= c && c <= 197) return 'a'; + if (200 <= c && c <= 203) return 'e'; + if (204 <= c && c <= 207) return 'i'; + if (210 <= c && c <= 214) return 'o'; + if (217 <= c && c <= 220) return 'u'; + if (c == 221) return 'y'; + if (224 <= c && c <= 229) return 'a'; + if (232 <= c && c <= 235) return 'e'; + if (236 <= c && c <= 239) return 'i'; + if (242 <= c && c <= 246) return 'o'; + if (249 <= c && c <= 252) return 'u'; + if (c == 253 || c == 255) return 'y'; + return c; + } + + // maxRewritePaths W:/database/interfaces/anims_max + + static std::string rewritePath(const std::string &path, const std::string &databaseDirectory) + { + static std::set fileNameCache; + + std::string stdPath = standardizePath(path, false); + for (std::string::size_type i = 0; i < stdPath.size(); ++i) + stdPath[i] = stripFrenchLocale(stdPath[i]); + stdPath = NLMISC::toLower(stdPath); + + // TODO: remove ./stuff/caravan/agents/_textures/actors/trame.png + + NLMISC::strFindReplace(stdPath, "w:\\database\\", databaseDirectory); + NLMISC::strFindReplace(stdPath, "\\\\amiga\\3d\\database\\", databaseDirectory); + NLMISC::strFindReplace(stdPath, "tr_hof_underwear_torso", "tr_hof_underwear_torse"); + NLMISC::strFindReplace(stdPath, "tr_hof_underwear_hand-", "tr_hof_underwear_hand_"); + NLMISC::strFindReplace(stdPath, "ma_hom_armor_01", "ma_hom_armor01"); + NLMISC::strFindReplace(stdPath, "spec_yeux", "spec_eye_tryker"); + NLMISC::strFindReplace(stdPath, "fy_hof_cheveux_shave01", "fy_hof_cheveux_shave"); + + if (stdPath.find("\\trame.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/lod_actors/texture_lod/trame.png", false); + + if (stdPath.size() > path.size()) + { + nlwarning("Path size becomes too large: '%s' -> '%s'", path.c_str(), stdPath.c_str()); + return path; + } + + if (NLMISC::CFile::getFilename(stdPath) == stdPath) + { + breakable + { + if (fileNameCache.find(stdPath) == fileNameCache.end()) + { + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { + stdPath[stdPath.size() - 3] = 'p'; + stdPath[stdPath.size() - 2] = 'n'; + stdPath[stdPath.size() - 1] = 'g'; + if (fileNameCache.find(stdPath) != fileNameCache.end()) + break; + } + nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str()); + return stdPath; + } + } + return stdPath; + } + else + { + if (stdPath.find(databaseDirectory) == std::string::npos) + { + if (stdPath[1] == ':' || (stdPath[0] == '\\' && stdPath[1] == '\\')) + { + nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str()); + MissingFiles.insert(path); + return stdPath; + } + else + { + // invalid path, don't care too much + return stdPath; + } + } + + breakable + { + if (!NLMISC::CFile::fileExists(stdPath)) + { + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { + stdPath[stdPath.size() - 3] = 'p'; + stdPath[stdPath.size() - 2] = 'n'; + stdPath[stdPath.size() - 1] = 'g'; + if (NLMISC::CFile::fileExists(stdPath)) + break; + } + { + std::string stdPathVv2 = standardizePath(NLMISC::CFile::getPath(stdPath) + "/vv2/" + NLMISC::CFile::getFilename(stdPath), false); + bool vv2works = false; + if (NLMISC::CFile::fileExists(stdPathVv2)) + { + vv2works = true; + } + else + { + if (stdPathVv2[stdPathVv2.size() - 3] == 'p' && stdPathVv2[stdPathVv2.size() - 2] == 'n' && stdPathVv2[stdPathVv2.size() - 1] == 'g') + { + stdPathVv2[stdPathVv2.size() - 3] = 't'; + stdPathVv2[stdPathVv2.size() - 2] = 'g'; + stdPathVv2[stdPathVv2.size() - 1] = 'a'; + if (NLMISC::CFile::fileExists(stdPathVv2)) + { + vv2works = true; + } + } + } + if (vv2works) + { + // try with vv2 + if (stdPathVv2.size() > path.size()) + { + nlwarning("Path with vv2 size becomes too large: '%s' -> '%s'", path.c_str(), stdPathVv2.c_str()); + return stdPath; + } + else + { + stdPath = stdPathVv2; + break; + } + } + } + nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str()); + MissingFiles.insert(path); + return stdPath; + } + } + } + + fileNameCache.insert(NLMISC::CFile::getFilename(stdPath)); + return stdPath; + } + + void doFile(const std::string &filePath) + { + if (filePath[filePath.size() - 3] == 'm' && filePath[filePath.size() - 2] == 'a' && filePath[filePath.size() - 1] == 'x') + { + nldebug("File: '%s'", filePath.c_str()); + + std::vector buffer; + buffer.resize(NLMISC::CFile::getFileSize(filePath)); + + // read + { + NLMISC::CIFile ifile; + ifile.open(filePath, false); + ifile.serialBuffer((uint8 *)(&buffer[0]), buffer.size()); + ifile.close(); + } + + // find paths + for (std::vector::size_type i = 256; i < buffer.size(); ++i) // skip the first 256 lol :) + { + if (((NLMISC::toLower(buffer[i - 1]) == 'x' && NLMISC::toLower(buffer [i - 2]) == 'a' && NLMISC::toLower(buffer[i - 3]) == 'm') + || (NLMISC::toLower(buffer[i - 1]) == 'a' && NLMISC::toLower(buffer [i - 2]) == 'g' && NLMISC::toLower(buffer[i - 3]) == 't') + || (NLMISC::toLower(buffer[i - 1]) == 'g' && NLMISC::toLower(buffer [i - 2]) == 'n' && NLMISC::toLower(buffer[i - 3]) == 'p')) + && (NLMISC::toLower(buffer[i - 4]) == '.')) + { + // buffer[i] is the character after the path! :) + std::vector::size_type beginPath = 0; + for (std::vector::size_type j = i - 4; j > 0; --j) + { + if (!isCharacter(buffer[j])) + { + if (buffer[j + 1] == '\\' && buffer[j + 2] == '\\' || buffer[j] == 0) + { + beginPath = j + 1; + break; + } + // nlwarning("Invalid characters '%i' in path at %i, len %i!", (uint32)buffer[j], (uint32)j, (uint32)(i - j)); + // beginPath = j + 1; // test + break; + } + if (buffer[j] == ':') + { + beginPath = j - 1; + break; + } + } + if (beginPath != 0) + { + std::vector::size_type sizePath = i - beginPath; + std::string foundPath = std::string(&buffer[beginPath], sizePath); //std::string(buffer.at(beginPath), buffer.at(i)); + //nldebug("Found path: '%s' from %i to %i", foundPath.c_str(), (uint32)beginPath, (uint32)i); + std::string fixedPath = rewritePath(foundPath, DatabaseDirectory); + //nldebug("Rewrite to: '%s'", fixedPath.c_str()); + } + } + } + + //NLMISC::CFile::re + + // ... + } + } + + // maxRewritePaths W:/database/interfaces/anims_max + + void doDirectory(const std::string &directoryPath) + { + nldebug("Directory: '%s'", directoryPath.c_str()); + + std::string dirPath = standardizePath(SrcDirectoryRecursive, true); + std::vector dirContents; + + NLMISC::CPath::getPathContent(dirPath, false, true, true, dirContents); + + for (std::vector::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it) + { + const std::string &subPath = *it; + + if (NLMISC::CFile::isDirectory(subPath)) + doDirectory(subPath); + else + doFile(subPath); + + if (PIPELINE::IPipelineInterface::getInstance()->isExiting()) + return; + } + } + + virtual void run() + { + std::string tempDirectory = PIPELINE::IPipelineProcess::getInstance()->getTempDirectory(); + DatabaseDirectory = standardizePath(PIPELINE::IPipelineInterface::getInstance()->getConfigFile().getVar("DatabaseDirectory").asString(0), true); + nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory.c_str()); + + doDirectory(SrcDirectoryRecursive); + + for (std::set::iterator it = MissingFiles.begin(), end = MissingFiles.end(); it != end; ++it) + nlinfo("Missing: '%s'", (*it).c_str()); + + PIPELINE::IPipelineInterface::getInstance()->endedRunnableTask(); + } +}; +CMaxRewritePathsCommand s_MaxRewritePathsCommand; + +} /* anonymous namespace */ + } /* namespace PIPELINE */ +NLMISC_CATEGORISED_COMMAND(max, maxRewritePaths, "Rewrite all paths to .tga, .png and .max files found in a max file", "") +{ + if(args.size() != 1) return false; + PIPELINE::s_MaxRewritePathsCommand.Log = &log; + PIPELINE::s_MaxRewritePathsCommand.SrcDirectoryRecursive = args[0]; + if (!PIPELINE::IPipelineInterface::getInstance()->tryRunnableTask("COMMAND_MAX_REWRITE_PATHS", &PIPELINE::s_MaxRewritePathsCommand)) + { + log.displayNL("Busy."); + return false; + } + return true; +} + /* end of file */ diff --git a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h index dd45afd80..784cdb915 100644 --- a/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h +++ b/code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h @@ -32,6 +32,8 @@ // STL includes // NeL includes +#include +#include // Project includes diff --git a/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp b/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp index 4d473b85a..0ce85f8b3 100644 --- a/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp +++ b/code/nel/tools/pipeline/service/pipeline_interface_impl.cpp @@ -88,6 +88,11 @@ void CPipelineInterfaceImpl::endedDirectCode() PIPELINE::endedDirectTask(); } +bool CPipelineInterfaceImpl::isExiting() +{ + return g_IsExiting; +} + } /* namespace PIPELINE */ /* end of file */ diff --git a/code/nel/tools/pipeline/service/pipeline_interface_impl.h b/code/nel/tools/pipeline/service/pipeline_interface_impl.h index 9a7dbabcb..79fb09823 100644 --- a/code/nel/tools/pipeline/service/pipeline_interface_impl.h +++ b/code/nel/tools/pipeline/service/pipeline_interface_impl.h @@ -61,6 +61,7 @@ public: virtual void endedRunnableTask(); virtual bool tryDirectCode(const std::string &stateName); virtual void endedDirectCode(); + virtual bool isExiting(); }; /* class CPipelineInterfaceImpl */ } /* namespace PIPELINE */