Added: #1440 Experimental tool for finding missing assets refered to in max files

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/plugin_library/pipeline_interface.h

@@ -60,6 +60,7 @@ public:
 
 	// ***************** PLUGIN UTILITY FUNCTIONS *****************
 	// ************* DO NOT USE FROM INSIDE A PROCESS *************
+	// ************ USE ONLY FOR SPECIAL TOOL COMMANDS ************
 
 	/// Get the configuration file of the pipeline service. Must only be used for configuration values that may be different on different services, such as tool paths.
 	virtual NLMISC::CConfigFile &getConfigFile() = 0;
@@ -78,6 +79,9 @@ public:
 
 	/// Call when a runnable task has ended to reset to STATE_IDLE.
 	virtual void endedDirectCode() = 0;
+
+	/// Check if service is exiting
+	virtual bool isExiting() = 0;
 	
 }; /* class IPipelineInterface */
 

code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp

@@ -36,6 +36,9 @@
 // NeL includes
 #include "nel/misc/dynloadlib.h"
 #include "nel/misc/debug.h"
+#include "nel/misc/path.h"
+#include "nel/misc/algo.h"
+#include "nel/misc/file.h"
 
 // Project includes
 #include "../plugin_library/pipeline_interface.h"
@@ -83,6 +86,329 @@ CPipelinePluginMax::~CPipelinePluginMax()
 	
 }*/
 
+namespace {
+
+std::set<std::string> MissingFiles;
+
+class CMaxRewritePathsCommand : public NLMISC::IRunnable
+{
+public:
+	NLMISC::CLog *Log;
+	std::string SrcDirectoryRecursive;
+	std::string DatabaseDirectory;
+
+	virtual void getName(std::string &result) const 
+	{ result = "CMaxRewritePathsCommand"; }
+
+	// COPY FROM PIPELINE_SERVICE.CPP WITH BACKSLASHES INSTEAD OF FORWARD SLASHES
+	static std::string standardizePath(const std::string &path, bool addFinalSlash)
+	{
+		// check empty path
+		if (path.empty())
+			return "";
+
+		std::string newPath;
+		newPath.resize(path.size() + 1);
+
+		std::string::size_type j = 0;
+		for (std::string::size_type i = 0; i < path.size(); ++i)
+		{
+			if (path[i] == '\\' || path[i] == '/')
+			{
+				if (j <= 1 && path[i] == '\\')
+				{
+					// for windows network
+					newPath[j] = '\\';
+					++j;
+				}
+				else if (j == 0 || newPath[j - 1] != '\\')
+				{
+					newPath[j] = '\\';
+					++j;
+				}
+			}
+			else
+			{
+				newPath[j] = path[i];
+				++j;
+			}
+		}
+		newPath[j] = 0;
+		newPath.resize(j);
+
+		// add terminal slash
+		if (addFinalSlash && newPath[newPath.size()-1] != '\\')
+			newPath += '\\';
+
+		return newPath;
+	}
+
+	static inline bool isCharacter(char c)
+	{
+		return (32 <= c /*&& c <= 127) || (161 <= c*/ && c <= 255);
+	}
+
+	static inline char stripFrenchLocale(char c)
+	{
+		if (192 <= c && c <= 197) return 'a';
+		if (200 <= c && c <= 203) return 'e';
+		if (204 <= c && c <= 207) return 'i';
+		if (210 <= c && c <= 214) return 'o';
+		if (217 <= c && c <= 220) return 'u';
+		if (c == 221) return 'y';
+		if (224 <= c && c <= 229) return 'a';
+		if (232 <= c && c <= 235) return 'e';
+		if (236 <= c && c <= 239) return 'i';
+		if (242 <= c && c <= 246) return 'o';
+		if (249 <= c && c <= 252) return 'u';
+		if (c == 253 || c == 255) return 'y';
+		return c;
+	}
+
+	// maxRewritePaths W:/database/interfaces/anims_max
+
+	static std::string rewritePath(const std::string &path, const std::string &databaseDirectory)
+	{
+		static std::set<std::string> fileNameCache;
+
+		std::string stdPath = standardizePath(path, false);
+		for (std::string::size_type i = 0; i < stdPath.size(); ++i)
+			stdPath[i] = stripFrenchLocale(stdPath[i]);
+		stdPath = NLMISC::toLower(stdPath);
+
+		// TODO: remove ./stuff/caravan/agents/_textures/actors/trame.png
+
+		NLMISC::strFindReplace(stdPath, "w:\\database\\", databaseDirectory);
+		NLMISC::strFindReplace(stdPath, "\\\\amiga\\3d\\database\\", databaseDirectory);
+		NLMISC::strFindReplace(stdPath, "tr_hof_underwear_torso", "tr_hof_underwear_torse");
+		NLMISC::strFindReplace(stdPath, "tr_hof_underwear_hand-", "tr_hof_underwear_hand_");
+		NLMISC::strFindReplace(stdPath, "ma_hom_armor_01", "ma_hom_armor01");
+		NLMISC::strFindReplace(stdPath, "spec_yeux", "spec_eye_tryker");
+		NLMISC::strFindReplace(stdPath, "fy_hof_cheveux_shave01", "fy_hof_cheveux_shave");
+
+		if (stdPath.find("\\trame.") != std::string::npos)
+			stdPath = standardizePath(databaseDirectory + "/stuff/lod_actors/texture_lod/trame.png", false);
+
+		if (stdPath.size() > path.size())
+		{
+			nlwarning("Path size becomes too large: '%s' -> '%s'", path.c_str(), stdPath.c_str());
+			return path;
+		}
+
+		if (NLMISC::CFile::getFilename(stdPath) == stdPath)
+		{
+			breakable
+			{
+				if (fileNameCache.find(stdPath) == fileNameCache.end())
+				{
+					if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a')
+					{
+						stdPath[stdPath.size() - 3] = 'p';
+						stdPath[stdPath.size() - 2] = 'n';
+						stdPath[stdPath.size() - 1] = 'g';
+						if (fileNameCache.find(stdPath) != fileNameCache.end())
+							break;
+					}
+					nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str());
+					return stdPath;
+				}
+			}
+			return stdPath;
+		}
+		else
+		{
+			if (stdPath.find(databaseDirectory) == std::string::npos)
+			{
+				if (stdPath[1] == ':' || (stdPath[0] == '\\' && stdPath[1] == '\\'))
+				{
+					nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str());
+					MissingFiles.insert(path);
+					return stdPath;
+				}
+				else
+				{
+					// invalid path, don't care too much
+					return stdPath;
+				}
+			}
+
+			breakable
+			{
+				if (!NLMISC::CFile::fileExists(stdPath))
+				{
+					if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a')
+					{
+						stdPath[stdPath.size() - 3] = 'p';
+						stdPath[stdPath.size() - 2] = 'n';
+						stdPath[stdPath.size() - 1] = 'g';
+						if (NLMISC::CFile::fileExists(stdPath))
+							break;
+					}
+					{
+						std::string stdPathVv2 = standardizePath(NLMISC::CFile::getPath(stdPath) + "/vv2/" + NLMISC::CFile::getFilename(stdPath), false);
+						bool vv2works = false;
+						if (NLMISC::CFile::fileExists(stdPathVv2))
+						{
+							vv2works = true;
+						}
+						else
+						{
+							if (stdPathVv2[stdPathVv2.size() - 3] == 'p' && stdPathVv2[stdPathVv2.size() - 2] == 'n' && stdPathVv2[stdPathVv2.size() - 1] == 'g')
+							{
+								stdPathVv2[stdPathVv2.size() - 3] = 't';
+								stdPathVv2[stdPathVv2.size() - 2] = 'g';
+								stdPathVv2[stdPathVv2.size() - 1] = 'a';
+								if (NLMISC::CFile::fileExists(stdPathVv2))
+								{
+									vv2works = true;
+								}
+							}
+						}
+						if (vv2works)
+						{
+							// try with vv2
+							if (stdPathVv2.size() > path.size())
+							{
+								nlwarning("Path with vv2 size becomes too large: '%s' -> '%s'", path.c_str(), stdPathVv2.c_str());
+								return stdPath;
+							}
+							else
+							{
+								stdPath = stdPathVv2;
+								break;
+							}
+						}
+					}
+					nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str());
+					MissingFiles.insert(path);
+					return stdPath;
+				}
+			}
+		}
+
+		fileNameCache.insert(NLMISC::CFile::getFilename(stdPath));
+		return stdPath;
+	}
+
+	void doFile(const std::string &filePath)
+	{
+		if (filePath[filePath.size() - 3] == 'm' && filePath[filePath.size() - 2] == 'a' && filePath[filePath.size() - 1] == 'x')
+		{
+			nldebug("File: '%s'", filePath.c_str());
+
+			std::vector<char> buffer;
+			buffer.resize(NLMISC::CFile::getFileSize(filePath));
+
+			// read
+			{
+				NLMISC::CIFile ifile;
+				ifile.open(filePath, false);
+				ifile.serialBuffer((uint8 *)(&buffer[0]), buffer.size());
+				ifile.close();
+			}
+
+			// find paths
+			for (std::vector<char>::size_type i = 256; i < buffer.size(); ++i) // skip the first 256 lol :)
+			{
+				if (((NLMISC::toLower(buffer[i - 1]) == 'x' && NLMISC::toLower(buffer [i - 2]) == 'a' && NLMISC::toLower(buffer[i - 3]) == 'm')
+					|| (NLMISC::toLower(buffer[i - 1]) == 'a' && NLMISC::toLower(buffer [i - 2]) == 'g' && NLMISC::toLower(buffer[i - 3]) == 't')
+					|| (NLMISC::toLower(buffer[i - 1]) == 'g' && NLMISC::toLower(buffer [i - 2]) == 'n' && NLMISC::toLower(buffer[i - 3]) == 'p'))
+					&& (NLMISC::toLower(buffer[i - 4]) == '.'))
+				{
+					// buffer[i] is the character after the path! :)
+					std::vector<char>::size_type beginPath = 0;
+					for (std::vector<char>::size_type j = i - 4; j > 0; --j)
+					{
+						if (!isCharacter(buffer[j]))
+						{
+							if (buffer[j + 1] == '\\' && buffer[j + 2] == '\\' || buffer[j] == 0)
+							{
+								beginPath = j + 1;
+								break;
+							}
+							// nlwarning("Invalid characters '%i' in path at %i, len %i!", (uint32)buffer[j], (uint32)j, (uint32)(i - j));
+							// beginPath = j + 1; // test
+							break;
+						}
+						if (buffer[j] == ':')
+						{
+							beginPath = j - 1;
+							break;
+						}
+					}
+					if (beginPath != 0)
+					{
+						std::vector<char>::size_type sizePath = i - beginPath;
+						std::string foundPath = std::string(&buffer[beginPath], sizePath); //std::string(buffer.at(beginPath), buffer.at(i));
+						//nldebug("Found path: '%s' from %i to %i", foundPath.c_str(), (uint32)beginPath, (uint32)i);
+						std::string fixedPath = rewritePath(foundPath, DatabaseDirectory);
+						//nldebug("Rewrite to: '%s'", fixedPath.c_str());
+					}
+				}
+			}
+
+			//NLMISC::CFile::re
+
+			// ...
+		}
+	}
+
+	// maxRewritePaths W:/database/interfaces/anims_max
+
+	void doDirectory(const std::string &directoryPath)
+	{
+		nldebug("Directory: '%s'", directoryPath.c_str());
+
+		std::string dirPath = standardizePath(SrcDirectoryRecursive, true);
+		std::vector<std::string> dirContents;
+
+		NLMISC::CPath::getPathContent(dirPath, false, true, true, dirContents);
+		
+		for (std::vector<std::string>::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it)
+		{
+			const std::string &subPath = *it;
+
+			if (NLMISC::CFile::isDirectory(subPath))
+				doDirectory(subPath);
+			else
+				doFile(subPath);
+			
+			if (PIPELINE::IPipelineInterface::getInstance()->isExiting())
+				return;
+		}
+	}
+
+	virtual void run()
+	{
+		std::string tempDirectory = PIPELINE::IPipelineProcess::getInstance()->getTempDirectory();
+		DatabaseDirectory = standardizePath(PIPELINE::IPipelineInterface::getInstance()->getConfigFile().getVar("DatabaseDirectory").asString(0), true);
+		nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory.c_str());
+
+		doDirectory(SrcDirectoryRecursive);
+
+		for (std::set<std::string>::iterator it = MissingFiles.begin(), end = MissingFiles.end(); it != end; ++it)
+			nlinfo("Missing: '%s'", (*it).c_str());
+
+		PIPELINE::IPipelineInterface::getInstance()->endedRunnableTask();
+	}
+};
+CMaxRewritePathsCommand s_MaxRewritePathsCommand;
+
+} /* anonymous namespace */
+
 } /* namespace PIPELINE */
 
+NLMISC_CATEGORISED_COMMAND(max, maxRewritePaths, "Rewrite all paths to .tga, .png and .max files found in a max file", "<srcDirectoryRecursive>")
+{
+	if(args.size() != 1) return false;
+	PIPELINE::s_MaxRewritePathsCommand.Log = &log;
+	PIPELINE::s_MaxRewritePathsCommand.SrcDirectoryRecursive = args[0];
+	if (!PIPELINE::IPipelineInterface::getInstance()->tryRunnableTask("COMMAND_MAX_REWRITE_PATHS", &PIPELINE::s_MaxRewritePathsCommand))
+	{
+		log.displayNL("Busy.");
+		return false;
+	}
+	return true;
+}
+
 /* end of file */

code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h

@@ -32,6 +32,8 @@
 // STL includes
 
 // NeL includes
+#include <nel/misc/command.h>
+#include <nel/misc/task_manager.h>
 
 // Project includes
 

code/nel/tools/pipeline/service/pipeline_interface_impl.cpp

@@ -88,6 +88,11 @@ void CPipelineInterfaceImpl::endedDirectCode()
 	PIPELINE::endedDirectTask();
 }
 
+bool CPipelineInterfaceImpl::isExiting()
+{
+	return g_IsExiting;
+}
+
 } /* namespace PIPELINE */
 
 /* end of file */

code/nel/tools/pipeline/service/pipeline_interface_impl.h

@@ -61,6 +61,7 @@ public:
 	virtual void endedRunnableTask();
 	virtual bool tryDirectCode(const std::string &stateName);
 	virtual void endedDirectCode();
+	virtual bool isExiting();
 }; /* class CPipelineInterfaceImpl */
 
 } /* namespace PIPELINE */