From 7a8b6fc1f86715b86607605a62271914a9c51842 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Mon, 27 Aug 2012 18:07:01 +0200 Subject: [PATCH] Added: #1440 File map initialization for asset path rewriting tool --HG-- branch : build_pipeline_v3 --- .../pipeline/max_rewrite_assets/main.cpp | 335 +++++++++++++++++- 1 file changed, 334 insertions(+), 1 deletion(-) diff --git a/code/nel/tools/pipeline/max_rewrite_assets/main.cpp b/code/nel/tools/pipeline/max_rewrite_assets/main.cpp index 2a36de4e4..13a27e651 100644 --- a/code/nel/tools/pipeline/max_rewrite_assets/main.cpp +++ b/code/nel/tools/pipeline/max_rewrite_assets/main.cpp @@ -22,6 +22,11 @@ #include #include +#include "nel/misc/dynloadlib.h" +#include "nel/misc/debug.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/file.h" #include "../max/storage_stream.h" #include "../max/storage_object.h" @@ -56,6 +61,333 @@ bool DisplayStream = false; bool WriteModified = false; bool WriteDummy = true; +const char *DatabaseDirectory = "w:\\database\\"; +const char *LinuxDatabaseDirectory = "/srv/work/database/"; +bool RunningLinux = true; + +const char *SrcDirectoryRecursive = "w:\\database\\interfaces\\"; + +std::set MissingFiles; +std::map KnownFileCache; + +std::string nativeDatabasePath(const std::string &standardizedPath) +{ + if (RunningLinux) + { + std::string result = standardizedPath; + NLMISC::strFindReplace(result, DatabaseDirectory, LinuxDatabaseDirectory); + while (NLMISC::strFindReplace(result, "\\", "/")) { } + return result; + } + return standardizedPath; +} + +std::string unnativeDatabasePath(const std::string &nativizedPath) +{ + if (RunningLinux) + { + std::string result = nativizedPath; + NLMISC::strFindReplace(result, LinuxDatabaseDirectory, DatabaseDirectory); + while (NLMISC::strFindReplace(result, "/", "\\")) { } + return result; + } + return nativizedPath; +} + +// COPY FROM PIPELINE_SERVICE.CPP WITH BACKSLASHES INSTEAD OF FORWARD SLASHES +std::string standardizePath(const std::string &path, bool addFinalSlash) +{ + // check empty path + if (path.empty()) + return ""; + + std::string newPath; + newPath.resize(path.size() + 1); + + std::string::size_type j = 0; + for (std::string::size_type i = 0; i < path.size(); ++i) + { + if (path[i] == '\\' || path[i] == '/') + { + if (j <= 1 && path[i] == '\\') + { + // for windows network + newPath[j] = '\\'; + ++j; + } + else if (j == 0 || newPath[j - 1] != '\\') + { + newPath[j] = '\\'; + ++j; + } + } + else + { + newPath[j] = path[i]; + ++j; + } + } + newPath[j] = 0; + newPath.resize(j); + + // add terminal slash + if (addFinalSlash && newPath[newPath.size()-1] != '\\') + newPath += '\\'; + + return newPath; +} + +inline bool isCharacter(uint8 c) +{ + return (32 <= c /*&& c <= 127) || (161 <= c*/ /*&& c <= 255*/); +} + +inline uint8 stripFrenchLocale(uint8 c) +{ + if (192 <= c && c <= 197) return 'a'; + if (200 <= c && c <= 203) return 'e'; + if (204 <= c && c <= 207) return 'i'; + if (210 <= c && c <= 214) return 'o'; + if (217 <= c && c <= 220) return 'u'; + if (c == 221) return 'y'; + if (224 <= c && c <= 229) return 'a'; + if (232 <= c && c <= 235) return 'e'; + if (236 <= c && c <= 239) return 'i'; + if (242 <= c && c <= 246) return 'o'; + if (249 <= c && c <= 252) return 'u'; + if (c == 253 || c == 255) return 'y'; + return c; +} + +std::string rewritePath(const std::string &path, const std::string &databaseDirectory) +{ + static std::set fileNameCache; + + std::string stdPath = standardizePath(path, false); + for (std::string::size_type i = 0; i < stdPath.size(); ++i) + stdPath[i] = stripFrenchLocale(stdPath[i]); + stdPath = NLMISC::toLower(stdPath); + + // TODO: remove ./stuff/caravan/agents/_textures/actors/trame.png + + NLMISC::strFindReplace(stdPath, "w:\\database\\", databaseDirectory); + NLMISC::strFindReplace(stdPath, "\\\\amiga\\3d\\database\\", databaseDirectory); + NLMISC::strFindReplace(stdPath, "ma_hom_armor_01", "ma_hom_armor01"); + NLMISC::strFindReplace(stdPath, "ma_hof_armor_01", "ma_hof_armor01"); + NLMISC::strFindReplace(stdPath, "ma_hom_armor_00", "ma_hom_armor00"); + NLMISC::strFindReplace(stdPath, "ma_hof_armor_00", "ma_hof_armor00"); + NLMISC::strFindReplace(stdPath, "zo_hom_armor_00", "zo_hom_armor00"); + NLMISC::strFindReplace(stdPath, "zo_hof_armor_00", "zo_hof_armor00"); + NLMISC::strFindReplace(stdPath, "fy_hof_cheveux_shave01", "fy_hof_cheveux_shave"); + + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_avtbras", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_avtbras"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_epaule", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_epaule"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_hand-downside", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_hand-downside"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_hand-upside", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_hand-upside"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_molet", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_molet"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hof_underwear_hand-downside", "tryker\\agents\\_textures\\actors\\tr_hof_underwear_hand-downside"); + NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hof_underwear_hand-upside", "tryker\\agents\\_textures\\actors\\tr_hof_underwear_hand-upside"); + + NLMISC::strFindReplace(stdPath, "tr_hof_underwear_torso", "tr_hof_underwear_torse"); + NLMISC::strFindReplace(stdPath, "tr_hof_underwear_hand-", "tr_hof_underwear_hand_"); + + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_avbras.", "fy_hom_armor00_avbras_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_bottes.", "fy_hom_armor00_bottes_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_bras.", "fy_hom_armor00_bras_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_cuissear.", "fy_hom_armor00_cuissear_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_cuisseav.", "fy_hom_armor00_cuisseav_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_dos.", "fy_hom_armor00_dos_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_hand-downside.", "fy_hom_armor00_hand-downside_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_hand-upside.", "fy_hom_armor00_hand-upside_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_pieds.", "fy_hom_armor00_pieds_c1."); + NLMISC::strFindReplace(stdPath, "fy_hom_armor00_torse.", "fy_hom_armor00_torse_c1."); + + NLMISC::strFindReplace(stdPath, "interfaces_visage", "visage_interface"); + + if (stdPath.find("\\trame.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/lod_actors/texture_lod/trame.png", false); + if (stdPath.find("\\tr_hof_visage_c1.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c1.png", false); + if (stdPath.find("\\tr_hof_visage_c2.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c2.png", false); + if (stdPath.find("\\tr_hof_visage_c3.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c3.png", false); + if (stdPath.find("\\ma_hof_cheveux_medium02.") != std::string::npos) + stdPath = standardizePath(databaseDirectory + "/stuff/matis/agents/_textures/actors/ma_hof_cheveux_medium02.png", false); + + /*if (stdPath.size() > path.size()) + { + nlwarning("Path size becomes too large: '%s' -> '%s'", path.c_str(), stdPath.c_str()); + return path; + }*/ + + if (NLMISC::CFile::getFilename(stdPath) == stdPath) + { + breakable + { + if (fileNameCache.find(stdPath) == fileNameCache.end() || KnownFileCache.find(stdPath) == KnownFileCache.end()) + { + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { + stdPath[stdPath.size() - 3] = 'p'; + stdPath[stdPath.size() - 2] = 'n'; + stdPath[stdPath.size() - 1] = 'g'; + if (fileNameCache.find(stdPath) != fileNameCache.end()) + break; + } + nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str()); + return stdPath; + } + } + return stdPath; + } + else + { + if (stdPath.find(databaseDirectory) == std::string::npos) + { + if (stdPath[1] == ':' || (stdPath[0] == '\\' && stdPath[1] == '\\')) + { + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { stdPath[stdPath.size() - 3] = 'p'; stdPath[stdPath.size() - 2] = 'n'; stdPath[stdPath.size() - 1] = 'g'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 'p' && stdPath[stdPath.size() - 2] == 'n' && stdPath[stdPath.size() - 1] == 'g') + { stdPath[stdPath.size() - 3] = 't'; stdPath[stdPath.size() - 2] = 'g'; stdPath[stdPath.size() - 1] = 'a'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str()); + MissingFiles.insert(path); + return stdPath; + } + else + { + // invalid path, don't care too much + return stdPath; + } + } + + breakable + { + if (!NLMISC::CFile::fileExists(nativeDatabasePath(stdPath))) + { + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { + stdPath[stdPath.size() - 3] = 'p'; + stdPath[stdPath.size() - 2] = 'n'; + stdPath[stdPath.size() - 1] = 'g'; + if (NLMISC::CFile::fileExists(nativeDatabasePath(stdPath))) + break; + } + { + std::string stdPathVv2 = standardizePath(NLMISC::CFile::getPath(stdPath) + "/vv2/" + NLMISC::CFile::getFilename(stdPath), false); + bool vv2works = false; + if (NLMISC::CFile::fileExists(nativeDatabasePath(stdPathVv2))) + { + vv2works = true; + } + else + { + if (stdPathVv2[stdPathVv2.size() - 3] == 'p' && stdPathVv2[stdPathVv2.size() - 2] == 'n' && stdPathVv2[stdPathVv2.size() - 1] == 'g') + { + stdPathVv2[stdPathVv2.size() - 3] = 't'; + stdPathVv2[stdPathVv2.size() - 2] = 'g'; + stdPathVv2[stdPathVv2.size() - 1] = 'a'; + if (NLMISC::CFile::fileExists(nativeDatabasePath(stdPathVv2))) + { + vv2works = true; + } + } + } + if (vv2works) + { + // try with vv2 + /*if (stdPathVv2.size() > path.size()) + { + nlwarning("Path with vv2 size becomes too large: '%s' -> '%s'", path.c_str(), stdPathVv2.c_str()); + return stdPath; + } + else + {*/ + stdPath = stdPathVv2; + break; + //} + } + } + // try find + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a') + { stdPath[stdPath.size() - 3] = 'p'; stdPath[stdPath.size() - 2] = 'n'; stdPath[stdPath.size() - 1] = 'g'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + if (stdPath[stdPath.size() - 3] == 'p' && stdPath[stdPath.size() - 2] == 'n' && stdPath[stdPath.size() - 1] == 'g') + { stdPath[stdPath.size() - 3] = 't'; stdPath[stdPath.size() - 2] = 'g'; stdPath[stdPath.size() - 1] = 'a'; } + if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) + return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; + + nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str()); + MissingFiles.insert(path); + return stdPath; + } + } + } + + fileNameCache.insert(NLMISC::CFile::getFilename(stdPath)); + return stdPath; +} + +void doFileInitialize(const std::string &filePath) +{ + // nldebug("File: '%s'", filePath.c_str()); + // nldebug("Native: '%s'", nativeDatabasePath(filePath).c_str()); + + KnownFileCache[NLMISC::CFile::getFilename(filePath)] = standardizePath(filePath, true); +} + +// maxRewritePaths W:/database/interfaces/anims_max + +void doDirectoryInitialize(const std::string &directoryPath) +{ + nldebug("Directory: '%s'", directoryPath.c_str()); + + std::string dirPath = standardizePath(directoryPath, true); + std::vector dirContents; + + NLMISC::CPath::getPathContent(nativeDatabasePath(dirPath), false, true, true, dirContents); + + // nldebug("Native: '%s'", nativeDatabasePath(dirPath).c_str()); + // nldebug("Contents: %i", dirContents.size()); + + for (std::vector::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it) + { + std::string subPath = standardizePath(unnativeDatabasePath(*it), false); + // nldebug("Path: '%s'", subPath.c_str()); + // nldebug("Native: '%s'", nativeDatabasePath(subPath + " ").c_str()); + + if (NLMISC::CFile::isDirectory(nativeDatabasePath(subPath))) + { + if (subPath.find("\\.") == std::string::npos) + doDirectoryInitialize(subPath); + } + else + doFileInitialize(subPath); + } +} + +void runInitialize() +{ + nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory); + + doDirectoryInitialize(std::string(SrcDirectoryRecursive)); +} + void serializeStorageContainer(PIPELINE::MAX::CStorageContainer *storageContainer, GsfInfile *infile, const char *streamName) { GsfInput *input = gsf_infile_child_by_name(infile, streamName); @@ -239,7 +571,8 @@ int main(int argc, char **argv) CUpdate1::registerClasses(&SceneClassRegistry); CEPoly::registerClasses(&SceneClassRegistry); - handleFile("/srv/work/database/interfaces/anims_max/cp_fy_hof_species.max"); + //handleFile("/srv/work/database/interfaces/anims_max/cp_fy_hof_species.max"); + runInitialize(); gsf_shutdown();