Ryzom
/
ryzom-core
1
1
Fork 0
Code Issues 38 Pull Requests Projects 1 Releases 1 Activity

Added: #1440 Rewrite chunks containing asset paths

Browse Source 
--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 12 years ago
parent 5361e3cc3d
commit 8d91f412b4
1 changed files with 254 additions and 17 deletions
Show Stats Download Patch File Download Diff File

271
code/nel/tools/pipeline/max_rewrite_assets/main.cpp
Unescape Escape View File

@ -27,6 +27,7 @@
#include "nel/misc/path.h" #include "nel/misc/path.h"
#include "nel/misc/algo.h" #include "nel/misc/algo.h"
#include "nel/misc/file.h" #include "nel/misc/file.h"
#include "nel/misc/mem_stream.h"
#include "../max/storage_stream.h" #include "../max/storage_stream.h"
#include "../max/storage_object.h" #include "../max/storage_object.h"
@ -49,6 +50,10 @@
#include "../max/update1/editable_mesh.h" #include "../max/update1/editable_mesh.h"
#include "../max/epoly/editable_poly.h" #include "../max/epoly/editable_poly.h"
#include <boost/algorithm/string.hpp>
using namespace std;
using namespace boost::algorithm;
using namespace PIPELINE::MAX; using namespace PIPELINE::MAX;
using namespace PIPELINE::MAX::BUILTIN; using namespace PIPELINE::MAX::BUILTIN;
using namespace PIPELINE::MAX::BUILTIN::STORAGE; using namespace PIPELINE::MAX::BUILTIN::STORAGE;
@ -70,9 +75,10 @@ const char *LinuxDatabaseDirectory = "/srv/work/database/";
bool RunningLinux = true; bool RunningLinux = true;
//const char *SrcDirectoryRecursive = "w:\\database\\interfaces\\"; //const char *SrcDirectoryRecursive = "w:\\database\\interfaces\\";
const char *SrcDirectoryRecursive = "w:\\database\\"; //const char *SrcDirectoryRecursive = "w:\\database\\";
const char *SrcDirectoryRecursive = "w:\\database\\stuff\\fyros\\city\\newpositionville\\";
const char *FallbackTga = "w:\\database\\todo\\todo.tga"; const char *FallbackTga = "w:\\database\\stuff\\generique\\agents\\_textures\\accessories\\lost_texture.tga";
std::set<std::string> MissingFiles; std::set<std::string> MissingFiles;
std::map<std::string, std::string> KnownFileCache; std::map<std::string, std::string> KnownFileCache;
@ -144,13 +150,15 @@ std::string standardizePath(const std::string &path, bool addFinalSlash)
return newPath; return newPath;
} }
inline bool isCharacter(uint8 c) inline bool isCharacter(char c0)
{ {
uint8 c = *(uint8 *)(void *)(&c0);
return (32 <= c /*&& c <= 127) || (161 <= c*/ /*&& c <= 255*/); return (32 <= c /*&& c <= 127) || (161 <= c*/ /*&& c <= 255*/);
} }
inline uint8 stripFrenchLocale(uint8 c) inline char stripFrenchLocale(char c0)
{ {
uint8 c = *(uint8 *)(void *)(&c0);
if (192 <= c && c <= 197) return 'a'; if (192 <= c && c <= 197) return 'a';
if (200 <= c && c <= 203) return 'e'; if (200 <= c && c <= 203) return 'e';
if (204 <= c && c <= 207) return 'i'; if (204 <= c && c <= 207) return 'i';
@ -163,7 +171,7 @@ inline uint8 stripFrenchLocale(uint8 c)
if (242 <= c && c <= 246) return 'o'; if (242 <= c && c <= 246) return 'o';
if (249 <= c && c <= 252) return 'u'; if (249 <= c && c <= 252) return 'u';
if (c == 253 || c == 255) return 'y'; if (c == 253 || c == 255) return 'y';
return c; return c0;
} }
std::string rewritePath(const std::string &path, const std::string &databaseDirectory) std::string rewritePath(const std::string &path, const std::string &databaseDirectory)
@ -242,7 +250,7 @@ std::string rewritePath(const std::string &path, const std::string &databaseDire
if (fileNameCache.find(stdPath) != fileNameCache.end()) if (fileNameCache.find(stdPath) != fileNameCache.end())
break; break;
} }
nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str()); //#nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str());
// MissingFiles.insert(path); // MissingFiles.insert(path);
return stdPath; return stdPath;
} }
@ -268,7 +276,7 @@ std::string rewritePath(const std::string &path, const std::string &databaseDire
if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str()); //#nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str());
MissingFiles.insert(path); MissingFiles.insert(path);
return stdPath; return stdPath;
} }
@ -340,7 +348,7 @@ std::string rewritePath(const std::string &path, const std::string &databaseDire
if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end()) if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
return KnownFileCache[NLMISC::CFile::getFilename(stdPath)]; return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str()); //#nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str());
MissingFiles.insert(path); MissingFiles.insert(path);
return stdPath; return stdPath;
} }
@ -602,24 +610,45 @@ void serializeRaw(std::vector<uint8> &rawOutput, GsfOutfile *outfile, const char
g_object_unref(G_OBJECT(output)); g_object_unref(G_OBJECT(output));
} }
std::string cleanString(const std::string &str)
{
std::string res = str;
trim(res);
// \\Amiga\3D (10 chars)
if (res.size() > 10)
{
if (res.substr(res.size() - 10) == "\\\\Amiga\\3D")
res = res.substr(0, res.size() - 10);
}
if (res.size() > 1)
{
if (res.substr(res.size() - 1) == "0")
res = res.substr(0, res.size() - 1);
}
return res;
}
std::string rewritePathFinal(const std::string &str) std::string rewritePathFinal(const std::string &str)
{ {
std::string result = rewritePath(str, DatabaseDirectory); std::string strtrimmed = cleanString(str);
std::string result = rewritePath(strtrimmed, DatabaseDirectory);
if (NLMISC::CFile::getFilename(result) != result && !NLMISC::CFile::fileExists(nativeDatabasePath(result)) && if (NLMISC::CFile::getFilename(result) != result && !NLMISC::CFile::fileExists(nativeDatabasePath(result)) &&
((result[result.size() - 3] == 't' && result[result.size() - 2] == 'g' && result[result.size() - 1] == 'a') || (result[result.size() - 3] == 'p' && result[result.size() - 2] == 'n' && result[result.size() - 1] == 'g')) ((result[result.size() - 3] == 't' && result[result.size() - 2] == 'g' && result[result.size() - 1] == 'a') || (result[result.size() - 3] == 'p' && result[result.size() - 2] == 'n' && result[result.size() - 1] == 'g'))
) )
{ {
nlwarning("Replacing missing '%s' with '%s'", result.c_str(), FallbackTga); // nlwarning("Replacing missing '%s' with '%s'", result.c_str(), FallbackTga);
return FallbackTga; return FallbackTga;
} }
// nldebug("Replacing '%s' with '%s'", str.c_str(), result.c_str());
return result; return result;
} }
bool isImportantFilePath(const std::string &str) bool isImportantFilePath(const std::string &str)
{ {
if (str.size() > 4) std::string strtrimmed = cleanString(str);
if (strtrimmed.size() >= 4)
{ {
std::string strlw = NLMISC::toLower(str); std::string strlw = NLMISC::toLower(strtrimmed);
return (strlw[strlw.size() - 3] == 'm' && strlw[strlw.size() - 2] == 'a' && strlw[strlw.size() - 1] == 'x') return (strlw[strlw.size() - 3] == 'm' && strlw[strlw.size() - 2] == 'a' && strlw[strlw.size() - 1] == 'x')
|| (strlw[strlw.size() - 3] == 't' && strlw[strlw.size() - 2] == 'g' && strlw[strlw.size() - 1] == 'a') || (strlw[strlw.size() - 3] == 't' && strlw[strlw.size() - 2] == 'g' && strlw[strlw.size() - 1] == 'a')
|| (strlw[strlw.size() - 3] == 'p' && strlw[strlw.size() - 2] == 'n' && strlw[strlw.size() - 1] == 'g'); || (strlw[strlw.size() - 3] == 'p' && strlw[strlw.size() - 2] == 'n' && strlw[strlw.size() - 1] == 'g');
@ -629,9 +658,48 @@ bool isImportantFilePath(const std::string &str)
bool hasImportantFilePath(CStorageRaw *raw) bool hasImportantFilePath(CStorageRaw *raw)
{ {
if (raw->Value.size() > 8) if (raw->Value.size() >= 4)
{ {
// Find any occurences of .max, .png or .tga in ascii or utf16 // Find any occurences of .max, .png or .tga in ascii or utf16
for (uint i = 0; i < raw->Value.size() - 3; ++i)
{
if (NLMISC::toLower(((char *)&raw->Value[0])[i]) == '.'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 1]) == 'm'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 2]) == 'a'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 3]) == 'x')
return true;
if (NLMISC::toLower(((char *)&raw->Value[0])[i]) == '.'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 1]) == 't'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 2]) == 'g'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 3]) == 'a')
return true;
if (NLMISC::toLower(((char *)&raw->Value[0])[i]) == '.'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 1]) == 'p'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 2]) == 'n'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 3]) == 'g')
return true;
}
}
if (raw->Value.size() >= 6)
{
for (uint i = 0; i < raw->Value.size() - 6; ++i)
{
if (NLMISC::toLower(((char *)&raw->Value[0])[i]) == '.'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 2]) == 'm'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 4]) == 'a'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 6]) == 'x')
return true;
if (NLMISC::toLower(((char *)&raw->Value[0])[i]) == '.'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 2]) == 't'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 4]) == 'g'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 6]) == 'a')
return true;
if (NLMISC::toLower(((char *)&raw->Value[0])[i]) == '.'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 2]) == 'p'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 4]) == 'n'
&& NLMISC::toLower(((char *)&raw->Value[0])[i + 6]) == 'g')
return true;
}
} }
return false; return false;
} }
@ -650,8 +718,8 @@ void fixChunk(uint16 id, IStorageObject *chunk)
if (asUCString) if (asUCString)
{ {
// nldebug("UCString: %s", asUCString->Value.toUtf8().c_str()); // nldebug("UCString: %s", asUCString->Value.toUtf8().c_str());
if (isImportantFilePath(asUCString->Value.toUtf8())) if (isImportantFilePath(asUCString->Value.toString()))
asUCString->Value.fromUtf8(rewritePathFinal(asUCString->Value.toUtf8())); asUCString->Value.fromUtf8(rewritePathFinal(asUCString->Value.toString()));
return; return;
} }
CStorageRaw *asRaw = dynamic_cast<CStorageRaw *>(chunk); CStorageRaw *asRaw = dynamic_cast<CStorageRaw *>(chunk);
@ -659,12 +727,181 @@ void fixChunk(uint16 id, IStorageObject *chunk)
{ {
switch (id) switch (id)
{ {
case 256:
case 4656:
case 16385:
if (hasImportantFilePath(asRaw))
{
// generic ucstring really
nlassert(asRaw->Value.size() % 2 == 0);
ucstring str;
str.resize(asRaw->Value.size() / 2);
memcpy(&str[0], &asRaw->Value[0], asRaw->Value.size());
nlassert(isImportantFilePath(str.toString()));
str.fromUtf8(rewritePathFinal(str.toString()));
asRaw->Value.resize(str.size() * 2);
memcpy(&asRaw->Value[0], &str[0], asRaw->Value.size());
break;
}
case 10:
if (hasImportantFilePath(asRaw))
{
// 10 00 08 00 00 00 02 00 80 00 40 // 11 bytes O_O
// 4d 00 00 00
// 57 3a 5c 44 61 74 61 62 61 73
// 65 5c 73 74 75 66 66 5c 74 72
// 79 6b 65 72 5c 61 67 65 6e 74
// 73 5c 5f 74 65 78 74 75 72 65
// 73 5c 61 63 74 6f 72 73 5c 54
// 52 5f 48 4f 46 5f 63 69 76 69
// 6c 30 31 5f 74 6f 72 73 6f 5f
// 43 31 2e 74 67 61 00
if (!(asRaw->Value[asRaw->Value.size() - 1] == 0))
{
nlinfo("Id: %i, size: %i", (uint32)id, asRaw->Value.size());
asRaw->toString(std::cout);
nldebug("x");
nlwarning("not null term");
std::string x;
std::cin >> x;
break;
}
uint32 size = ((uint32 *)&asRaw->Value[11])[0];
if (!(asRaw->Value.size() == size + 4 + 11))
{
nlinfo("Id: %i, size: %i", (uint32)id, asRaw->Value.size());
asRaw->toString(std::cout);
nldebug("x");
nlwarning("size does not match, use different algo :)");
uint8 nonsense[11];
uint32 counter;
std::vector<std::string> strings;
{
NLMISC::CMemStream mem;
asRaw->serial(mem);
mem.invert();
mem.serialBuffer(nonsense, 11);
mem.serial(counter);
uint i = 0;
while ((sint)mem.getPos() != (sint)mem.size())
{
nldebug("pos %i", mem.getPos());
nldebug("size %i", mem.size());
char funny;
mem.serial(funny);
nlassert(funny == '@');
uint32 size;
mem.serial(size);
nldebug("size %i", size);
std::string v;
v.resize(size);
mem.serialBuffer((uint8 *)&v[0], size);
if (!(v[v.size() - 1] == 0))
{
nlinfo("Id: %i, size: %i", (uint32)id, asRaw->Value.size());
asRaw->toString(std::cout);
nldebug("x");
nlwarning("not null term inside array stuff %i '%s'", i, v.c_str());
std::string x;
std::cin >> x;
return;
}
v.resize(v.size() - 1);
nldebug("%s", v.c_str());
strings.push_back(v);
++i;
nldebug("ok");
}
nlassert(strings.size() == counter);
asRaw->Value.resize(0);
}
bool foundone = false;
for (uint i = 0; i < strings.size(); ++i)
{
if (isImportantFilePath(strings[i]))
{
foundone = true;
strings[i] = rewritePathFinal(strings[i]);
}
}
nlassert(foundone);
{
nldebug("go");
NLMISC::CMemStream mem;
mem.serialBuffer(nonsense, 11);
mem.serial(counter);
for (uint i = 0; i < strings.size(); ++i)
{
nldebug("one");
char funny = '@';
mem.serial(funny);
strings[i].resize(strings[i].size() + 1);
strings[i][strings[i].size() - 1] = 0;
uint32 size = strings[i].size();
mem.serial(size);
mem.serialBuffer((uint8 *)&strings[i][0], size);
}
asRaw->setSize(mem.getPos());
mem.invert();
asRaw->serial(mem);
}
//std::string x;
//std::cin >> x;
return;
}
std::string str;
str.resize(size - 1);
memcpy(&str[0], &asRaw->Value[15], str.size());
// nldebug("test '%s'", str.c_str());
// asRaw->toString(std::cout);
if (!isImportantFilePath(str))
{
nlinfo("Id: %i, size: %i", (uint32)id, asRaw->Value.size());
asRaw->toString(std::cout);
nldebug("x");
nlwarning("not important");
std::string x;
std::cin >> x;
break;
}
str = rewritePathFinal(str);
asRaw->Value.resize(str.size() + 11 + 4 + 1);
memcpy(&asRaw->Value[15], &str[0], str.size());
((uint32 *)&asRaw->Value[11])[0] = str.size() + 1;
asRaw->Value[asRaw->Value.size() - 1] = 0;
break;
}
case 304:
if (hasImportantFilePath(asRaw))
{
// null term c string
nlassert(asRaw->Value[asRaw->Value.size() - 1] == 0);
std::string str;
str.resize(asRaw->Value.size() - 1);
memcpy(&str[0], &asRaw->Value[0], str.size());
if (!isImportantFilePath(str))
{
nlinfo("Id: %i", (uint32)id);
asRaw->toString(std::cout);
nlerror("not important");
}
str = rewritePathFinal(str);
asRaw->Value.resize(str.size() + 1);
memcpy(&asRaw->Value[0], &str[0], str.size());
asRaw->Value[asRaw->Value.size() - 1] = 0;
break;
}
case 9730:
// ignore
break;
default: default:
if (hasImportantFilePath(asRaw)) if (hasImportantFilePath(asRaw))
{ {
nlinfo("Id: %i", (uint32)id); nlinfo("Id: %i", (uint32)id);
asRaw->toString(std::cout); asRaw->toString(std::cout);
nlerror("Found important file path"); nlwarning("Found important file path");
std::string x;
std::cin >> x;
return; return;
} }
break; break;
@ -820,7 +1057,7 @@ int main(int argc, char **argv)
CEPoly::registerClasses(&SceneClassRegistry); CEPoly::registerClasses(&SceneClassRegistry);
//handleFile("/srv/work/database/interfaces/anims_max/cp_fy_hof_species.max"); //handleFile("/srv/work/database/interfaces/anims_max/cp_fy_hof_species.max");
//runInitialize(); runInitialize();
runHandler(); runHandler();
//runScanner(); //runScanner();

Write Preview
Loading…
Cancel
Save