From dd50972c0c0385a4bd04933fa93f33e5625fbe2b Mon Sep 17 00:00:00 2001 From: kaetemi Date: Mon, 27 Aug 2012 13:17:39 +0200 Subject: [PATCH] Changed: #1440 Simplify AppData interface --HG-- branch : build_pipeline_v3 --- .../pipeline/max/builtin/storage/app_data.cpp | 65 ++++++----- .../pipeline/max/builtin/storage/app_data.h | 104 +++++++++++++++++- .../nel/tools/pipeline/max/storage_object.cpp | 42 +------ code/nel/tools/pipeline/max_dump/main.cpp | 41 ++++++- 4 files changed, 181 insertions(+), 71 deletions(-) diff --git a/code/nel/tools/pipeline/max/builtin/storage/app_data.cpp b/code/nel/tools/pipeline/max/builtin/storage/app_data.cpp index 46eaef530..7bc72f151 100644 --- a/code/nel/tools/pipeline/max/builtin/storage/app_data.cpp +++ b/code/nel/tools/pipeline/max/builtin/storage/app_data.cpp @@ -61,7 +61,6 @@ namespace STORAGE { #define PMBS_APP_DATA_HEADER_CHUNK_ID 0x0100 #define PMBS_APP_DATA_ENTRY_CHUNK_ID 0x0110 #define PMBS_APP_DATA_ENTRY_KEY_CHUNK_ID 0x0120 -#define PMBS_APP_DATA_ENTRY_VALUE_CHUNK_ID 0x0130 //////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// @@ -122,7 +121,12 @@ CAppData::CAppData() CAppData::~CAppData() { - + if (!m_ChunksOwnsPointers) + { + for (TMap::const_iterator subit = m_Entries.begin(), subend = m_Entries.end(); subit != subend; ++subit) + delete subit->second; + m_Entries.clear(); + } } std::string CAppData::className() const @@ -197,6 +201,11 @@ void CAppData::clean() if (m_Chunks.begin()->first != PMBS_APP_DATA_HEADER_CHUNK_ID) { nlerror("Bad id %x, expected %x", (uint32)m_Chunks.begin()->first, PMBS_APP_DATA_HEADER_CHUNK_ID); return; } // Cannot happen, because we won't have local ownership if parsing failed delete m_Chunks.begin()->second; // Delete the header chunk, since we own it m_Chunks.clear(); // Clear the remaining chunks + + // Clean raw storage + for (TMap::const_iterator subit = m_Entries.begin(), subend = m_Entries.end(); subit != subend; ++subit) + subit->second->clean(); + #else CStorageContainer::clean(); #endif @@ -217,6 +226,9 @@ void CAppData::build(uint16 version, uint filter) // Set up the entries for (TMap::iterator it = m_Entries.begin(), end = m_Entries.end(); it != end; ++it) m_Chunks.push_back(TStorageObjectWithId(PMBS_APP_DATA_ENTRY_CHUNK_ID, it->second)); + + // Rebuild raw storage + CStorageContainer::build(version); #else CStorageContainer::build(version); #endif @@ -251,7 +263,7 @@ void CAppData::init() // We own this m_ChunksOwnsPointers = false; } - +/* const uint8 *CAppData::read(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId, uint32 &size) const { if (m_ChunksOwnsPointers) { nlwarning("Not parsed"); return NULL; } @@ -305,7 +317,7 @@ void CAppData::fill(NLMISC::CClassId classId, TSClassId superClassId, uint32 sub // Outside classes should unlock in case the implementation changes. // unlock(classId, superClassId, subId, size); } - +*/ void CAppData::erase(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId) { if (m_ChunksOwnsPointers) { nlwarning("Not parsed"); return; } @@ -372,14 +384,15 @@ void CAppDataEntryKey::toString(std::ostream &ostream, const std::string &pad) c //////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// -CAppDataEntry::CAppDataEntry() : m_Key(NULL), m_Value(NULL) +CAppDataEntry::CAppDataEntry() : m_Key(NULL), m_Raw(NULL), m_Value(NULL) { } CAppDataEntry::~CAppDataEntry() { - + delete m_Value; + m_Value = NULL; } std::string CAppDataEntry::className() const @@ -417,7 +430,7 @@ void CAppDataEntry::parse(uint16 version, uint filter) ++it; if (it->first != PMBS_APP_DATA_ENTRY_VALUE_CHUNK_ID) { nlwarning("Bad id %x, expected %x", (uint32)it->first, PMBS_APP_DATA_ENTRY_VALUE_CHUNK_ID); disown(); return; } - m_Value = static_cast(it->second); + m_Raw = static_cast(it->second); // m_ChunksOwnsPointers = false; nlassert(m_ChunksOwnsPointers); // Never set false here @@ -425,18 +438,28 @@ void CAppDataEntry::parse(uint16 version, uint filter) void CAppDataEntry::clean() { - nlassert(false); // CStorageContainer::clean(); - // if (m_ChunksOwnsPointers) { nlwarning("Not parsed"); return; } - // Nothing to do here! + if (m_Value) + { + nlassert(m_Raw); + m_Raw->Value.resize(0); + m_Key->Size = 0xFFFFFFFF; + } } void CAppDataEntry::build(uint16 version, uint filter) { - nlassert(false); - // if (m_ChunksOwnsPointers) { nlwarning("Not parsed"); return; } - // Nothing to do here! // CStorageContainer::build(version); + if (m_Value) + { + nlassert(m_Raw); + NLMISC::CMemStream mem; + m_Value->serial(mem); + m_Raw->setSize(mem.getPos()); + mem.invert(); + m_Raw->serial(mem); + m_Key->Size = m_Raw->Value.size(); + } } void CAppDataEntry::disown() @@ -444,6 +467,8 @@ void CAppDataEntry::disown() // CStorageContainer::disown(); if (m_Chunks.size() != 2) { nlerror("Not built"); return; } // Built chunks must match the parsed data m_Key = NULL; + m_Raw = NULL; + delete m_Value; m_Value = NULL; } @@ -452,18 +477,8 @@ void CAppDataEntry::init() nlassert(m_Chunks.size() == 0); m_Key = new CAppDataEntryKey(); m_Chunks.push_back(TStorageObjectWithId(PMBS_APP_DATA_ENTRY_KEY_CHUNK_ID, m_Key)); - m_Value = new CStorageRaw(); - m_Chunks.push_back(TStorageObjectWithId(PMBS_APP_DATA_ENTRY_VALUE_CHUNK_ID, m_Value)); -} - -CAppDataEntryKey *CAppDataEntry::key() -{ - return m_Key; -} - -CStorageRaw *CAppDataEntry::value() -{ - return m_Value; + m_Raw = new CStorageRaw(); + m_Chunks.push_back(TStorageObjectWithId(PMBS_APP_DATA_ENTRY_VALUE_CHUNK_ID, m_Raw)); } IStorageObject *CAppDataEntry::createChunkById(uint16 id, bool container) diff --git a/code/nel/tools/pipeline/max/builtin/storage/app_data.h b/code/nel/tools/pipeline/max/builtin/storage/app_data.h index 4ba306492..4cdbb01e8 100644 --- a/code/nel/tools/pipeline/max/builtin/storage/app_data.h +++ b/code/nel/tools/pipeline/max/builtin/storage/app_data.h @@ -33,6 +33,7 @@ // NeL includes #include +#include // Project includes #include "../../typedefs.h" @@ -45,6 +46,7 @@ namespace BUILTIN { namespace STORAGE { #define PMBS_APP_DATA_CHUNK_ID 0x2150 +#define PMBS_APP_DATA_ENTRY_VALUE_CHUNK_ID 0x0130 class CAppDataEntry; @@ -86,7 +88,7 @@ public: void init(); // public // TODO: Simplify using templates and returning a specialized storage object, auto-converted to the requested type. - /// Gets a pointer to an appdata chunk buffer. Returns NULL if it does not exist. Size is returned in the size parameter. + /*/// Gets a pointer to an appdata chunk buffer. Returns NULL if it does not exist. Size is returned in the size parameter. const uint8 *read(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId, uint32 &size) const; /// Locks a pointer to an appdata chunk buffer for writing to with specified capacity. May return NULL if this chunk is unparsable or no memory can be allocated. uint8 *lock(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId, uint32 capacity); @@ -95,6 +97,16 @@ public: /// Fills an appdata chunk buffer with specified data, which will be copied. void fill(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId, uint8 *buffer, uint32 size); /// Erases an appdata chunk. + void erase(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId);*/ + + // public + /// Gets an appdata chunk storage object, returns NULL if it does not exist + template + T *get(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId); + /// Gets or creates a chunk storage object + template + T *getOrCreate(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId); + /// Erases an appdata chunk. void erase(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId); // read access @@ -142,6 +154,9 @@ public: */ class CAppDataEntry : public CStorageContainer { +public: + friend class CAppData; + public: CAppDataEntry(); virtual ~CAppDataEntry(); @@ -158,19 +173,98 @@ public: // Initializes a new entry void init(); // Returns the key - CAppDataEntryKey *key(); - // Returns the blob - CStorageRaw *value(); + inline CAppDataEntryKey *key() { return m_Key; } + // Returns the value + template + T *value(); protected: virtual IStorageObject *createChunkById(uint16 id, bool container); private: CAppDataEntryKey *m_Key; - CStorageRaw *m_Value; + CStorageRaw *m_Raw; + IStorageObject *m_Value; }; /* class CAppDataEntry */ +template +T *CAppDataEntry::value() +{ + if (m_Value) + { + T *result = dynamic_cast(m_Value); + if (result) + { + return result; + } + else + { + nlwarning("AppData value has already been cast to another type, recasting, the previous chunk becomes invalid, this may not be intended"); + NLMISC::CMemStream mem; + m_Value->serial(mem); + result = new T(); + result->setSize(mem.getPos()); + mem.invert(); + delete m_Value; + m_Value = result; + result->serial(mem); + return result; + } + } + else + { + nlassert(!m_Value); + nlassert(m_Raw); + // nldebug("Casting raw to user type"); + NLMISC::CMemStream mem; + m_Raw->serial(mem); + T *result = new T(); + result->setSize(mem.getPos()); + // nldebug("Read %i", mem.getPos()); + mem.invert(); + m_Raw->Value.resize(0); + m_Key->Size = 0xFFFFFFFF; + m_Value = result; + result->serial(mem); + // nldebug("Wrote %i", mem.getPos()); + return result; + } +} + +template +T *CAppData::get(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId) +{ + if (m_ChunksOwnsPointers) { nlwarning("Not parsed"); return NULL; } + TKey key(classId, superClassId, subId); + TMap::const_iterator it = m_Entries.find(key); + if (it == m_Entries.end()) { nldebug("Trying to read non-existant key, this is allowed, returning NULL"); return NULL; } + return it->second->value(); +} + +template +T *CAppData::getOrCreate(NLMISC::CClassId classId, TSClassId superClassId, uint32 subId) +{ + if (m_ChunksOwnsPointers) { nlwarning("Not parsed"); return NULL; } + TKey key(classId, superClassId, subId); + TMap::const_iterator it = m_Entries.find(key); + CAppDataEntry *appDataEntry; + if (it == m_Entries.end()) + { + appDataEntry = new CAppDataEntry(); + appDataEntry->init(); + m_Entries[key] = appDataEntry; + appDataEntry->key()->ClassId = classId; + appDataEntry->key()->SuperClassId = superClassId; + appDataEntry->key()->SubId = subId; + } + else + { + appDataEntry = it->second; + } + return appDataEntry->value(); +} + } /* namespace STORAGE */ } /* namespace BUILTIN */ } /* namespace MAX */ diff --git a/code/nel/tools/pipeline/max/storage_object.cpp b/code/nel/tools/pipeline/max/storage_object.cpp index 674547ec7..53cacdc55 100644 --- a/code/nel/tools/pipeline/max/storage_object.cpp +++ b/code/nel/tools/pipeline/max/storage_object.cpp @@ -129,45 +129,15 @@ void CStorageContainer::serial(NLMISC::IStream &stream) } } #ifdef NL_DEBUG_STORAGE - nldebug("Wrapping the container inside a stream with necessary size markers"); + nldebug("Wrap the container inside a chunk, as the size is not known"); #endif { - const uint32 magic = 0xC0C01473; - stream.serialCheck(magic); - uint version = stream.serialVersion(1); - sint32 sizePos; - bool reading = stream.isReading(); - if (!reading) - { - sizePos = stream.getPos(); - } - sint64 size = 0; - stream.serial(size); - CStorageChunks chunks(stream, size); + // Use dummy size value so the system can at least read the header + CStorageChunks chunks(stream, stream.isReading() ? 0xFF : 0); + bool ok = chunks.enterChunk(0x4352, true); + nlassert(ok); serial(chunks); - if (!reading) - { - sint32 returnPos = stream.getPos(); -#ifdef NL_DEBUG_STORAGE - nldebug("current (return) pos is %i", stream.getPos()); -#endif - size = returnPos - sizePos - 8; - stream.seek(sizePos, NLMISC::IStream::begin); -#ifdef NL_DEBUG_STORAGE - nldebug("current (size) pos is %i", stream.getPos()); -#endif - stream.serial(size); - stream.seek(returnPos, NLMISC::IStream::begin); -#ifdef NL_DEBUG_STORAGE - nldebug("sizePos is %i", sizePos); - nldebug("returnPos is %i", returnPos); - nldebug("current (return) pos is %i", stream.getPos()); -#endif - } -#ifdef NL_DEBUG_STORAGE - nldebug("Chunk container wrapper size is %i", size); -#endif - return; + chunks.leaveChunk(); } } diff --git a/code/nel/tools/pipeline/max_dump/main.cpp b/code/nel/tools/pipeline/max_dump/main.cpp index 0ba0026f7..08e0eda9b 100644 --- a/code/nel/tools/pipeline/max_dump/main.cpp +++ b/code/nel/tools/pipeline/max_dump/main.cpp @@ -241,7 +241,7 @@ int main(int argc, char **argv) nldebug("PARSE"); scene.parse(PIPELINE::MAX::VersionUnknown); // parse the structure to readable data nldebug("CLEAN"); - scene.clean(); // cleanup unused file structure, don't clean up if we want direct access to chunks as well + //scene.clean(); // cleanup unused file structure, don't clean up if we want direct access to chunks as well // <- TEST //scene.toString(std::cout);//## std::cout << "\n"; @@ -283,17 +283,48 @@ int main(int argc, char **argv) PIPELINE::MAX::BUILTIN::STORAGE::CAppData *appData = dynamic_cast(storageChunk); if (appData) { + nlinfo("Found AppData"); - uint32 size; - const uint8 *buffer = appData->read(MAXSCRIPT_UTILITY_CLASS_ID, UTILITY_CLASS_ID, NEL3D_APPDATA_ENV_FX, size); - if (buffer) + CStorageRaw *raw = appData->get(MAXSCRIPT_UTILITY_CLASS_ID, UTILITY_CLASS_ID, NEL3D_APPDATA_ENV_FX); + if (raw) { - nlinfo("Found NEL3D_APPDATA_ENV_FX, size %i", size); + nlinfo("Found NEL3D_APPDATA_ENV_FX, size %i", raw->Value.size()); + //raw->Value.resize(200); } + } } } +/* + scene.clean(); + scene.build(PIPELINE::MAX::VersionUnknown); + scene.disown(); + scene.parse(PIPELINE::MAX::VersionUnknown); + + ssc = scene.container(); + for (PIPELINE::MAX::CStorageContainer::TStorageObjectConstIt it = ssc->chunks().begin(), end = ssc->chunks().end(); it != end; ++it) + { + PIPELINE::MAX::CStorageContainer *subc = static_cast(it->second); + for (PIPELINE::MAX::CStorageContainer::TStorageObjectConstIt subit = subc->chunks().begin(), subend = subc->chunks().end(); subit != subend; ++subit) + { + PIPELINE::MAX::IStorageObject *storageChunk = subit->second; + PIPELINE::MAX::BUILTIN::STORAGE::CAppData *appData = dynamic_cast(storageChunk); + if (appData) + { + + nlinfo("Found AppData"); + const CStorageRaw *raw = appData->get(MAXSCRIPT_UTILITY_CLASS_ID, UTILITY_CLASS_ID, NEL3D_APPDATA_ENV_FX); + if (raw) + { + nlinfo("Found NEL3D_APPDATA_ENV_FX, size %i", raw->Value.size()); + } + + } + } + } + */ + /* GsfInput *input = gsf_infile_child_by_name(infile, streamname);