From b1547fe1d58655e72f72e0dd20e31d1289d5a89b Mon Sep 17 00:00:00 2001 From: kaetemi Date: Mon, 30 Jul 2012 17:03:43 +0200 Subject: [PATCH] Changed: #1440 Moved some metadata utility functions --HG-- branch : build_pipeline_v3 --- .../pipeline/service/database_status.cpp | 70 +-------------- .../tools/pipeline/service/database_status.h | 68 -------------- .../pipeline/service/metadata_storage.cpp | 74 ++++++++++++++- .../tools/pipeline/service/metadata_storage.h | 90 ++++++++++++++++--- .../pipeline/service/workspace_storage.h | 2 + 5 files changed, 156 insertions(+), 148 deletions(-) diff --git a/code/nel/tools/pipeline/service/database_status.cpp b/code/nel/tools/pipeline/service/database_status.cpp index 5c3bbbe24..8faa4ad72 100644 --- a/code/nel/tools/pipeline/service/database_status.cpp +++ b/code/nel/tools/pipeline/service/database_status.cpp @@ -56,45 +56,6 @@ std::string getStatusFilePath(const std::string &path) } /* anonymous namespace */ -void CFileError::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) -{ - uint version = stream.serialVersion(1); - stream.serial(Project); - stream.serial(Process); - stream.serial(Message); -} - -void CFileStatus::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) -{ - uint version = stream.serialVersion(2); - // if (version >= 3) stream.serial(LastRemoved); else LastRemoved = 0; - stream.serial(FirstSeen); - stream.serial(LastChangedReference); - if (version >= 2) stream.serial(LastFileSizeReference); else LastFileSizeReference = 0; - stream.serial(LastUpdate); - stream.serial(CRC32); -} - -void CFileRemove::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) -{ - uint version = stream.serialVersion(1); - stream.serial(Lost); -} - -void CProjectOutput::CFileOutput::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) -{ - uint version = stream.serialVersion(1); - stream.serial(CRC32); - stream.serial((uint8 &)Level); // test this :o) -} - -void CProjectOutput::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) -{ - uint version = stream.serialVersion(1); - stream.serialCont(FilePaths); - stream.serialCont(FileOutputs); -} - CDatabaseStatus::CDatabaseStatus() { //CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR); @@ -112,28 +73,14 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string & std::string stdPath = unMacroPath(filePath); std::string statusPath = getStatusFilePath(filePath); m_StatusMutex.lock_shared(); - if (CFile::fileExists(statusPath)) + if (CMetadataStorage::readStatus(fileStatus, statusPath)) { - nlassert(!CFile::isDirectory(filePath)); - - CIFile ifs(statusPath, false); - fileStatus.serial(ifs); - ifs.close(); uint32 fmdt = CFile::getFileModificationDate(stdPath); uint32 fisz = CFile::getFileSize(stdPath); seemsValid = ((fmdt == fileStatus.LastChangedReference) && (fisz == fileStatus.LastFileSizeReference)); } - else - { - // fileStatus.LastRemoved = 0; - fileStatus.FirstSeen = 0; - fileStatus.LastChangedReference = 0; - fileStatus.LastFileSizeReference = ~0; - fileStatus.LastUpdate = 0; - fileStatus.CRC32 = 0; - } m_StatusMutex.unlock_shared(); return seemsValid; } @@ -277,14 +224,8 @@ public: uint32 fmdt = CFile::getFileModificationDate(FilePath); std::string statusPath = getStatusFilePath(FilePath); // g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status"; StatusMutex->lock_shared(); - bool statusFileExists = CFile::fileExists(statusPath); - if (statusFileExists) - { - CIFile ifs(statusPath, false); - fs.serial(ifs); - ifs.close(); - } - else + bool statusFileExists = CMetadataStorage::readStatus(fs, statusPath); + if (!statusFileExists) { firstSeen = true; fs.LastChangedReference = 0; @@ -331,10 +272,7 @@ public: StatusMutex->lock(); { - COFile ofs(statusPath, false, false, true); - fs.serial(ofs); - ofs.flush(); - ofs.close(); + CMetadataStorage::writeStatus(fs, statusPath); } { // Important that we remove the remove after creating the status in case the service is killed inbetween. diff --git a/code/nel/tools/pipeline/service/database_status.h b/code/nel/tools/pipeline/service/database_status.h index 22f11cef9..5ee3ce81b 100644 --- a/code/nel/tools/pipeline/service/database_status.h +++ b/code/nel/tools/pipeline/service/database_status.h @@ -49,74 +49,6 @@ namespace NLMISC { namespace PIPELINE { -//#define PIPELINE_DATABASE_STATUS_SUBDIR "database.status/" -//#define PIPELINE_DATABASE_ERRORS_SUBDIR "database.errors/" -//#define PIPELINE_DATABASE_DEPEND_SUBDIR "database.depend/" - -// Status is generated CRC32 for reference. -// Errors are errors caused by using this file as an input or output file. - -enum TFileState -{ - Unknown = 0, - Success = 1, - Warning = 2, - Error = 3, - Removal = 4, -}; - -struct CFileError -{ -public: - uint32 Time; // The time when this error occured. - // TFileState Level; // Success, Warning, Error, Removal - std::string Project; - std::string Process; - // std::string Plugin; - std::string Message; - - void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); -}; - -/// Errors set by a process when the file causes a build failure. -typedef std::vector CFileErrors; - -struct CFileStatus -{ -public: - // uint32 LastRemoved; // The last time this file was removed, purely informational (at the moment) because we can detect past removal and re-addition by comparing FirstSeen with our reference build time as well. - uint32 FirstSeen; // The time when this status file was first created (if the file was removed before this means the time when the file returned). - uint32 LastChangedReference; // The modification date value read when the CRC32 was calculated. - uint32 LastFileSizeReference; // The filesize when the CRC32 was calculated. - uint32 LastUpdate; // The start time when the CRC32 was calculated. - uint32 CRC32; - - void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); -}; - -struct CFileRemove -{ -public: - uint32 Lost; // The time when it was noticed the file was removed. - - void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); -}; - -struct CProjectOutput -{ - std::vector FilePaths; - struct CFileOutput - { - uint32 CRC32; - TFileState Level; - - void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); - }; - std::vector FileOutputs; - - void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); -}; - typedef CCallback TFileStatusCallback; /** diff --git a/code/nel/tools/pipeline/service/metadata_storage.cpp b/code/nel/tools/pipeline/service/metadata_storage.cpp index 6a0998fae..fea9d0e12 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.cpp +++ b/code/nel/tools/pipeline/service/metadata_storage.cpp @@ -32,6 +32,9 @@ // NeL includes // #include +#include +#include +#include // Project includes @@ -40,14 +43,77 @@ using namespace std; namespace PIPELINE { -CMetadataStorage::CMetadataStorage() +void CFileError::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) { - + uint version = stream.serialVersion(1); + stream.serial(Project); + stream.serial(Process); + stream.serial(Message); } -CMetadataStorage::~CMetadataStorage() +void CFileStatus::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) { - + uint version = stream.serialVersion(2); + // if (version >= 3) stream.serial(LastRemoved); else LastRemoved = 0; + stream.serial(FirstSeen); + stream.serial(LastChangedReference); + if (version >= 2) stream.serial(LastFileSizeReference); else LastFileSizeReference = 0; + stream.serial(LastUpdate); + stream.serial(CRC32); +} + +void CFileRemove::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) +{ + uint version = stream.serialVersion(1); + stream.serial(Lost); +} + +void CProjectOutput::CFileOutput::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) +{ + uint version = stream.serialVersion(1); + stream.serial(CRC32); + stream.serial((uint8 &)Level); // test this :o) +} + +void CProjectOutput::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) +{ + uint version = stream.serialVersion(1); + stream.serialCont(FilePaths); + stream.serialCont(FileOutputs); +} + +bool CMetadataStorage::readStatus(CFileStatus &status, const std::string &path) +{ + if (!NLMISC::CFile::fileExists(path)) + { + status.FirstSeen = 0; + status.LastChangedReference = 0; + status.LastFileSizeReference = ~0; + status.LastUpdate = 0; + status.CRC32 = 0; + return false; + } + + nlassert(!NLMISC::CFile::isDirectory(path)); + + NLMISC::CIFile is(path, false); + status.serial(is); + is.close(); + + return true; +} + +void CMetadataStorage::writeStatus(const CFileStatus &status, const std::string &path) +{ + NLMISC::COFile os(path, false, false, true); + const_cast(status).serial(os); + os.flush(); + os.close(); +} + +void CMetadataStorage::eraseStatus(const std::string &path) +{ + NLMISC::CFile::deleteFile(path); } } /* namespace PIPELINE */ diff --git a/code/nel/tools/pipeline/service/metadata_storage.h b/code/nel/tools/pipeline/service/metadata_storage.h index 0ea460d8d..6110d4b33 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.h +++ b/code/nel/tools/pipeline/service/metadata_storage.h @@ -36,16 +36,88 @@ // Project includes #include "workspace_storage.h" +namespace NLMISC { + class IStream; + struct EStream; +} + namespace PIPELINE { +// Status is generated CRC32 for reference. +// Errors are errors caused by using this file as an input or output file. + +enum TFileState +{ + Unknown = 0, + Success = 1, + Warning = 2, + Error = 3, + Removal = 4, +}; + /// Suffix for metafiles that contain the CRC32 etc #define PIPELINE_DATABASE_STATUS_SUFFIX ".status" +struct CFileStatus +{ +public: + // uint32 LastRemoved; // The last time this file was removed, purely informational (at the moment) because we can detect past removal and re-addition by comparing FirstSeen with our reference build time as well. + uint32 FirstSeen; // The time when this status file was first created (if the file was removed before this means the time when the file returned). + uint32 LastChangedReference; // The modification date value read when the CRC32 was calculated. + uint32 LastFileSizeReference; // The filesize when the CRC32 was calculated. + uint32 LastUpdate; // The start time when the CRC32 was calculated. + uint32 CRC32; + + void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); +}; + /// Suffix for metafiles that contain error info on database files #define PIPELINE_DATABASE_ERRORS_SUFFIX ".errors" -/// Suffix for metafiles that contain dependencies for a file -#define PIPELINE_DATABASE_DEPEND_SUFFIX ".depend" +struct CFileError +{ +public: + uint32 Time; // The time when this error occured. + // TFileState Level; // Success, Warning, Error, Removal + std::string Project; + std::string Process; + // std::string Plugin; + std::string Message; + + void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); +}; + +/// Errors set by a process when the file causes a build failure. +typedef std::vector CFileErrors; + /// Suffix for metafiles that refer to a previously known file that no longer exists #define PIPELINE_DATABASE_REMOVE_SUFFIX ".remove" +struct CFileRemove +{ +public: + uint32 Lost; // The time when it was noticed the file was removed. + + void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); +}; + +/// Suffix for metafiles that contain dependencies for a file +#define PIPELINE_DATABASE_DEPEND_SUFFIX ".depend" +// ....................... + +/// Suffix for metafiles that contain the output of the last build of a project +#define PIPELINE_DATABASE_OUTPUT_SUFFIX ".output" +struct CProjectOutput +{ + std::vector FilePaths; + struct CFileOutput + { + uint32 CRC32; + TFileState Level; + + void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); + }; + std::vector FileOutputs; + + void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); +}; /** * \brief CMetadataStorage @@ -55,15 +127,13 @@ namespace PIPELINE { */ class CMetadataStorage { -protected: - // pointers - // ... - - // instances - // ... public: - CMetadataStorage(); - virtual ~CMetadataStorage(); + + /// Note: Use the functions provided by CDatabaseStatus for manipulating status files. + static bool readStatus(CFileStatus &status, const std::string &path); + static void writeStatus(const CFileStatus &status, const std::string &path); + static void eraseStatus(const std::string &path); + }; /* class CMetadataStorage */ } /* namespace PIPELINE */ diff --git a/code/nel/tools/pipeline/service/workspace_storage.h b/code/nel/tools/pipeline/service/workspace_storage.h index 2cc889687..be78e2f5e 100644 --- a/code/nel/tools/pipeline/service/workspace_storage.h +++ b/code/nel/tools/pipeline/service/workspace_storage.h @@ -49,7 +49,9 @@ namespace PIPELINE { class CWorkspaceStorage { public: + /// Get the path for a metadata file, based on a filepath it represents, and the suffix of it's contents static std::string getMetaFilePath(const std::string &path, const std::string &dotSuffix); + }; /* class CWorkspaceStorage */ } /* namespace PIPELINE */