From b6127aa582d22f215cd73f2cce2eb63708ebf8a1 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 26 Jul 2012 22:50:17 +0200 Subject: [PATCH] Added: #1440 Track file removal --HG-- branch : build_pipeline_v3 --- .../pipeline/service/database_status.cpp | 86 +++++++++++++++++-- .../tools/pipeline/service/database_status.h | 12 ++- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/code/nel/tools/pipeline/service/database_status.cpp b/code/nel/tools/pipeline/service/database_status.cpp index 7af134ff6..8c716f41b 100644 --- a/code/nel/tools/pipeline/service/database_status.cpp +++ b/code/nel/tools/pipeline/service/database_status.cpp @@ -164,13 +164,20 @@ void CFileError::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) void CFileStatus::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) { uint version = stream.serialVersion(2); + // if (version >= 3) stream.serial(LastRemoved); else LastRemoved = 0; stream.serial(FirstSeen); stream.serial(LastChangedReference); - if (version >= 2) stream.serial(LastFileSizeReference); + if (version >= 2) stream.serial(LastFileSizeReference); else LastFileSizeReference = 0; stream.serial(LastUpdate); stream.serial(CRC32); } +void CFileRemoved::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) +{ + uint version = stream.serialVersion(1); + stream.serial(Lost); +} + CDatabaseStatus::CDatabaseStatus() { //CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR); @@ -201,6 +208,7 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string & } else { + // fileStatus.LastRemoved = 0; fileStatus.FirstSeen = 0; fileStatus.LastChangedReference = 0; fileStatus.LastFileSizeReference = ~0; @@ -213,11 +221,14 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string & bool CDatabaseStatus::getFileStatus(std::map &fileStatusMap, const std::vector &paths) { + nlassert(false); // i don't think this is used? (yet?) (maybe for slave?) + // probably just some ghost code spooking around. + for (std::vector::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it) { } - return false; + return false; // it fails of course } namespace { @@ -242,7 +253,8 @@ public: uint32 fmdt = CFile::getFileModificationDate(FilePath); std::string statusPath = getStatusFilePath(FilePath); // g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status"; StatusMutex->lock_shared(); - if (CFile::fileExists(statusPath)) + bool statusFileExists = CFile::fileExists(statusPath); + if (statusFileExists) { CIFile ifs(statusPath, false); fs.serial(ifs); @@ -291,6 +303,8 @@ public: fs.LastFileSizeReference = fisz; } + std::string removePath = getMetaFilePath(FilePath, PIPELINE_DATABASE_REMOVE_SUFFIX); + StatusMutex->lock(); { COFile ofs(statusPath, false, false, true); @@ -298,6 +312,19 @@ public: ofs.flush(); ofs.close(); } + { + // Important that we remove the remove after creating the status in case the service is killed inbetween. + if (CFile::fileExists(removePath)) + { + nlwarning("File '%s' was removed before, and now it exists again!", FilePath.c_str()); + CFile::deleteFile(removePath); + nlinfo("Deleted '%s'", removePath.c_str()); + if (statusFileExists) + { + nlwarning("The remove file was not deleted last time the status file was created, this is either a bug, a sign of data corruption or it means the service was killed inbetween!"); + } + } + } StatusMutex->unlock(); } Callback(FilePath, fs, true); @@ -437,7 +464,7 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, for (std::vector::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it) { - const std::string subPath = *it; + const std::string &subPath = *it; updatePathStatus(ds, updater, subPath, recurse, false); @@ -454,7 +481,56 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, // Sanely get the process build start time by creating a dummy file at the beginning of the build and using that timestamp. // This file also unrelatedly can let us know if the service crashed or was killed during a build. - // ... TODO ... + // Note that we won't notice when a directory is deleted, this is also not necessary because we never work recursively in the service. + // Any recursive lookups are done in specific unsafe build tools that are tagged as unsafe and not monitored by the pipeline. + + std::string dirPathMeta = getMetaFilePath(dirPath, ""); + // nldebug("META DIR: %s", dirPathMeta.c_str()); + std::vector dirContentsMeta; + + CPath::getPathContent(dirPathMeta, false, false, true, dirContentsMeta); + + // This code is not guaranteed to work reliably. + for (std::vector::iterator it = dirContentsMeta.begin(), end = dirContentsMeta.end(); it != end; ++it) + { + const std::string &subPath = *it; + + std::string::size_type statusPos = subPath.find(PIPELINE_DATABASE_STATUS_SUFFIX); + + if (statusPos != std::string::npos) + { + std::string subPathFilename = CFile::getFilename(subPath.substr(0, statusPos)); + std::string subPathOrig = dirPath + subPathFilename; + if (std::find(dirContents.begin(), dirContents.end(), subPathOrig) == dirContents.end()) + { + // File no longer exists, warn the user. + nlwarning("The file '%s' no longer exists!", subPathOrig.c_str()); + + // Create the removed tag. + std::string removedTagFile = dirPathMeta + subPathFilename + PIPELINE_DATABASE_REMOVE_SUFFIX; + uint32 time = CTime::getSecondsSince1970(); + CFileRemoved removed; + removed.Lost = time; // Yes you're wasting time, you wouldn't have to delete anything if you did your work properly in the first place! :) + COFile of(removedTagFile, false, false, true); + removed.serial(of); + of.flush(); + of.close(); + nldebug("Created '%s'", removedTagFile.c_str()); + + // Delete the status. + NLMISC::CFile::deleteFile(subPath); + nlinfo("Removed '%s'", subPath.c_str()); + } + } + /*else + { + // TEMP + // nldebug("Invalid file %s", subPath.c_str()); + }*/ + + if (g_IsExiting) + return; + } // <-- END REMOVE } diff --git a/code/nel/tools/pipeline/service/database_status.h b/code/nel/tools/pipeline/service/database_status.h index 85326ede8..ddb105cc9 100644 --- a/code/nel/tools/pipeline/service/database_status.h +++ b/code/nel/tools/pipeline/service/database_status.h @@ -55,6 +55,7 @@ namespace PIPELINE { #define PIPELINE_DATABASE_STATUS_SUFFIX ".status" #define PIPELINE_DATABASE_ERRORS_SUFFIX ".errors" #define PIPELINE_DATABASE_DEPEND_SUFFIX ".depend" +#define PIPELINE_DATABASE_REMOVE_SUFFIX ".remove" #define PIPELINE_DATABASE_META_SUFFIX ".meta" @@ -81,7 +82,8 @@ typedef std::vector CFileErrors; struct CFileStatus { public: - uint32 FirstSeen; + // uint32 LastRemoved; // The last time this file was removed, purely informational (at the moment) because we can detect past removal and re-addition by comparing FirstSeen with our reference build time as well. + uint32 FirstSeen; // The time when this status file was first created (if the file was removed before this means the time when the file returned). uint32 LastChangedReference; // The modification date value read when the CRC32 was calculated. uint32 LastFileSizeReference; // The filesize when the CRC32 was calculated. uint32 LastUpdate; // The start time when the CRC32 was calculated. @@ -90,6 +92,14 @@ public: void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); }; +struct CFileRemoved +{ +public: + uint32 Lost; // The time when it was noticed the file was removed. + + void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); +}; + typedef CCallback TFileStatusCallback; /**