Ryzom
/
ryzom-core
1
1
Fork 0
Code Issues 38 Pull Requests Projects 1 Releases 1 Activity

Added: #1440 Track file removal

Browse Source 
--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 12 years ago
parent f5dafabc12
commit b6127aa582
2 changed files with 92 additions and 6 deletions
Show Stats Download Patch File Download Diff File

86
code/nel/tools/pipeline/service/database_status.cpp
Unescape Escape View File

@ -164,13 +164,20 @@ void CFileError::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)
void CFileStatus::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)
{
uint version = stream.serialVersion(2);
// if (version >= 3) stream.serial(LastRemoved); else LastRemoved = 0;
stream.serial(FirstSeen);
stream.serial(LastChangedReference);
if (version >= 2) stream.serial(LastFileSizeReference);
if (version >= 2) stream.serial(LastFileSizeReference); else LastFileSizeReference = 0;
stream.serial(LastUpdate);
stream.serial(CRC32);
}
void CFileRemoved::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)
{
uint version = stream.serialVersion(1);
stream.serial(Lost);
}
CDatabaseStatus::CDatabaseStatus()
{
//CFile::createDirectoryTree(g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR);
@ -201,6 +208,7 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
}
else
{
// fileStatus.LastRemoved = 0;
fileStatus.FirstSeen = 0;
fileStatus.LastChangedReference = 0;
fileStatus.LastFileSizeReference = ~0;
@ -213,11 +221,14 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, const std::vector<std::string> &paths)
{
nlassert(false); // i don't think this is used? (yet?) (maybe for slave?)
// probably just some ghost code spooking around.
for (std::vector<std::string>::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it)
{
}
return false;
return false; // it fails of course
}
namespace {
@ -242,7 +253,8 @@ public:
uint32 fmdt = CFile::getFileModificationDate(FilePath);
std::string statusPath = getStatusFilePath(FilePath); // g_WorkspaceDirectory + PIPELINE_DATABASE_STATUS_SUBDIR + dropDatabaseDirectory(FilePath) + ".status";
StatusMutex->lock_shared();
if (CFile::fileExists(statusPath))
bool statusFileExists = CFile::fileExists(statusPath);
if (statusFileExists)
{
CIFile ifs(statusPath, false);
fs.serial(ifs);
@ -291,6 +303,8 @@ public:
fs.LastFileSizeReference = fisz;
}
std::string removePath = getMetaFilePath(FilePath, PIPELINE_DATABASE_REMOVE_SUFFIX);
StatusMutex->lock();
{
COFile ofs(statusPath, false, false, true);
@ -298,6 +312,19 @@ public:
ofs.flush();
ofs.close();
}
{
// Important that we remove the remove after creating the status in case the service is killed inbetween.
if (CFile::fileExists(removePath))
{
nlwarning("File '%s' was removed before, and now it exists again!", FilePath.c_str());
CFile::deleteFile(removePath);
nlinfo("Deleted '%s'", removePath.c_str());
if (statusFileExists)
{
nlwarning("The remove file was not deleted last time the status file was created, this is either a bug, a sign of data corruption or it means the service was killed inbetween!");
}
}
}
StatusMutex->unlock();
}
Callback(FilePath, fs, true);
@ -437,7 +464,7 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater,
for (std::vector<std::string>::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it)
{
const std::string subPath = *it;
const std::string &subPath = *it;
updatePathStatus(ds, updater, subPath, recurse, false);
@ -454,7 +481,56 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater,
// Sanely get the process build start time by creating a dummy file at the beginning of the build and using that timestamp.
// This file also unrelatedly can let us know if the service crashed or was killed during a build.
// ... TODO ...
// Note that we won't notice when a directory is deleted, this is also not necessary because we never work recursively in the service.
// Any recursive lookups are done in specific unsafe build tools that are tagged as unsafe and not monitored by the pipeline.
std::string dirPathMeta = getMetaFilePath(dirPath, "");
// nldebug("META DIR: %s", dirPathMeta.c_str());
std::vector<std::string> dirContentsMeta;
CPath::getPathContent(dirPathMeta, false, false, true, dirContentsMeta);
// This code is not guaranteed to work reliably.
for (std::vector<std::string>::iterator it = dirContentsMeta.begin(), end = dirContentsMeta.end(); it != end; ++it)
{
const std::string &subPath = *it;
std::string::size_type statusPos = subPath.find(PIPELINE_DATABASE_STATUS_SUFFIX);
if (statusPos != std::string::npos)
{
std::string subPathFilename = CFile::getFilename(subPath.substr(0, statusPos));
std::string subPathOrig = dirPath + subPathFilename;
if (std::find(dirContents.begin(), dirContents.end(), subPathOrig) == dirContents.end())
{
// File no longer exists, warn the user.
nlwarning("The file '%s' no longer exists!", subPathOrig.c_str());
// Create the removed tag.
std::string removedTagFile = dirPathMeta + subPathFilename + PIPELINE_DATABASE_REMOVE_SUFFIX;
uint32 time = CTime::getSecondsSince1970();
CFileRemoved removed;
removed.Lost = time; // Yes you're wasting time, you wouldn't have to delete anything if you did your work properly in the first place! :)
COFile of(removedTagFile, false, false, true);
removed.serial(of);
of.flush();
of.close();
nldebug("Created '%s'", removedTagFile.c_str());
// Delete the status.
NLMISC::CFile::deleteFile(subPath);
nlinfo("Removed '%s'", subPath.c_str());
}
}
/*else
{
// TEMP
// nldebug("Invalid file %s", subPath.c_str());
}*/
if (g_IsExiting)
return;
}
// <-- END REMOVE
}

12
code/nel/tools/pipeline/service/database_status.h
Unescape Escape View File

@ -55,6 +55,7 @@ namespace PIPELINE {
#define PIPELINE_DATABASE_STATUS_SUFFIX ".status"
#define PIPELINE_DATABASE_ERRORS_SUFFIX ".errors"
#define PIPELINE_DATABASE_DEPEND_SUFFIX ".depend"
#define PIPELINE_DATABASE_REMOVE_SUFFIX ".remove"
#define PIPELINE_DATABASE_META_SUFFIX ".meta"
@ -81,7 +82,8 @@ typedef std::vector<CFileError> CFileErrors;
struct CFileStatus
{
public:
uint32 FirstSeen;
// uint32 LastRemoved; // The last time this file was removed, purely informational (at the moment) because we can detect past removal and re-addition by comparing FirstSeen with our reference build time as well.
uint32 FirstSeen; // The time when this status file was first created (if the file was removed before this means the time when the file returned).
uint32 LastChangedReference; // The modification date value read when the CRC32 was calculated.
uint32 LastFileSizeReference; // The filesize when the CRC32 was calculated.
uint32 LastUpdate; // The start time when the CRC32 was calculated.
@ -90,6 +92,14 @@ public:
void serial(NLMISC::IStream &stream) throw (NLMISC::EStream);
};
struct CFileRemoved
{
public:
uint32 Lost; // The time when it was noticed the file was removed.
void serial(NLMISC::IStream &stream) throw (NLMISC::EStream);
};
typedef CCallback<void, const std::string &/*filePath*/, const CFileStatus &/*fileStatus*/, bool /*success*/> TFileStatusCallback;
/**

Write Preview
Loading…
Cancel
Save