diff --git a/code/nel/tools/pipeline/service/database_status.cpp b/code/nel/tools/pipeline/service/database_status.cpp index dda84a3cb..1fbfd7b8d 100644 --- a/code/nel/tools/pipeline/service/database_status.cpp +++ b/code/nel/tools/pipeline/service/database_status.cpp @@ -74,7 +74,7 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string & m_StatusMutex.unlock_shared(); return seemsValid; } - +/* bool CDatabaseStatus::getFileStatus(std::map &fileStatusMap, std::map &fileRemoveMap, const std::vector &paths) { nlassert(false); // not used @@ -193,7 +193,7 @@ bool CDatabaseStatus::getFileStatus(std::map &fileStat // no issues occured apparently return true; } - +*/ bool CDatabaseStatus::getRemoved(std::map &fileRemoveMap, const std::vector &paths) { for (std::vector::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it) @@ -402,8 +402,33 @@ public: } }; +struct CRememberTheFileStatus +{ + CFileStatus FileStatus; + bool Success; + void cb(const std::string &/*filePath*/, const CFileStatus &fileStatus, bool success) + { + FileStatus = fileStatus; + Success = success; + } +}; + } /* anonymous namespace */ +/// Updates a file status synchronously. Returns the result in metaStatus. +bool CDatabaseStatus::updateFileStatus(CFileStatus &metaStatus, const std::string &filePath) +{ + CRememberTheFileStatus result; + CUpdateFileStatus *ufs = new CUpdateFileStatus(); + ufs->StatusMutex = &m_StatusMutex; + ufs->FilePath = filePath; + ufs->Callback = TFileStatusCallback(&result, &CRememberTheFileStatus::cb); + ufs->run(); + ufs = NULL; // deleted by run + metaStatus = result.FileStatus; + return result.Success; +} + IRunnable *CDatabaseStatus::updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath) { /*if (!g_IsMaster) @@ -620,10 +645,10 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, // <-- END REMOVE } -} /* anonymous namespace */ - void dummyFileStatusCallback(const std::string &/*filePath*/, const CFileStatus &/*fileStatus*/, bool /*success*/) { } +} /* anonymous namespace */ + void CDatabaseStatus::updateDatabaseStatus(const CCallback &callback) { /*if (!g_IsMaster) diff --git a/code/nel/tools/pipeline/service/database_status.h b/code/nel/tools/pipeline/service/database_status.h index d762cfaf3..88168ce8a 100644 --- a/code/nel/tools/pipeline/service/database_status.h +++ b/code/nel/tools/pipeline/service/database_status.h @@ -69,14 +69,16 @@ public: /// Tries to read the last file status. Return false if the status is invalid. Call updateFileStatus if the result is false to update asynchronously. bool getFileStatus(CFileStatus &fileStatus, const std::string &filePath) const; - /// Updates the file status asynchronously. The new file status is broadcast to clients and slaves afterwards. Warning: If g_IsExiting during callback then update likely did not happen. + /// Updates a file status synchronously. Returns the result in metaStatus. filepPath is NOT a macro path. + bool updateFileStatus(CFileStatus &metaStatus, const std::string &filePath); + /// Updates the file status asynchronously. Warning: If g_IsExiting during callback then update likely did not happen. NLMISC::IRunnable *updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath); /// Runs an update of the complete {{DatabaseDirectory}} status asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). void updateDatabaseStatus(const CCallback &callback); /// Runs an update of the file status of given paths asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). Do NOT use the wait parameter. Do NOT use recurse, please. Recurse directories beforehand. Paths may contain db and pl macros. void updateDatabaseStatus(const CCallback &callback, const TFileStatusCallback &fileStatusCallback, const std::vector &paths, bool wait = false, bool recurse = false); /// Gets the last file statuses of given paths in a map. Directories are scanned for files, non recursively. Returns false if one of the statuses is bad (not updated; file changed inbetween). Considered as build error. - bool getFileStatus(std::map &fileStatusMap, std::map &fileRemoveMap, const std::vector &paths); + // bool getFileStatus(std::map &fileStatusMap, std::map &fileRemoveMap, const std::vector &paths); /// Gets all removed files bool getRemoved(std::map &fileRemoveMap, const std::vector &paths); diff --git a/code/nel/tools/pipeline/service/metadata_storage.h b/code/nel/tools/pipeline/service/metadata_storage.h index 60a890b3f..afd0718a3 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.h +++ b/code/nel/tools/pipeline/service/metadata_storage.h @@ -178,8 +178,8 @@ public: // static void eraseRemove(const std::string &path); static std::string getDependPath(const std::string &file); - static bool readDepend(CFileDepend &status, const std::string &metaPath); - static void writeDepend(const CFileDepend &status, const std::string &metaPath); + static bool readDepend(CFileDepend &depend, const std::string &metaPath); + static void writeDepend(const CFileDepend &depend, const std::string &metaPath); // Pathname for result metadata is like .../project.projectname.meta/pluginname.result static std::string getResultPath(const std::string &projectName, const std::string &pluginName); diff --git a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp index 3088b5828..138d88fb2 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp @@ -713,6 +713,7 @@ public: for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) { const std::string &path = *it; + // FIXME: What if an input file does not exist? Deleted long ago? Just print a warning (& notify terminal), keep calm, and carry on. nlassert(!CFile::isDirectory(path)); // All input are files! Coding error otherwise, because should already be checked. std::string metaOutputPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler); nlassert(CFile::fileExists(metaOutputPath)); // Coding error otherwise, already must have checked beforehand that they all exist. @@ -844,134 +845,108 @@ public: // Sanity check of all the output paths // These must all be files, no directories allowed + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (path[path.size() - 1] == '/') // isDirectory + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory"; + return false; // Error, cannot rebuild. + } + } // Check if any of the output paths are part of the removed output files - // If so, rebuild - // Check if any of the output files don't exist - // If so, rebuild - - // bool outputChanged = Check if any of the output paths are part of the changed output files - - // Check the .depend files of all the output files // also check that they exist :) - // If outputChanged - // Compare the output checksum with the cached output checksum - // If inputModified - // Compare the input checksums with the cached input checksums - // (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed) - } - - /// Returns false if the file does not need to be built, or if an error occured. - /// Must verify needsExit() afterwards. - /// Input paths may be files or directories. - /// Output paths can ONLY be files! - bool needsToBeRebuiltOld(const std::vector &inputPaths, const std::vector &outputPaths) - { - // TODO: REWRITE THIS - - if (m_SubTaskResult != FINISH_SUCCESS) - return false; // Cannot continue on previous failure. - - m_SubTaskResult = FINISH_NOT; - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) { const std::string &path = *it; - if (path[path.size() - 1] == '/') // isDirectory + if (m_ListOutputRemoved.find(*it) != m_ListOutputRemoved.end()) { - // Check if this directory is in the dependencies. - if (!isDirectoryDependency(path)) - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies"; - return false; // Error, cannot rebuild. - } - // Check if any files added/changed/removed are part of this directory (slow). - for (std::set::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr) - { - const std::string &pathr = *it; - if ((pathr.size() > path.size()) - && (pathr.substr(0, path.size()) == path) // inside the path - && (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory - { - nldebug("Found added '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - for (std::set::const_iterator itr = m_ListInputChanged.begin(), endr = m_ListInputChanged.end(); itr != endr; ++itr) - { - const std::string &pathr = *it; - if ((pathr.size() > path.size()) - && (pathr.substr(0, path.size()) == path) // inside the path - && (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory - { - nldebug("Found changed '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - for (std::set::const_iterator itr = m_ListInputRemoved.begin(), endr = m_ListInputRemoved.end(); itr != endr; ++itr) - { - const std::string &pathr = *it; - if ((pathr.size() > path.size()) - && (pathr.substr(0, path.size()) == path) // inside the path - && (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory - { - nldebug("Found removed '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } + // If so, rebuild + nldebug("Output file '%s' has been removed, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. } - else // isFile + } + + // Check if any of the output files don't exist + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + // If so, rebuild + const std::string &path = *it; + if (!CFile::isExists(path)) { - // Check if this file is in the dependencies. - if (!isFileDependency(path)) - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; - return false; // Error, cannot rebuild. - } - // Check if this file is in added/changed/removed. - if (m_ListInputAdded.find(path) != m_ListInputAdded.end() - || m_ListInputChanged.find(path) != m_ListInputChanged.end() - || m_ListInputRemoved.find(path) != m_ListInputRemoved.end()) - { - // Found! - nldebug("Found added/changed/removed input file '%s', rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } + // If so, rebuild + nldebug("Output file '%s' does not exist, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. } } + + // Check if any of the output paths are part of the changed output files + bool outputChanged = false; for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) { const std::string &path = *it; - if (path[path.size() - 1] == '/') // isDirectory + if (m_ListOutputChanged.find(*it) != m_ListOutputChanged.end()) { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory"; - return false; // Error, cannot rebuild. + nldebug("Output file '%s' has been changed", path.c_str()); + outputChanged = true; + break; } - if (m_ListOutputRemoved.find(path) != m_ListOutputRemoved.end()) + } + + if (!outputChanged && !inputModified) + { + nlerror("Should never reach this, this must have been cought earlier normally"); + } + + // Check the .depend files of all the output files // also check that they exist :) + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + std::string metaDependPath = CMetadataStorage::getDependPath(path); + CFileDepend metaDepend; + if (!CMetadataStorage::readDepend(metaDepend, metaDependPath)) { - nldebug("Found removed output file '%s', rebuild", path.c_str()); + nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str()); m_SubTaskResult = FINISH_SUCCESS; - return true; + return true; // Rebuild. } - if (m_ListOutputChanged.find(path) != m_ListOutputChanged.end()) + else { - nlwarning("Changed output files not implemented yet. Previous build was likely incomplete. Please wipe the output directory. Rebuilding anyways"); - // For now always rebuild and don't check if the output file is up-to-date from a previous incomplete build. - nldebug("Found changed output file '%s', rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; - //m_SubTaskResult = FINISH_ERROR; - //m_SubTaskErrorMessage = std::string("Changed output files not implemented yet. Previous build was likely incomplete. Please wipe the output directory"); - //return false; // Error, cannot rebuild. + if (outputChanged) + { + // Compare the output checksum with the status output checksum + CFileStatus metaStatus; + if (!CDatabaseStatus::updateFileStatus(metaStatus, path)) + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "'"; + return false; // Error, cannot rebuild. + } + else + { + if (metaStatus.CRC32 != metaDepend.CRC32) + { + nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + } + if (inputModified) + { + // Compare the input checksums with the cached input checksums + + } } } + + // (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed) + nldebug("No differences found, no rebuild needed"); m_SubTaskResult = FINISH_SUCCESS; - return false; // Does not need rebuild. + return false; // Rebuild not necessary. } /// Set the exit message, exit the plugin immediately afterwards.