Added: #1440 Read status and remove metadata. Use a more sane standardizePath function

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/database_status.cpp

@@ -57,7 +57,7 @@ bool isInRootDirectoryFast(std::string &rootDirectoryName, std::string &rootDire
 	{
 		rootDirectoryName = rootDirectories.asString(i);
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
-		rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		rootDirectoryPath = standardizePath(dir.asString(), true);
 		if (path.find(rootDirectoryPath) == 0) return true;
 	}
 	return false;
@@ -68,13 +68,13 @@ bool isInSheetsDirectoryFast(std::string &sheetDirectoryName, std::string &sheet
 	{
 		sheetDirectoryName = "WorkspaceDfnDirectory";
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(sheetDirectoryName);
-		sheetDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		sheetDirectoryPath = standardizePath(dir.asString(), true);
 		if (path.find(sheetDirectoryPath) == 0) return true;
 	}
 	{
 		sheetDirectoryName = "WorkspaceSheetDirectory";
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(sheetDirectoryName);
-		sheetDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		sheetDirectoryPath = standardizePath(dir.asString(), true);
 		if (path.find(sheetDirectoryPath) == 0) return true;
 	}
 	return false;
@@ -108,7 +108,7 @@ std::string dropWorkspaceDirectoryFast(const std::string &path)
 
 std::string getMetaFilePath(const std::string &path, const std::string &dotSuffix)
 {
-	std::string stdPath = CPath::standardizePath(path, false);
+	std::string stdPath = standardizePath(path, false);
 	if (isInWorkspaceDirectoryFast(stdPath))
 	{
 		// TODO_TEST
@@ -197,6 +197,8 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
 	m_StatusMutex.lock_shared();
 	if (CFile::fileExists(statusPath))
 	{
+		nlassert(!CFile::isDirectory(filePath));
+		
 		CIFile ifs(statusPath, false);
 		fileStatus.serial(ifs);
 		ifs.close();
@@ -221,7 +223,9 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
 
 bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths)
 {
-	nlassert(false); // i don't think this is used? (yet?) (maybe for slave?)
+	// ARE THE PATHS UNMACRO'D OR NOT???
+
+	// nlassert(false); // i don't think this is used? (yet?) (maybe for slave?)
 	// probably just some ghost code spooking around.
 
 	// this code shall be used by the slave to get all the infos
@@ -229,9 +233,101 @@ bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStat
 
 	for (std::vector<std::string>::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it)
 	{
+		std::string path = *it; // assume already unmacro'd!
+		
+		if (CFile::isExists(path))
+		{
+			if (CFile::isDirectory(path)) // perhaps it is possible to check only on / assuming it's properly formatted!!
+			{
+				nlassert(path.find_last_of('/') == path.size() - 1); // ensure sanity
+
+				// std::string dirPath = standardizePath(path, true);
+				std::string &dirPath = path;
+				std::vector<std::string> dirContents;
+
+				CPath::getPathContent(dirPath, false, true, true, dirContents);
+				
+				for (std::vector<std::string>::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it)
+				{
+					const std::string &subPath = *it;
+					
+					CFileStatus fs;
+					if (!getFileStatus(fs, subPath))
+						return false; // bad status, data corruption // TODO_PROCESS_ERROR
+					fileStatusMap[subPath] = fs; // i'll assume macropath might be easiest
 					
+					// TODO_PROCESS_ERROR_EXIT
+					if (g_IsExiting)
+						return false;
 				}
-	return false; // it fails of course
+
+				std::string dirPathMeta = getMetaFilePath(dirPath, "");
+				std::vector<std::string> dirContentsMeta;
+
+				CPath::getPathContent(dirPathMeta, false, false, true, dirContentsMeta);
+
+				for (std::vector<std::string>::iterator it = dirContentsMeta.begin(), end = dirContentsMeta.end(); it != end; ++it)
+				{
+					const std::string &subPath = *it;
+					
+					std::string::size_type removePos = subPath.find(PIPELINE_DATABASE_REMOVE_SUFFIX);
+
+					if (removePos != std::string::npos)
+					{
+						std::string subPathFilename = CFile::getFilename(subPath.substr(0, removePos));
+						std::string subPathOrig = dirPath + subPathFilename;
+						
+						nldebug("Found remove meta for file '%s'!", subPathOrig.c_str());
+						// Read the removed tag.
+						std::string removedTagFile = dirPathMeta + subPathFilename + PIPELINE_DATABASE_REMOVE_SUFFIX;
+						CFileRemove fr;
+						CIFile ifr(removedTagFile, false);
+						fr.serial(ifr);
+						ifr.close();
+
+						fileRemoveMap[subPathOrig] = fr;
+					}
+					
+					// TODO_PROCESS_ERROR_EXIT
+					if (g_IsExiting)
+						return false;
+				}
+			}
+			else
+			{
+				CFileStatus fs;
+				if (!getFileStatus(fs, path))
+					return false; // bad status, data corruption // TODO_PROCESS_ERROR
+				fileStatusMap[path] = fs; // i'll assume macropath might be easiest
+			}
+		}
+		else
+		{
+			// TODO_PROCESS_WARNING
+			nlwarning("Requesting status on file or directory '%s' that does not exist!", path.c_str());
+			CFileRemove fr;
+			std::string removedTagFile = getMetaFilePath(path, PIPELINE_DATABASE_REMOVE_SUFFIX);
+			if (CFile::isExists(removedTagFile))
+			{
+				// file existed before
+				CIFile ifr(removedTagFile, false);
+				fr.serial(ifr);
+				ifr.close();
+			}
+			else
+			{
+				// file never existed or is directory
+				fr.Lost = 0;
+			}
+			fileRemoveMap[path] = fr;
+		}
+		// TODO_PROCESS_ERROR_EXIT
+		if (g_IsExiting)
+			return false;
+	}
+	
+	// no issues occured apparently
+	return true;
 }
 
 namespace {
@@ -436,6 +532,11 @@ public:
 void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, const std::string &dir, bool recurse);
 void updatePathStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, const std::string &subPath, bool recurse, bool recurseOnce)
 {
+	if (!CFile::isExists(subPath))
+	{
+		nlwarning("Updating status on path '%s' that does not exist!", subPath.c_str());
+		return;
+	}
 	if (CFile::isDirectory(subPath)) // if the file is a directory!
 	{
 		if (recurse || recurseOnce)
@@ -460,7 +561,7 @@ void updatePathStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, cons
 
 void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater, const std::string &dir, bool recurse)
 {
-	std::string dirPath = CPath::standardizePath(dir, true);
+	std::string dirPath = standardizePath(dir, true);
 	std::vector<std::string> dirContents;
 
 	CPath::getPathContent(dirPath, false, true, true, dirContents);
@@ -554,7 +655,7 @@ void CDatabaseStatus::updateDatabaseStatus(const CCallback<void> &callback)
 	{
 		std::string rootDirectoryName = rootDirectories.asString(i);
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
-		std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		std::string rootDirectoryPath = standardizePath(dir.asString(), true);
 		paths.push_back(rootDirectoryPath);
 	}
 	updateDatabaseStatus(callback, paths, false, true);

code/nel/tools/pipeline/service/pipeline_project.cpp

@@ -76,11 +76,11 @@ bool CPipelineProject::getValue(std::string &result, const std::string &name)
 	std::string typComment = valueElm->getType()->getComment();
 	if (typComment == "PIPELINE_PATH")
 	{
-		parsed = NLMISC::CPath::standardizePath(parsed, false);
+		parsed = standardizePath(parsed, false);
 	}
 	else if (typComment == "PIPELINE_PATH_ENDSLASH")
 	{
-		parsed = NLMISC::CPath::standardizePath(parsed, true);
+		parsed = standardizePath(parsed, true);
 	}
 	result = parsed;
 	return true;
@@ -123,11 +123,11 @@ bool CPipelineProject::getValues(std::vector<std::string> &resultAppend, const s
 		std::string typComment = valueElm->getType()->getComment();
 		if (typComment == "PIPELINE_PATH")
 		{
-			parsed = NLMISC::CPath::standardizePath(parsed, false);
+			parsed = standardizePath(parsed, false);
 		}
 		else if (typComment == "PIPELINE_PATH_ENDSLASH")
 		{
-			parsed = NLMISC::CPath::standardizePath(parsed, true);
+			parsed = standardizePath(parsed, true);
 		}
 		resultAppend.push_back(parsed);
 	}

code/nel/tools/pipeline/service/pipeline_service.cpp

@@ -87,17 +87,17 @@ std::string unMacroPath(const std::string &path)
 	{
 		std::string rootDirectoryName = rootDirectories.asString(i);
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
-		std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		std::string rootDirectoryPath = standardizePath(dir.asString(), true);
 		std::string macroName = "[$" + rootDirectoryName + "]";
 		strFindReplace(result, macroName, rootDirectoryPath);
 	}
 
-	return CPath::standardizePath(result, false);
+	return standardizePath(result, false);
 }
 
 std::string macroPath(const std::string &path)
 {
-	std::string result = CPath::standardizePath(path, false);
+	std::string result = standardizePath(path, false);
 
 	strFindReplace(result, g_WorkDir, PIPELINE_MACRO_WORKSPACE_DIRECTORY "/");
 	
@@ -106,7 +106,7 @@ std::string macroPath(const std::string &path)
 	{
 		std::string rootDirectoryName = rootDirectories.asString(i);
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootDirectoryName);
-		std::string rootDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		std::string rootDirectoryPath = standardizePath(dir.asString(), true);
 		std::string macroName = "[$" + rootDirectoryName + "]";
 		strFindReplace(result, rootDirectoryPath,  macroName);
 	}
@@ -117,6 +117,48 @@ std::string macroPath(const std::string &path)
 	return result;
 }
 
+std::string standardizePath(const std::string &path, bool addFinalSlash)
+{
+	// check empty path
+	if (path.empty())
+		return "";
+
+	std::string newPath;
+	newPath.resize(path.size() + 1);
+
+	std::string::size_type j = 0;
+	for (std::string::size_type i = 0; i < path.size(); ++i)
+	{
+		if (path[i] == '\\' || path[i] == '/')
+		{
+			if (j <= 1 && path[i] == '\\')
+			{
+				// for windows network
+				newPath[j] = '\\';
+				++j;
+			}
+			else if (j == 0 || newPath[j - 1] != '/')
+			{
+				newPath[j] = '/';
+				++j;
+			}
+		}
+		else
+		{
+			newPath[j] = path[i];
+			++j;
+		}
+	}
+	newPath[j] = 0;
+	newPath.resize(j);
+
+	// add terminal slash
+	if (addFinalSlash && newPath[path.size()-1] != '/')
+		newPath += '/';
+
+	return newPath;
+}
+
 extern void module_pipeline_master_forceLink();
 extern void module_pipeline_slave_forceLink();
 
@@ -354,9 +396,9 @@ namespace {
 
 void initSheets()
 {
-	std::string dfnDirectory = CPath::standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString(), true);
+	std::string dfnDirectory = standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString(), true);
 	IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").setAsString(dfnDirectory);
-	std::string sheetDirectory = CPath::standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString(), true);
+	std::string sheetDirectory = standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString(), true);
 	IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").setAsString(sheetDirectory);
 	
 	if (!CFile::isDirectory(dfnDirectory)) nlerror("'WorkspaceDfnDirectory' does not exist! (%s)", dfnDirectory.c_str());
@@ -372,7 +414,7 @@ void initSheets()
 	{
 		std::string rootName = georgesDirectories.asString(i);
 		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(rootName);
-		std::string dirName = CPath::standardizePath(dir.asString(), true);
+		std::string dirName = standardizePath(dir.asString(), true);
 		if (!CFile::isDirectory(dirName)) nlerror("'%s' does not exist! (%s)", rootName.c_str(), dirName.c_str());
 		CPath::addSearchPath(dirName, true, false);
 	}
@@ -508,10 +550,10 @@ public:
 
 		s_InfoFlags->addFlag("INIT");
 
-		//g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
+		//g_DatabaseDirectory = standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
 		//if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str());
 		//ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory);
-		g_WorkDir = CPath::standardizePath(ConfigFile.getVar("WorkspaceDirectory").asString(), true);
+		g_WorkDir = standardizePath(ConfigFile.getVar("WorkspaceDirectory").asString(), true);
 		if (!CFile::isDirectory(g_WorkDir)) nlerror("'WorkspaceDirectory' does not exist! (%s)", g_WorkDir.c_str());
 		ConfigFile.getVar("WorkspaceDirectory").setAsString(g_WorkDir);
 
@@ -521,7 +563,7 @@ public:
 		{
 			std::string rootName = rootDirectories.asString(i);
 			CConfigFile::CVar &dir = ConfigFile.getVar(rootName);
-			std::string dirName = CPath::standardizePath(dir.asString(), true);
+			std::string dirName = standardizePath(dir.asString(), true);
 			if (!CFile::isDirectory(dirName)) nlerror("'%s' does not exist! (%s)", rootName.c_str(), dirName.c_str());
 			dir.setAsString(dirName);
 		}

code/nel/tools/pipeline/service/pipeline_service.h

@@ -69,6 +69,9 @@ std::string unMacroPath(const std::string &path);
 /// Macros a path, and standardizes it in advance.
 std::string macroPath(const std::string &path);
 
+/// Based on CPath::standardizePath, but strips double slashes as well
+std::string standardizePath(const std::string &path, bool addFinalSlash);
+
 bool tryRunnableTask(const std::string &stateName, NLMISC::IRunnable *task);
 void endedRunnableTask();