Merge branch 'develop' into feature/develop-atys

nel/include/nel/3d/async_file_manager_3d.h

@@ -40,7 +40,7 @@ class CTextureFile;
 class CAsyncFileManager3D
 {
 
-	NLMISC_SAFE_SINGLETON_DECL(CAsyncFileManager3D);
+	NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CAsyncFileManager3D);
 	CAsyncFileManager3D();
 public:
 

nel/include/nel/georges/form.h

@@ -22,6 +22,7 @@
 
 #include "nel/misc/types_nl.h"
 #include "nel/georges/u_form.h"
+#include "form_dfn.h"
 #include "form_elm.h"
 #include "header.h"
 
@@ -124,6 +125,10 @@ private:
 
 	// The form filename
 	std::string			_Filename;
+
+	// The dfn
+	NLMISC::CSmartPtr<CFormDfn> _Dfn;
+
 };
 
 } // NLGEORGES

nel/include/nel/georges/form_elm.h

@@ -282,6 +282,10 @@ public:
 		{
 			Element = NULL;
 		}
+		~CFormElmStructElm()
+		{
+			nlassert(!Element);
+		}
 
 		std::string		Name;
 		CFormElm*		Element;
@@ -400,6 +404,10 @@ public:
 		{
 			Element = NULL;
 		}
+		~CElement ()
+		{
+			nlassert(!Element);
+		}
 
 		std::string		Name;
 		CFormElm*		Element;

nel/include/nel/gui/action_handler.h

@@ -69,7 +69,7 @@ namespace NLGUI
 
 		static CAHManager* getInstance()
 		{
-			if (_GlobalInstance == NULL)
+			if (_GlobalInstance == NULL && !s_Deleted)
 				_GlobalInstance = new CAHManager;
 			return _GlobalInstance;
 		}
@@ -134,6 +134,19 @@ namespace NLGUI
 		static CAHManager *_GlobalInstance;
 		static bool editorMode;
 
+		class CDeleter
+		{
+		public:
+			~CDeleter()
+			{
+				delete _GlobalInstance;
+				_GlobalInstance = NULL;
+				s_Deleted = true;
+			}
+		};
+		static CDeleter s_Deleter;
+		static bool s_Deleted;
+
 	};
 
 	/// Ah name must all be lower case

nel/include/nel/gui/reflect.h

@@ -42,9 +42,13 @@ namespace NLGUI
 					 UInt32,
 					 Float,
 					 String,
+#ifdef RYZOM_LUA_UCSTRING
 					 UCString,
+#endif
 					 StringRef,
+#ifdef RYZOM_LUA_UCSTRING
 					 UCStringRef,
+#endif
 					 RGBA,
 					 LuaMethod
 					}; // other types will be added when needed
@@ -79,9 +83,13 @@ namespace NLGUI
 			TGetUInt32		GetUInt32;
 			TGetFloat		GetFloat;
 			TGetString		GetString;
+#ifdef RYZOM_LUA_UCSTRING
 			TGetUCString	GetUCString;
+#endif
 			TGetStringRef	GetStringRef;
+#ifdef RYZOM_LUA_UCSTRING
 			TGetUCStringRef	GetUCStringRef;
+#endif
 			TGetRGBA		GetRGBA;
 			TLuaMethod		GetLuaMethod; // lua method can only be obtained, not written ...
 		} GetMethod;
@@ -92,7 +100,9 @@ namespace NLGUI
 			TSetUInt32		SetUInt32;
 			TSetFloat		SetFloat;
 			TSetString		SetString;
+#ifdef RYZOM_LUA_UCSTRING
 			TSetUCString	SetUCString;
+#endif
 			TSetRGBA		SetRGBA;
 		} SetMethod;
 		// name of the property

nel/include/nel/misc/common.h

@@ -249,11 +249,16 @@ void appendToLower(std::string &res, const char *str, ptrdiff_t &i);
 void appendToLower(std::string &res, const std::string &str, ptrdiff_t &i);
 void appendToUpper(std::string &res, const char *str, ptrdiff_t &i);
 void appendToUpper(std::string &res, const std::string &str, ptrdiff_t &i);
+void appendToTitle(std::string &res, const char *str, ptrdiff_t &i);
+void appendToTitle(std::string &res, const std::string &str, ptrdiff_t &i);
 
 /** UTF-8 case insensitive compare */
 int compareCaseInsensitive(const char *a, const char *b);
 int compareCaseInsensitive(const char *a, size_t lenA, const char *b, size_t lenB);
 inline int compareCaseInsensitive(const std::string &a, const std::string &b) { return compareCaseInsensitive(&a[0], a.size(), &b[0], b.size()); }
+inline bool ltCaseInsensitive(const std::string &a, const std::string &b) { return compareCaseInsensitive(&a[0], a.size(), &b[0], b.size()) < 0; }
+std::string	toCaseInsensitive(const char *str); // UTF-8, case-insensitive toLower
+std::string	toCaseInsensitive(const std::string &str); // UTF-8, case-insensitive toLower
 
 /** ASCII to lowercase. Useful for internal identifiers.
 * Characters outside of the 7-bit ASCII space, and control characters, are replaced.

nel/include/nel/misc/deep_ptr.h

@@ -0,0 +1,83 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_DEEP_PTR_H
+#define NLMISC_DEEP_PTR_H
+
+#include <nel/misc/types_nl.h>
+
+namespace NLMISC {
+
+/// Pointer template with deep copy and move semantics
+template<class T>
+class CDeepPtr
+{
+public:
+	NL_FORCE_INLINE CDeepPtr() : m(NULL) { } //< Null
+	NL_FORCE_INLINE ~CDeepPtr() { delete m; }
+
+	NL_FORCE_INLINE CDeepPtr(const CDeepPtr &p) : m(p.m ? new T(*p) : NULL) { } //< Copy operator
+	NL_FORCE_INLINE CDeepPtr &operator=(const CDeepPtr &p) { if (p.m) { if (!m) m = new T(*p); else *m = *p; } else { delete m; m = NULL; } return *this; } //< Copy operator
+
+#ifdef NL_CPP14
+	NL_FORCE_INLINE CDeepPtr(CDeepPtr &&p) noexcept : m(p.m) { p.m = NULL; } //< Move operator
+	NL_FORCE_INLINE CDeepPtr &operator=(CDeepPtr &&p) noexcept { delete m; m = p.m; p.m = NULL; return *this; } //< Move operator
+#endif
+
+	NL_FORCE_INLINE CDeepPtr(T *p) : m(p) { } //< Initializer
+	NL_FORCE_INLINE CDeepPtr &operator=(T *p) { delete m; m = p; return *this; } //< Initializer
+
+	NL_FORCE_INLINE bool operator==(const CDeepPtr &p) const { return /* (m == p.m) || */ (m && p.m && *m == *p); }
+	NL_FORCE_INLINE bool operator!=(const CDeepPtr &p) const { return !(*this == p); }
+
+	NL_FORCE_INLINE bool operator==(const T *p) const { return (m == p) || (m && p && *m == *p); }
+	NL_FORCE_INLINE bool operator!=(const T *p) const { return !(*this == p); }
+
+	NL_FORCE_INLINE bool operator==(const T &p) const { return (m == &p) || (m && *m == p); }
+	NL_FORCE_INLINE bool operator!=(const T &p) const { return !(*this == p); }
+
+	NL_FORCE_INLINE bool operator==(long int p) const { return (*this == (const T *)(ptrdiff_t)p); } //< == NULL
+	NL_FORCE_INLINE bool operator!=(long int p) const { return (*this != (const T *)(ptrdiff_t)p); } //< != NULL
+
+	NL_FORCE_INLINE bool operator==(int p) const { return (*this == (const T *)(ptrdiff_t)p); } //< == 0
+	NL_FORCE_INLINE bool operator!=(int p) const { return (*this != (const T *)(ptrdiff_t)p); } //< != 0
+
+#ifdef NL_CPP14
+	NL_FORCE_INLINE bool operator==(nullptr_t p) const { return (*this == (const T *)(ptrdiff_t)p); } //< == nullptr
+	NL_FORCE_INLINE bool operator!=(nullptr_t p) const { return (*this != (const T *)(ptrdiff_t)p); } //< != nullptr
+#endif
+
+	NL_FORCE_INLINE T &operator*() { return *m; }
+	NL_FORCE_INLINE const T &operator*() const { return *m; }
+	NL_FORCE_INLINE T *operator->() { return m; }
+	NL_FORCE_INLINE const T *operator->() const { return m; }
+
+	NL_FORCE_INLINE operator bool() const { return m; }
+	NL_FORCE_INLINE bool operator!() const { return !m; }
+
+	NL_FORCE_INLINE T *ptr() { return m; }
+	NL_FORCE_INLINE const T *ptr() const { return m; }
+
+private:
+	T *m;
+
+};
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_DEEP_PTR_H */
+
+/* end of file */

nel/include/nel/misc/factory.h

@@ -58,12 +58,8 @@ public:
 	static CFactory &instance()
 	{
 		// Singleton instance pointer.
-		static CFactory	*instance = NULL;
+		static CFactory	instance;
-		if (!instance)
+		return instance;
-		{
-			instance = new CFactory();
-		}
-		return *instance;
 	}
 
 	/** Register a factorable object in the factory.
@@ -185,12 +181,8 @@ public:
 	/// Get the singleton instance reference.
 	static CFactoryIndirect &instance()
 	{
-		static CFactoryIndirect	*instance = NULL;
+		static CFactoryIndirect	instance;
-		if (!instance)
+		return instance;
-		{
-			instance = new CFactoryIndirect();
-		}
-		return *instance;
 	}
 
 	void registerClass(const KeyType &key, IFactoryIndirectRegister<BaseFactoryClass> *factoryRegister)

nel/include/nel/misc/fixed_size_allocator.h

@@ -107,6 +107,7 @@ private:
 	uint	_NumBlockPerChunk;
 	//
 	uint	_NumAlloc;
+	uint8	*_SpareMem;
 };
 
 } // NLMISC

nel/include/nel/misc/string_view.h

@@ -0,0 +1,131 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STRING_VIEW_H
+#define NLMISC_STRING_VIEW_H
+
+#include <nel/misc/types_nl.h>
+#include <string>
+
+#ifdef NL_CPP14
+using namespace std::string_literals;
+#ifdef NL_CPP17
+#include <string_view>
+using namespace std::string_view_literals;
+#endif
+#endif
+
+#ifdef NL_CPP14
+/// Obtain an std::string from a string literal.
+#define nlstr(strLit) (strLit##s)
+#else
+/// Obtain an std::string from a string literal.
+#define nlstr(strLit) (std::string(strLit))
+#endif
+
+#ifdef NL_CPP17
+/// Obtain a string view from a string literal.
+#define nlsv(strLit) (strLit##sv)
+/// Obtain an std::string from a string view.
+#define nlsvs(strView) (std::string(strView))
+#else
+/// Obtain a string view from a string literal.
+#define nlsv(strLit) (CStringView(strLit, ::strlen(strLit)))
+/// Obtain an std::string from a string view.
+#define nlsvs(strView) (std::string(strView.data(), strView.size()))
+#endif
+
+/// Obtain a temporary C-string from a string view. Use directly in argument, do not store.
+#define nlsvc(strView) ((strView.data()[strView.size()]) ? nlsvs(strView).c_str() : strView.data())
+
+namespace NLMISC {
+
+/// String view literals allow bypassing allocation and strlen calls.
+/// CStringView is a 100% drop-in replacement for (const char *str, size_t len) tuples. It's a non-owning reference.
+/// Always use `CStringView` where previously `const std::string &` would have been used. It avoids accidental copy.
+/// Gotcha: CStringView doesn't need to end with \0, so there's no guarantee with functions that expect \0 terminated strings, 
+/// use the `nlsvc` macro to get a temporary C-string from a CStringView.
+/// Use the `nlsv` macro to get a CStringView from a string literal.
+/// Use the `nlstr` macro to get an std::string from a string literal.
+/// Use the `nlsvs` macro to get an std::string from a CStringView.
+#ifdef NL_CPP17
+typedef std::string_view CStringView;
+#else
+class CStringView
+{
+public:
+	CStringView(const std::string &str) : m_Str(&str[0]), m_Len(str.size()) {}
+	CStringView(const char *const str, const size_t len) : m_Str(str), m_Len(len) {}
+	CStringView(const char *const str) : m_Str(str), m_Len(sizeof(str)) {}
+
+	inline const char *data() const { return m_Str; }
+	inline size_t length() const { return m_Len; }
+	inline size_t size() const { return m_Len; }
+
+	inline CStringView substr(const size_t offset, const size_t count = -1) { return CStringView(m_Str + offset, std::min(m_Len - offset, count)); }
+
+	inline bool operator==(const CStringView o) { if (m_Len != o.m_Len) return false; return memcmp(m_Str, o.m_Str, m_Len) == 0; }
+	inline bool operator!=(const CStringView o) { if (m_Len != o.m_Len) return true; return memcmp(m_Str, o.m_Str, m_Len) != 0; }
+
+	struct const_iterator
+	{
+	public:
+		const_iterator() : m_Addr(NULL) { }
+
+		inline void operator++()
+		{
+			++m_Addr;
+		}
+		inline void operator+=(ptrdiff_t v)
+		{
+			m_Addr += v;
+		}
+		inline void operator--()
+		{
+			--m_Addr;
+		}
+		inline void operator-=(ptrdiff_t v)
+		{
+			m_Addr -= v;
+		}
+		inline bool operator!=(const const_iterator &o) const { return m_Addr != o.m_Addr; }
+		inline bool operator==(const const_iterator &o) const { return m_Addr == o.m_Addr; }
+		inline const char &operator*() const { return *m_Addr; }
+
+	private:
+		friend class CStringView;
+		inline const_iterator(const char *addr) : m_Addr(addr) {}
+		const char *m_Addr;
+
+	};
+
+	typedef const_iterator iterator;
+
+	iterator begin() const { return iterator(m_Str); }
+	inline iterator end() const { return iterator(m_Str + m_Len); }
+
+private:
+	const char *m_Str;
+	size_t m_Len;
+
+};
+#endif
+
+}
+
+#endif /* #ifndef NLMISC_STRING_VIEW_H */
+
+/* end of file */

nel/include/nel/misc/utf_string_view.h

@@ -18,9 +18,12 @@
 #define NLMISC_UTF_STRING_VIEW_H
 
 #include <nel/misc/types_nl.h>
-#include <nel/misc/ucstring.h>
+
 #include <string>
 
+#include <nel/misc/string_view.h>
+#include <nel/misc/ucstring.h>
+
 namespace NLMISC {
 
 class IStream;
@@ -40,6 +43,9 @@ public:
 	{
 		nlassert(len <= strlen(utf8Str));
 	}
+
+	inline CUtfStringView(CStringView utf8Str) : m_Str(utf8Str.data()), m_Size(utf8Str.size()), m_Iterator(utf8Iterator) {}
+
 #if defined(NL_OS_WINDOWS)
 	inline CUtfStringView(const wchar_t *utf16Str) : m_Str(utf16Str), m_Size(wcslen(utf16Str)), m_Iterator(utf16Iterator) {}
 	inline CUtfStringView(const wchar_t *utf16Str, size_t len): m_Str(utf16Str), m_Size(len), m_Iterator(utf16Iterator)
@@ -125,7 +131,8 @@ public:
 	inline bool empty() const { return !m_Size; }
 	const void *ptr() const { return m_Str; }
 
-	size_t count() const; // Slow count of UTF-32 characters
+	size_t count() const; //< Slow count of UTF-32 codepoints
+	ptrdiff_t offset(ptrdiff_t i); const //< Get byte offset by utf-32 codepoint index
 
 	inline CUtfStringView substr(const iterator &begin, const iterator &end) const
 	{
@@ -166,6 +173,6 @@ private:
 
 } /* namespace NLMISC */
 
-#endif /* #ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H */
+#endif /* #ifndef NLMISC_UTF_STRING_VIEW_H */
 
 /* end of file */

nel/src/3d/font_manager.cpp

@@ -146,8 +146,10 @@ void CFontManager::computeString (NLMISC::CUtfStringView sv,
 		{
 			// Creating font
 			k.Char = *it;
-			if (k.Char < 0x20)
+			if (k.Char < 0x20) // Control Characters
 				k.Char += 0x2400;
+			if (k.Char == 0x7F) // DEL
+				k.Char = 0x2421;
 			k.FontGenerator = fontGen;
 			k.Size = fontSize;
 			k.Embolden = embolden;

nel/src/3d/text_context_user.cpp

@@ -229,7 +229,7 @@ uint32 CTextContextUser::textPush(const char *format, ...)
 	char *str;
 	NLMISC_CONVERT_VARGS (str, format, NLMISC::MaxCStringSize);
 
-	return _TextContext.textPush(ucstring(str)) ;
+	return _TextContext.textPush(str) ;
 }
 uint32 CTextContextUser::textPush(NLMISC::CUtfStringView sv)
 {
@@ -340,7 +340,7 @@ void CTextContextUser::printfAt(float x, float y, const char * format, ...)
 	char *str;
 	NLMISC_CONVERT_VARGS (str, format, NLMISC::MaxCStringSize);
 
-	_TextContext.printAt(x, y, ucstring(str)) ;
+	_TextContext.printAt(x, y, str) ;
 	_DriverUser->restoreMatrixContext();
 }
 

nel/src/georges/form.cpp

@@ -191,6 +191,9 @@ void CForm::read (xmlNodePtr node, CFormLoader &loader, CFormDfn *dfn, const std
 	// Reset form
 	clean ();
 
+	// Save the dfn
+	_Dfn = dfn;
+
 	// Check node name
 	if ( ((const char*)node->name == NULL) || (strcmp ((const char*)node->name, "FORM") != 0) )
 	{

nel/src/georges/form_elm.cpp

@@ -1707,6 +1707,7 @@ CFormElmStruct::CFormElmStruct (CForm *form, CFormElm *parentNode, const CFormDf
 CFormElmStruct::~CFormElmStruct ()
 {
 	// Job done in clean()
+	clean();
 }
 
 // ***************************************************************************
@@ -1717,10 +1718,10 @@ void CFormElmStruct::clean ()
 	uint elm;
 	for (elm =0; elm<Elements.size(); elm++)
 	{
-		if (Elements[elm].Element)
 		delete Elements[elm].Element;
 		Elements[elm].Element = NULL;
 	}
+	Elements.clear();
 }
 
 // ***************************************************************************
@@ -2270,6 +2271,7 @@ CFormElmArray::CFormElmArray (CForm *form, const CFormDfn *formDfn, const CType
 CFormElmArray::~CFormElmArray ()
 {
 	// Job done in clean()
+	clean();
 }
 
 // ***************************************************************************
@@ -2280,8 +2282,8 @@ void CFormElmArray::clean ()
 	uint elm;
 	for (elm =0; elm<Elements.size(); elm++)
 	{
-		if (Elements[elm].Element)
 		delete Elements[elm].Element;
+		Elements[elm].Element = NULL;
 	}
 	Elements.clear ();
 }

nel/src/georges/form_loader.cpp

@@ -241,7 +241,7 @@ UForm *CFormLoader::loadForm (const std::string &filename)
 			name += ".dfn";
 
 			// Load the dfn
-			CFormDfn	*dfn = loadFormDfn (name, false);
+			CSmartPtr<CFormDfn> dfn = loadFormDfn (name, false);
 			if (dfn)
 			{
 				// Open the file

nel/src/gui/action_handler.cpp

@@ -42,6 +42,8 @@ namespace NLGUI
 	// ------------------------------------------------------------------------------------------------
 	CAHManager	*CAHManager::_GlobalInstance = NULL;
 	bool CAHManager::editorMode = false;
+	CAHManager::CDeleter CAHManager::s_Deleter;
+	bool CAHManager::s_Deleted = false;
 	// ------------------------------------------------------------------------------------------------
 
 	// ------------------------------------------------------------------------------------------------

nel/src/gui/group_editbox.cpp

@@ -54,6 +54,41 @@ namespace NLGUI
 	CGroupEditBox *CGroupEditBox::_MenuFather = NULL;
 	CGroupEditBox::IComboKeyHandler* CGroupEditBox::comboKeyHandler = NULL;
 
+	// For now, just trim unsupported codepoints to make emoji fallback to text form
+	static u32char supportedCodepoint(u32char c)
+	{
+		if (c >= 0xFE00 && c < 0xFE10)
+			return 0; // Variation Selectors, unsupported
+		else if (c >= 0xE0100 && c < 0xE01F0)
+			return 0; // Variation Selectors Supplement, unsupported
+		else if (c >= 0x200B && c < 0x2010)
+			return 0; // ZERO WIDTH JOINER, etcetera, unsupported
+		else if (c >= 0x2028 && c < 0x202F)
+			return 0; // PARAGRAPH SEPARATOR, etcetera, unsupported
+		else if (c >= 0x2060 && c < 0x2070)
+			return 0; // WORD JOINER, etcetera, unsupported
+		else if (c == 0xFEFF)
+			return 0; // BOM, unsupported
+		else if ((c & 0xFC00) == 0xD800)
+			return 0xFFFD; // UTF-16 surrogate, unmatched pair, invalid, set to replacement character
+		else if ((c & 0xFC00) == 0xDC00)
+			return 0xFFFD; // UTF-16 surrogate, unmatched pair, invalid, set to replacement character
+		return c;
+	}
+
+	// For now, just trim unsupported codepoints to make emoji fallback to text form
+	static ::u32string trimUnsupported(const ::u32string str)
+	{
+		::u32string res;
+		res.reserve(str.size());
+		for (::u32string::const_iterator it(str.begin()), end(str.end()); it != end; ++it)
+		{
+			u32char c = supportedCodepoint(*it);
+			if (c) // This also trims NUL
+				res.push_back(c);
+		}
+		return res;
+	}
 
 	// ----------------------------------------------------------------------------
 	NLMISC_REGISTER_OBJECT(CViewBase, CGroupEditBox, std::string, "edit_box");
@@ -850,7 +885,8 @@ namespace NLGUI
 	// ----------------------------------------------------------------------------
 	void CGroupEditBox::writeString(const std::string &str16, bool replace, bool atEnd)
 	{
-		::u32string str = CUtfStringView(str16).toUtf32();
+		// For now, just trim unsupported codepoints to make emoji fallback to text form
+		::u32string str = trimUnsupported(CUtfStringView(str16).toUtf32());
 		sint length = (sint)str.length();
 
 		::u32string toAppend;
@@ -1111,6 +1147,8 @@ namespace NLGUI
 
 						u32char c = isKeyRETURN ? '\n' : rEDK.getChar();
 						if (isFiltered(c)) return;
+						c = supportedCodepoint(c);
+						if (!c) return; // For now, just trim unsupported codepoints to make emoji fallback to text form
 						switch(_EntryType)
 						{
 							case Integer:

nel/src/gui/interface_expr_user_fct.cpp

@@ -553,15 +553,19 @@ namespace NLGUI
 			case CReflectedProperty::String:
 				result.setString ((elem->*(pRP->GetMethod.GetString))());
 			break;
+#ifdef RYZOM_LUA_UCSTRING
 			case CReflectedProperty::UCString:
 				result.setString ((elem->*(pRP->GetMethod.GetUCString))().toUtf8());
 			break;
+#endif
 			case CReflectedProperty::StringRef:
 				result.setString ((elem->*(pRP->GetMethod.GetStringRef))());
 			break;
+#ifdef RYZOM_LUA_UCSTRING
 			case CReflectedProperty::UCStringRef:
 				result.setString ((elem->*(pRP->GetMethod.GetUCStringRef))().toUtf8());
 			break;
+#endif
 			case CReflectedProperty::RGBA:
 				result.setRGBA ((elem->*(pRP->GetMethod.GetRGBA))());
 			break;

nel/src/gui/interface_link.cpp

@@ -117,18 +117,18 @@ namespace NLGUI
 					return false;
 				}
 			break;
+#ifdef RYZOM_LUA_UCSTRING
 			case CReflectedProperty::UCString:
 			case CReflectedProperty::UCStringRef:
-#ifdef RYZOM_LUA_UCSTRING
 				if (valueToAffect.toString())
 				{
 					(destElem.*(property.SetMethod.SetUCString))(ucstring::makeFromUtf8(valueToAffect.getString()));
 				}
 				else
-#endif
 				{
 					return false;
 				}
+#endif
 			break;
 			case CReflectedProperty::RGBA:
 				if (valueToAffect.toRGBA())

nel/src/gui/lua_ihm.cpp

@@ -1412,6 +1412,7 @@ namespace NLGUI
 			case CReflectedProperty::String:
 				ls.push( (reflectedObject.*(property.GetMethod.GetString))() );
 			break;
+#ifdef RYZOM_LUA_UCSTRING
 			case CReflectedProperty::UCString:
 			{
 				ucstring str = (reflectedObject.*(property.GetMethod.GetUCString))();
@@ -1434,6 +1435,7 @@ namespace NLGUI
 	#endif
 			}
 			break;
+#endif
 			case CReflectedProperty::StringRef:
 				ls.push( (reflectedObject.*(property.GetMethod.GetStringRef))() );
 			break;
@@ -1508,6 +1510,7 @@ namespace NLGUI
 					(target.*(property.SetMethod.SetString))(val);
 					return;
 				}
+#ifdef RYZOM_LUA_UCSTRING
 			case CReflectedProperty::UCString:
 			case CReflectedProperty::UCStringRef:
 				{
@@ -1532,6 +1535,7 @@ namespace NLGUI
 					(target.*(property.SetMethod.SetUCString))(val);
 					return;
 				}
+#endif
 			case CReflectedProperty::RGBA:
 				{
 					CRGBA color;

nel/src/gui/string_case.cpp

@@ -68,7 +68,7 @@ namespace NLGUI
 				if (!isSeparator(c))
 				{
 					if (newString)
-						NLMISC::appendToUpper(res, str, i);
+						NLMISC::appendToTitle(res, str, i);
 					else
 						NLMISC::appendToLower(res, str, i);
 					newString = false;
@@ -99,7 +99,7 @@ namespace NLGUI
 				else
 				{
 					if (newSentence)
-						NLMISC::appendToUpper(res, str, i);
+						NLMISC::appendToTitle(res, str, i);
 					else
 						NLMISC::appendToLower(res, str, i);
 
@@ -128,7 +128,7 @@ namespace NLGUI
 				else
 				{
 					if (newWord)
-						NLMISC::appendToUpper(res, str, i);
+						NLMISC::appendToTitle(res, str, i);
 					else
 						NLMISC::appendToLower(res, str, i);
 

nel/src/misc/app_context.cpp

@@ -135,12 +135,12 @@ CApplicationContext::~CApplicationContext()
 	while (it != iend)
 	{
 		// can't use nldebug there because it'll create new displayers
-		std::string message = toString("Instance '%s' still allocated at %p", it->first.c_str(), it->second);
+		std::string message = toString("Instance '%s' still allocated at %p\n", it->first.c_str(), it->second);
 
 #ifdef NL_OS_WINDOWS
 		OutputDebugStringW(nlUtf8ToWide(message));
 #else
-		printf("%s\n", message.c_str());
+		printf("%s", message.c_str());
 #endif
 
 		++it;

nel/src/misc/fixed_size_allocator.cpp

@@ -41,6 +41,7 @@ CFixedSizeAllocator::CFixedSizeAllocator(uint numBytesPerBlock, uint numBlockPer
 	nlassert(_NumBytesPerBlock >= numBytesPerBlock);
 	_NumBlockPerChunk = std::max(numBlockPerChunk, (uint) 3);
 	_NumAlloc = 0;
+	_SpareMem = NULL;
 }
 
 // *****************************************************************************************************************
@@ -49,10 +50,11 @@ CFixedSizeAllocator::~CFixedSizeAllocator()
 	if (_NumAlloc != 0)
 	{
 #ifdef NL_DEBUG
-			nlwarning("%d blocks were not freed", (int) _NumAlloc);
+		nlwarning("%d blocks were not freed, leaking memory", (int) _NumAlloc);
 #endif
-		return;
 	}
+	else
+	{
 		if (_NumChunks > 0)
 		{
 			nlassert(_NumChunks == 1);
@@ -60,6 +62,11 @@ CFixedSizeAllocator::~CFixedSizeAllocator()
 			delete _FreeSpace->Chunk;
 		}
 	}
+	if (_SpareMem)
+	{
+		aligned_free(_SpareMem);
+	}
+}
 
 // *****************************************************************************************************************
 void *CFixedSizeAllocator::alloc()
@@ -115,7 +122,10 @@ CFixedSizeAllocator::CChunk::~CChunk()
 	nlassert(NumFreeObjs == 0);
 	nlassert(Allocator->_NumChunks > 0);
 	-- (Allocator->_NumChunks);
+	if (Allocator->_SpareMem)
 		aligned_free(Mem); //delete[] Mem;
+	else
+		Allocator->_SpareMem = Mem;
 }
 
 // *****************************************************************************************************************
@@ -125,7 +135,15 @@ void CFixedSizeAllocator::CChunk::init(CFixedSizeAllocator *alloc)
 	nlassert(alloc != NULL);
 	Allocator = alloc;
 	//
+	if (Allocator->_SpareMem)
+	{
+		Mem = Allocator->_SpareMem;
+		Allocator->_SpareMem = NULL;
+	}
+	else
+	{
 		Mem = (uint8 *)aligned_malloc(getBlockSizeWithOverhead() * alloc->getNumBlockPerChunk(), NL_DEFAULT_MEMORY_ALIGNMENT); // new uint8[getBlockSizeWithOverhead() * alloc->getNumBlockPerChunk()];
+	}
 	//
 	getNode(0).Chunk = this;
 	getNode(0).Next = &getNode(1);

nel/src/misc/string_view.cpp

@@ -0,0 +1,31 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+#include "nel/misc/string_view.h"
+
+// STL includes
+
+// Project includes
+
+namespace NLMISC
+{
+
+void nothing_here_string_view_nl() { }
+
+} /* namespace NLMISC */
+
+/* end of file */

nel/src/misc/utf_string_view.cpp

@@ -230,6 +230,18 @@ size_t CUtfStringView::count() const
 	return res;
 }
 
+ptrdiff_t CUtfStringView::offset(ptrdiff_t i)
+{
+	size_t res = 0;
+	for (iterator it(begin()), end(this->end()); it != end; ++it)
+	{
+		if (res == i)
+			return (ptrdiff_t)it.ptr() - (ptrdiff_t)ptr();
+		++res;
+	}
+	return res;
+}
+
 u32char CUtfStringView::utf8Iterator(const void **addr)
 {
 	// Decode UTF-8
@@ -275,6 +287,12 @@ u32char CUtfStringView::utf8Iterator(const void **addr)
 								// Replacement character <20>
 								return 0xFFFD;
 							}
+							else if (c0 < 0x10000)
+							{
+								// Invalid encoding
+								// Replacement character <20>
+								return 0xFFFD;
+							}
 						}
 						else
 						{
@@ -308,6 +326,12 @@ u32char CUtfStringView::utf8Iterator(const void **addr)
 						// Replacement character <20>
 						return 0xFFFD;
 					}
+					else if (c0 < 0x0800)
+					{
+						// Invalid encoding
+						// Replacement character <20>
+						return 0xFFFD;
+					}
 				}
 				else
 				{
@@ -315,6 +339,12 @@ u32char CUtfStringView::utf8Iterator(const void **addr)
 					return 0xFFFD;
 				}
 			}
+			else if (c0 < 0x80)
+			{
+				// Invalid encoding
+				// Replacement character <20>
+				return 0xFFFD;
+			}
 		}
 		else
 		{

nel/src/sound/audio_mixer_user.cpp

@@ -695,6 +695,7 @@ void CAudioMixerUser::initDevice(const std::string &deviceName, const CInitInfo
 				setBackgroundFlags(flags);
 			}
 
+			form = NULL;
 			NLGEORGES::UFormLoader::releaseLoader(formLoader);
 		}
 	}

nel/tools/3d/build_clodtex/main.cpp

@@ -212,8 +212,8 @@ int main(int argc, char *argv[])
 			for(j=0;j<LodFilters.size();j++)
 			{
 				// Make the test case-unsensitive
-				string	lwrFileName= toLower(fileNameIn);
+				string	lwrFileName= toLowerAscii(fileNameIn);
-				string	lwrFilter= toLower(LodFilters[j]);
+				string	lwrFilter= toLowerAscii(LodFilters[j]);
 				if( testWildCard(lwrFileName.c_str(), lwrFilter.c_str()) )
 				{
 					string	clodFile= clod_dir_in+"/"+LodNames[j]+".clod";

nel/tools/3d/build_interface/main.cpp

@@ -180,7 +180,7 @@ bool writeFileDependingOnFilename(const std::string &filename, CBitmap &bitmap)
 
 	if (out.open(filename))
 	{
-		if (toLower(filename).find(".png") != string::npos)
+		if (toLowerAscii(filename).find(".png") != string::npos)
 		{
 			bitmap.writePNG(out, 32);
 		}
@@ -583,7 +583,7 @@ int main(int argc, char **argv)
 				continue;
 			}
 
-			sTGAname = toLower(string(tgaName));
+			sTGAname = toLowerAscii(string(tgaName));
 
 			// search position of extension
 			std::string tgaExt = CFile::getExtension(sTGAname);
@@ -597,7 +597,7 @@ int main(int argc, char **argv)
 			for (i = 0; i < mapSize; ++i)
 			{
 				// get the string whitout path
-				findTGAName = toLower(CFile::getFilenameWithoutExtension(AllMapNames[i]));
+				findTGAName = toLowerAscii(CFile::getFilenameWithoutExtension(AllMapNames[i]));
 				if( findTGAName == sTGAname )
 					break;
 			}

nel/tools/3d/ig_elevation/main.cpp

@@ -353,7 +353,7 @@ int main(int nNbArg, char**ppArgs)
 	vector<string> vAllFiles;
 	for(uint i = 0, len = (uint)vAllFilesUnfiltered.size(); i < len; ++i)
 	{
-		if (toLower(CFile::getExtension(vAllFilesUnfiltered[i])) == "ig")
+		if (toLowerAscii(CFile::getExtension(vAllFilesUnfiltered[i])) == "ig")
 		{
 			vAllFiles.push_back(vAllFilesUnfiltered[i]);
 		}

nel/tools/3d/ig_lighter_lib/ig_lighter_lib.cpp

@@ -170,7 +170,7 @@ void	CIgLighterLib::lightIg(CInstanceLighter &instanceLighter,
 			string name= igIn.getShapeName(i);
 			bool	shapeFound= true;
 
-			if (toLower (CFile::getExtension (name)) == "pacs_prim")
+			if (toLowerAscii (CFile::getExtension (name)) == "pacs_prim")
 			{
 				nlwarning("EXPORT BUG: Can't read %s (not a shape), should not be part of .ig!", name.c_str());
 				continue;

nel/tools/3d/lightmap_optimizer/main.cpp

@@ -481,7 +481,7 @@ int main(int nNbArg, char **ppArgs)
 					CTextureFile *pTF = dynamic_cast<CTextureFile*>(pIT);
 					if (pTF != NULL)
 					{
-						string sTexName = NLMISC::toLower(pTF->getFileName());
+						string sTexName = NLMISC::toLowerAscii(pTF->getFileName());
 						if(pTF->getUploadFormat()==ITexture::Luminance)
 							setLM8Bit.insert(sTexName);
 					}
@@ -530,7 +530,7 @@ int main(int nNbArg, char **ppArgs)
 		tmpLMs.clear();
 		for (i = 0; i < (sint32)AllLightmapNames.size(); ++i)
 		{
-			bool	lm8Bit= setLM8Bit.find( NLMISC::toLower(AllLightmapNames[i]) ) !=setLM8Bit.end();
+			bool	lm8Bit= setLM8Bit.find( NLMISC::toLowerAscii(AllLightmapNames[i]) ) !=setLM8Bit.end();
 			// if same mode
 			if( lm8Bit == (lmc8bitMode==1) )
 			{
@@ -824,13 +824,13 @@ int main(int nNbArg, char **ppArgs)
 									CTextureFile *pTF = dynamic_cast<CTextureFile*>(pIT);
 									if (pTF != NULL)
 									{
-										string sTexName = NLMISC::toLower(getBaseName(pTF->getFileName()));
+										string sTexName = NLMISC::toLowerAscii(getBaseName(pTF->getFileName()));
-										string sTexNameMoved = NLMISC::toLower(getBaseName(AllLightmapNames[i]));
+										string sTexNameMoved = NLMISC::toLowerAscii(getBaseName(AllLightmapNames[i]));
 										if (sTexName == sTexNameMoved)
 										{
 											// We must remap the name and indicate to remap uvs
 											bMustRemapUV = true;
-											//string sNewTexName = NLMISC::toLower(getBaseName(AllLightmapNames[j]));
+											//string sNewTexName = NLMISC::toLowerAscii(getBaseName(AllLightmapNames[j]));
 											//sNewTexName += NLMISC::toString(getLayerNb(pTF->getFileName())) + ".tga";
 											//pTF->setFileName (sNewTexName);
 										}
@@ -891,11 +891,11 @@ int main(int nNbArg, char **ppArgs)
 									CTextureFile *pTF = dynamic_cast<CTextureFile*>(pIT);
 									if (pTF != NULL)
 									{
-										string sTexName = NLMISC::toLower(getBaseName(pTF->getFileName()));
+										string sTexName = NLMISC::toLowerAscii(getBaseName(pTF->getFileName()));
-										string sTexNameMoved = NLMISC::toLower(getBaseName(AllLightmapNames[i]));
+										string sTexNameMoved = NLMISC::toLowerAscii(getBaseName(AllLightmapNames[i]));
 										if (sTexName == sTexNameMoved)
 										{
-											string sNewTexName = NLMISC::toLower(getBaseName(AllLightmapNames[j]));
+											string sNewTexName = NLMISC::toLowerAscii(getBaseName(AllLightmapNames[j]));
 											sNewTexName += NLMISC::toString(getLayerNb(pTF->getFileName())) + ".tga";
 											pTF->setFileName (sNewTexName);
 										}

nel/tools/3d/object_viewer/object_viewer.cpp

@@ -3696,7 +3696,7 @@ void		CObjectViewer::shootScene()
 			string extension = NLMISC::CFile::getExtension (tStrToUtf8(fileDlg.GetPathName()));
 
 			// The file name without extension
-			bool jpeg = toLower (extension) == "jpg";
+			bool jpeg = toLowerAscii (extension) == "jpg";
 
 			// Activate the driver
  			CNELU::Driver->activate ();

nel/tools/3d/object_viewer/snapshot_tool_dlg.cpp

@@ -458,7 +458,7 @@ void CSnapshotToolDlg::OnGo()
 			CString wildCard;
 			m_Filters.GetText(l, wildCard);
 			wildCard.MakeLower();
-			if (testWildCard(toLower(NLMISC::CFile::getFilename(files[k])).c_str(), tStrToUtf8(wildCard).c_str()))
+			if (testWildCard(toLowerAscii(NLMISC::CFile::getFilename(files[k])).c_str(), tStrToUtf8(wildCard).c_str()))
 			{
 				_FilteredFiles.push_back(files[k]);
 				break;

nel/tools/3d/panoply_maker/panoply_maker.cpp

@@ -325,7 +325,7 @@ int main(int argc, char* argv[])
 					NLMISC::CConfigFile::CVar &bitmap_extensions = cf.getVar ("bitmap_extensions");
 					for (uint k = 0; k < (uint) bitmap_extensions.size(); ++k)
 					{
-						std::string ext = "." + NLMISC::toLower(bitmap_extensions.asString(k));
+						std::string ext = "." + NLMISC::toLowerAscii(bitmap_extensions.asString(k));
 						if (std::find(bi.BitmapExtensions.begin(), bi.BitmapExtensions.end(), ext) == bi.BitmapExtensions.end())
 						{
 							bi.BitmapExtensions.push_back(ext);
@@ -483,7 +483,7 @@ static void BuildColoredVersions(const CBuildInfo &bi)
 		{
 			for (uint l = 0; l < bi.BitmapExtensions.size(); ++l)
 			{
-				std::string fileExt = "." + NLMISC::toLower(NLMISC::CFile::getExtension(files[k]));
+				std::string fileExt = "." + NLMISC::toLowerAscii(NLMISC::CFile::getExtension(files[k]));
 				if (fileExt == bi.BitmapExtensions[l])
 				{
 					//nlwarning("Processing : %s ", files[k].c_str());
@@ -532,7 +532,7 @@ static bool CheckIfNeedRebuildColoredVersionForOneBitmap(const CBuildInfo &bi, c
 	masks.clear();
 
 	std::string fileName = NLMISC::CFile::getFilenameWithoutExtension(fileNameWithExtension);
-	std::string fileExt  = NLMISC::toLower(NLMISC::CFile::getExtension(fileNameWithExtension));
+	std::string fileExt  = NLMISC::toLowerAscii(NLMISC::CFile::getExtension(fileNameWithExtension));
 
 	for (uint k = 0; k < bi.ColorMasks.size(); ++k)
 	{
@@ -721,7 +721,7 @@ static void BuildColoredVersionForOneBitmap(const CBuildInfo &bi, const std::str
 	masks.clear();
 
 	std::string fileName = NLMISC::CFile::getFilenameWithoutExtension(fileNameWithExtension);
-	std::string fileExt  = NLMISC::toLower(NLMISC::CFile::getExtension(fileNameWithExtension));
+	std::string fileExt  = NLMISC::toLowerAscii(NLMISC::CFile::getExtension(fileNameWithExtension));
 
 	uint	k;
 	for (k = 0; k < bi.ColorMasks.size(); ++k)

nel/tools/3d/shapes_exporter/main.cpp

@@ -169,7 +169,7 @@ sint main(int argc, char **argv)
 			if (filenames[i].find(".max") == std::string::npos)
 				continue;
 
-			std::string baseFilename = toLower(CFile::getFilenameWithoutExtension(filenames[i]));
+			std::string baseFilename = toLowerAscii(CFile::getFilenameWithoutExtension(filenames[i]));
 
 			// compute the md5 of .max file
 			std::string md5 = getNewMD5(filenames[i]).toString();

nel/tools/3d/textures_optimizer/main.cpp

@@ -104,7 +104,7 @@ bool parseOptions(int argc, char **argv)
 			// Filename
 			else
 			{
-				std::string ext = NLMISC::toLower(NLMISC::CFile::getExtension(option));
+				std::string ext = NLMISC::toLowerAscii(NLMISC::CFile::getExtension(option));
 
 				if (ext == "png" || ext == "tga")
 				{
@@ -135,7 +135,7 @@ int main(int argc, char **argv)
 
 	for(uint i = 0; i < InputFilenames.size(); ++i)
 	{
-		std::string ext = NLMISC::toLower(NLMISC::CFile::getExtension(InputFilenames[i]));
+		std::string ext = NLMISC::toLowerAscii(NLMISC::CFile::getExtension(InputFilenames[i]));
 
 		NLMISC::CIFile input;
 

nel/tools/3d/tga_2_dds/tga2dds.cpp

@@ -327,10 +327,10 @@ int main(int argc, char **argv)
 
 	if (args.haveArg("a"))
 	{
-		std::string strAlgo = args.getArg("a").front();
+		std::string strAlgo = toLowerAscii(args.getArg("a").front());
 
 		if (strAlgo == "1")					OptAlgo = DXT1;
-		else if (toLower(strAlgo) == "1a") 	OptAlgo = DXT1A;
+		else if (strAlgo == "1a") 			OptAlgo = DXT1A;
 		else if (strAlgo == "3")			OptAlgo = DXT3;
 		else if (strAlgo == "5") 			OptAlgo = DXT5;
 		else if (strAlgo == "tga8")			OptAlgo = TGA8;

nel/tools/3d/zone_dependencies/zone_dependencies.cpp

@@ -525,7 +525,7 @@ int main (int argc, char* argv[])
 
 							// Write the dependencies file
 							FILE *outputFile;
-							if ((outputFile = nlfopen (toLower (outputFileName), "w")))
+							if ((outputFile = nlfopen (toLowerAscii (outputFileName), "w")))
 							{
 								// Add a dependency entry
 								fprintf (outputFile, "dependencies =\n{\n");
@@ -540,7 +540,7 @@ int main (int argc, char* argv[])
 
 									// Write it
 									string message="\t\""+zoneName+"\"";
-									fprintf (outputFile, "%s", toLower (message).c_str());
+									fprintf (outputFile, "%s", toLowerAscii (message).c_str());
 
 									// Next ite;
 									ite++;
@@ -615,7 +615,7 @@ static void computeIGBBox(const NL3D::CInstanceGroup &ig, CLightingBBox &result,
 			{
 				nlwarning("Unable to find shape '%s'", it->Name.c_str());				
 			}
-			else if (toLower (CFile::getExtension (shapePathName)) == "pacs_prim")
+			else if (toLowerAscii (CFile::getExtension (shapePathName)) == "pacs_prim")
 			{
 				nlwarning("EXPORT BUG: Can't read %s (not a shape), should not be part of .ig!", shapePathName.c_str());
 			}
@@ -711,7 +711,7 @@ static void computeIGBBox(const NL3D::CInstanceGroup &ig, CLightingBBox &result,
 static void computeZoneIGBBox(const char *zoneName, CLightingBBox &result, TShapeMap &shapeMap, const TString2LightingBBox &additionnalIG)
 {
 	result = CLightingBBox(); // starts with a void box	
-	std::string lcZoneName = NLMISC::toLower(std::string(zoneName));
+	std::string lcZoneName = NLMISC::toLowerAscii(std::string(zoneName));
 	TString2LightingBBox::const_iterator zoneIt = additionnalIG.find(lcZoneName);
 	if (zoneIt != additionnalIG.end())
 	{		
@@ -901,7 +901,7 @@ static void computeIGBBoxFromContinent(NLMISC::CConfigFile &parameter,
 					nlwarning("Couldn't get zone name of village %d in continent %s", continentName.c_str(), k);
 					continue;
 				}
-				zoneName = NLMISC::toLower(CFile::getFilenameWithoutExtension(zoneName));				
+				zoneName = NLMISC::toLowerAscii(CFile::getFilenameWithoutExtension(zoneName));				
 				CLightingBBox result;				
 				// ok, it is in the dependant zones
 				computeBBoxFromVillage(currVillage, continentName, k, shapeMap, result);

nel/tools/3d/zone_elevation/zone_elevation.cpp

@@ -105,7 +105,7 @@ bool getXYFromZoneName(sint32 &x, sint32 &y, const string &zoneName)
 	}
 	if (xStr.size() != 2)
 		goto Fail;
-	xStr = NLMISC::toUpper(xStr);
+	xStr = NLMISC::toUpperAscii(xStr);
 	x = ((xStr[0] - 'A') * 26 + (xStr[1] - 'A'));
 	return true;
 Fail:

nel/tools/3d/zone_ig_lighter/zone_ig_lighter.cpp

@@ -116,7 +116,7 @@ int main(int argc, char* argv[])
 		if (inputFile.open (argv[1]))
 		{
 			// Zone name
-			string zoneName=toLower (string ("zone_"+getName (argv[1])));
+			string zoneName=toLowerAscii (string ("zone_"+getName (argv[1])));
 
 			// Load the zone
 			try
@@ -413,7 +413,7 @@ int main(int argc, char* argv[])
 						if(group->getInstance(instance).DontCastShadow || group->getInstance(instance).DontCastShadowForExterior)
 							continue;
 
-						if (toLower (CFile::getExtension (name)) == "pacs_prim")
+						if (toLowerAscii (CFile::getExtension (name)) == "pacs_prim")
 						{
 							nlwarning("EXPORT BUG: Can't read %s (not a shape), should not be part of .ig!", name.c_str());
 							continue;

nel/tools/3d/zone_lighter/zone_lighter.cpp

@@ -140,7 +140,7 @@ static void loadIGFromVillage(const NLGEORGES::UFormElm *villageItem, const std:
 		// verify that the ig is not already added (case of tr_water.ig in additional_igs)
 		for(uint igAdd= 0;igAdd<(uint)additionalIgNames.size();igAdd++)
 		{
-			if( toLower(additionalIgNames.asString()) == toLower(igName) )
+			if( toLowerAscii(additionalIgNames.asString()) == toLower(igName) )
 			{
 				nlwarning("Skipping Village Ig %s, cause already exist in additional ig", igName.c_str());
 				continue;
@@ -315,7 +315,7 @@ int main(int argc, char* argv[])
 		if (inputFile.open (argv[1]))
 		{
 			// Zone name
-			string zoneName=toLower (string ("zone_"+getName (argv[1])));
+			string zoneName=toLowerAscii (string ("zone_"+getName (argv[1])));
 
 			// Load the zone
 			try
@@ -702,14 +702,14 @@ int main(int argc, char* argv[])
 								if(group->getInstance(instance).DontCastShadow || group->getInstance(instance).DontCastShadowForExterior)
 									continue;
 
-								if (toLower (CFile::getExtension (name)) == "pacs_prim")
+								if (toLowerAscii (CFile::getExtension (name)) == "pacs_prim")
 								{
 									nlwarning("EXPORT BUG: Can't read %s (not a shape), should not be part of .ig!", name.c_str());
 									continue;
 								}
 
 								// PS ?
-								if (toLower (CFile::getExtension (name)) == "ps")
+								if (toLowerAscii (CFile::getExtension (name)) == "ps")
 									continue;
 								
 								// Add a .shape at the end ?

nel/tools/georges/georges2csv/georges2csv.cpp

@@ -738,7 +738,7 @@ void	convertCsvFile( const string &file, bool generate, const string& sheetType
 			}
 			else if ( nlstricmp( fields[i], "parent" ) == 0 )
 			{
-				fields[i] = toLower( fields[i] );
+				fields[i] = toLowerAscii( fields[i] );
 			}
 			else
 			{
@@ -883,7 +883,7 @@ void	convertCsvFile( const string &file, bool generate, const string& sheetType
 		{
 			filebase += "." + sheetType;
 		}
-		filebase = toLower(filebase);
+		filebase = toLowerAscii(filebase);
 		string	filename, dirbase;
 		bool	isNewSheet=true;
 
@@ -905,7 +905,7 @@ void	convertCsvFile( const string &file, bool generate, const string& sheetType
 			else
 			{
 				// Load template sheet
-				filename = toLower(filebase);
+				filename = toLowerAscii(filebase);
 				form = (CForm*)formLoader->loadForm( (string("_empty.") + sheetType).c_str() );
 				if (form == NULL)
 				{

nel/tools/ligo/unbuild_land/unbuild_land.cpp

@@ -104,7 +104,7 @@ bool getXYFromZoneName(sint32 &x, sint32 &y, const string &zoneName)
 	}
 	if (xStr.size() != 2)
 		goto Fail;
-	xStr = NLMISC::toUpper(xStr);
+	xStr = NLMISC::toUpperAscii(xStr);
 	x = ((xStr[0] - 'A') * 26 + (xStr[1] - 'A'));
 	return true;
 Fail:

nel/tools/misc/CMakeLists.txt

@@ -12,6 +12,7 @@ IF(WITH_NEL_TOOLS)
     lock
     make_sheet_id
     xml_packer
+    utf_generator
   )
 
   IF(WITH_QT OR WITH_QT5)

nel/tools/misc/bnp_make/main.cpp

@@ -56,7 +56,7 @@ bool keepFile (const std::string &fileName)
 	uint i;
 	bool ifPresent = false;
 	bool ifTrue = false;
-	string file = toLower(CFile::getFilename (fileName));
+	string file = toLowerAscii(CFile::getFilename (fileName));
 	for (i=0; i<WildCards.size(); i++)
 	{
 		if (WildCards[i].Not)
@@ -100,7 +100,7 @@ bool packSubRecurse(const std::string &srcDirectory)
 	// check for files with same name
 	for(uint i = 1, len = pathContent.size(); i < len; ++i)
 	{
-		if (toLower(CFile::getFilename(pathContent[i-1])) == toLower(CFile::getFilename(pathContent[i])))
+		if (toLowerAscii(CFile::getFilename(pathContent[i-1])) == toLowerAscii(CFile::getFilename(pathContent[i])))
 		{
 			nlwarning("File %s is not unique in BNP!", CFile::getFilename(pathContent[i]).c_str());
 			return false;
@@ -152,7 +152,7 @@ int main(int argc, char **argv)
 		for (uint i = 0; i < filters.size(); ++i)
 		{
 			CWildCard card;
-			card.Expression = toLower(filters[i]);
+			card.Expression = toLowerAscii(filters[i]);
 			card.Not = false;
 			WildCards.push_back(card);
 		}
@@ -163,7 +163,7 @@ int main(int argc, char **argv)
 		for (uint i = 0; i < filters.size(); ++i)
 		{
 			CWildCard card;
-			card.Expression = toLower(filters[i]);
+			card.Expression = toLowerAscii(filters[i]);
 			card.Not = true;
 			WildCards.push_back(card);
 		}

nel/tools/misc/data_mirror/data_mirror.cpp

@@ -108,21 +108,21 @@ BOOL CData_mirrorApp::InitInstance()
 
 		cf.load (path);
 		MainDirectory = cf.getVar ("MainDirectory").asString ();
-		MainDirectory = toLower (CPath::standardizePath (MainDirectory));
+		MainDirectory = toLowerAscii (CPath::standardizePath (MainDirectory));
 		
 		MirrorDirectory = cf.getVar ("MirrorDirectory").asString ();
-		MirrorDirectory = toLower (CPath::standardizePath (MirrorDirectory));
+		MirrorDirectory = toLowerAscii (CPath::standardizePath (MirrorDirectory));
 		
 		LogDirectory = cf.getVar ("LogDirectory").asString ();
-		LogDirectory = toLower (CPath::standardizePath (LogDirectory));
+		LogDirectory = toLowerAscii (CPath::standardizePath (LogDirectory));
 		
 		IgnoreDirectory = cf.getVar ("IgnoreDirectory").asString ();
-		IgnoreDirectory = toLower (CPath::standardizePath (IgnoreDirectory));
+		IgnoreDirectory = toLowerAscii (CPath::standardizePath (IgnoreDirectory));
 		if (IgnoreDirectory.empty())
 			IgnoreDirectory = MainDirectory;
 
 		string sBinaryCompare = cf.getVar ("BinaryCompare").asString ();
-		sBinaryCompare = toLower (sBinaryCompare);
+		sBinaryCompare = toLowerAscii (sBinaryCompare);
 		BinaryCompare = false;
 		if ((sBinaryCompare == "true") || (sBinaryCompare=="1"))
 			BinaryCompare = true;
@@ -145,12 +145,12 @@ BOOL CData_mirrorApp::InitInstance()
 		if (NLMISC::CFile::isDirectory (CurrentDir))
 		{
 			directory = true;
-			CurrentDir = toLower(CPath::standardizePath (CurrentDir));
+			CurrentDir = toLowerAscii(CPath::standardizePath (CurrentDir));
 		}
 		else if (NLMISC::CFile::fileExists (CurrentDir))
 		{
 			directory = false;
-			CurrentDir = toLower(CPath::standardizePath (NLMISC::CFile::getPath (CurrentDir)));
+			CurrentDir = toLowerAscii(CPath::standardizePath (NLMISC::CFile::getPath (CurrentDir)));
 		}
 		else
 		{

nel/tools/misc/data_mirror/data_mirrorDlg.cpp

@@ -628,7 +628,7 @@ void CData_mirrorDlg::buildSourceFiles ()
 	{
 		// Get the filename
 		string &str = fileSource[i];
-		str = toLower(str.substr (MainDirectory.size (), str.size ()));
+		str = toLowerAscii(str.substr (MainDirectory.size (), str.size ()));
 
 		// In the ignore list ?
 		if (IgnoreFiles.find (str) == IgnoreFiles.end () && (str != "ignore_list.txt"))
@@ -716,7 +716,7 @@ void CData_mirrorDlg::buildSourceFiles ()
 	{
 		// Get the filename
 		string &str = fileSource[i];
-		str = toLower(str.substr (MirrorDirectory.size (), str.size ()));
+		str = toLowerAscii(str.substr (MirrorDirectory.size (), str.size ()));
 
 		// In the ignore list ?
 		if (IgnoreFiles.find (str) == IgnoreFiles.end () && (str != "ignore_list.txt"))

nel/tools/misc/extract_filename/extract_filename.cpp

@@ -214,7 +214,7 @@ void extractStringsFromBinary (const vector<char> &fileArray, set<string> &filen
 				if (str != "")
 				{
 					// Lower case
-					str = toLower (str);
+					str = toLowerAscii (str);
 
 					// Filter extensions
 					if (filterExtension (str.c_str(), extensions))
@@ -280,7 +280,7 @@ void extractStringsFromASCII (const vector<char> &fileArray, set<string> &filena
 						temp[c] = begin[c];
 
 					// Lower case
-					temp = toLower (temp);
+					temp = toLowerAscii (temp);
 
 					// Filter extensions
 					if (filterExtension (temp.c_str(), extensions))
@@ -325,7 +325,7 @@ bool loadConfigFiles (const char *ext, const char *input_files, const char *avai
 		while (fgets (name, 512, file))
 		{
 			// To string and lower
-			temp = toLower (string(name));
+			temp = toLowerAscii (string(name));
 
 			// Remove return
 			removeChar (temp, '\n');
@@ -374,8 +374,8 @@ bool loadConfigFiles (const char *ext, const char *input_files, const char *avai
 					while (fgets (name, 512, file))
 					{
 						// To lower
-						temp = toLower (string(name));
+						temp = toLowerAscii (string(name));
-						temp2 = toLower (string(name));
+						temp2 = toLowerAscii (string(name));
 
 						// Remove space
 						removeBeginEndSpaces (temp);
@@ -597,7 +597,7 @@ int main(int argc, char* argv[])
 				// It is used ?
 				if (usedFiles.find (available->first) == usedFiles.end())
 				{
-					string temp = toLower (available->second);
+					string temp = toLowerAscii (available->second);
 					fprintf (stderr, "UNUSED: %s\n", temp.c_str());
 				}
 

nel/tools/misc/make_sheet_id/make_sheet_id.cpp

@@ -320,10 +320,10 @@ void addId( string fileName )
 	if( itFI == FormToId.end() )
 	{
 		// double check : if file not found we check with lower case version of filename
-		map<string,TFormId>::iterator itFILwr = FormToId.find( toLower(fileName) );
+		map<string,TFormId>::iterator itFILwr = FormToId.find( toLowerAscii(fileName) );
 		if( itFILwr != FormToId.end() )
 		{
-			nlwarning("Trying to add %s but the file %s is already known ! becareful with lower case and upper case.", fileName.c_str(), toLower(fileName).c_str());
+			nlwarning("Trying to add %s but the file %s is already known ! becareful with lower case and upper case.", fileName.c_str(), toLowerAscii(fileName).c_str());
 			NbFilesDiscarded++;
 			return;
 		}

nel/tools/misc/snp_make/main.cpp

@@ -64,7 +64,7 @@ bool keepFile (const char *fileName)
 	uint i;
 	bool ifPresent = false;
 	bool ifTrue = false;
-	string file = toLower(CFile::getFilename (fileName));
+	string file = toLowerAscii(CFile::getFilename (fileName));
 	for (i=0; i<WildCards.size(); i++)
 	{
 		if (WildCards[i].Not)

nel/tools/misc/utf_generator/CMakeLists.txt

@@ -0,0 +1,11 @@
+FILE(GLOB SRC *.cpp *.h *.rc *.rc2)
+
+SOURCE_GROUP("" FILES ${SRC})
+
+ADD_EXECUTABLE(nl_utf_generator ${SRC})
+
+TARGET_LINK_LIBRARIES(nl_utf_generator nelmisc)
+NL_DEFAULT_PROPS(nl_utf_generator "NeL, Tools, Misc: UTF Generator")
+NL_ADD_RUNTIME_FLAGS(nl_utf_generator)
+
+INSTALL(TARGETS nl_utf_generator RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc)

nel/tools/misc/utf_generator/main.cpp

@@ -0,0 +1,425 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2020  Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "nel/misc/types_nl.h"
+
+#include "nel/misc/debug.h"
+#include "nel/misc/common.h"
+#include "nel/misc/string_common.h"
+#include "nel/misc/string_view.h"
+#include "nel/misc/utf_string_view.h"
+
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <iostream>
+#include <iomanip>
+
+void printStringMap(const std::string &name, std::map<char, std::string> &m, bool trim)
+{
+	std::cout << "static const char " << name << "[" << std::dec << (trim ? "64" : "256") << " * 4] = {\n";
+	bool zero = false;
+	for (int i = 0; i < (trim ? 64 : 256); ++i)
+	{
+		int x = trim ? i + 0x80 : i;
+		if (m.find(x) == m.end())
+		{
+			if (x % 8 == 7)
+			{
+				zero = false;
+				std::cout << "0, 0, 0, 0,\n";
+			}
+			else
+			{
+				zero = true;
+				std::cout << "0, 0, 0, 0, ";
+			}
+		}
+		else
+		{
+			if (zero) std::cout << "\n";
+			std::stringstream ss;
+			ss << "'\\x" << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (m[x].length() > 0 ? (unsigned char)m[x][0] : 0)
+				<< "', '\\x" << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (m[x].length() > 1 ? (unsigned char)m[x][1] : 0)
+				<< "', '\\x" << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (m[x].length() > 2 ? (unsigned char)m[x][2] : 0)
+				<< "', 0,\n";
+			std::cout << ss.str();
+			zero = false;
+		}
+	}
+	if (zero) std::cout << "\n";
+	std::cout << "};\n\n";
+}
+
+void printMapMap(const std::string &name, const std::string &strName, std::map<char, std::map<char, std::string>> &m, int base, int size)
+{
+	std::cout << "static const char *" << name << "[" << size << "] = {\n";
+	bool zero = false;
+	for (int i = base; i < (base + size); ++i)
+	{
+		int x = i;
+		if (m.find(x) == m.end())
+		{
+			if (x % 32 == 1315)
+			{
+				zero = false;
+				std::cout << "0, \n";
+			}
+			else
+			{
+				zero = true;
+				std::cout << "0, ";
+			}
+		}
+		else
+		{
+			if (zero) std::cout << "\n";
+			std::stringstream n;
+			n << strName;
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)x;
+			std::cout << n.str() << ",\n";
+			zero = false;
+		}
+	}
+	if (zero) std::cout << "\n";
+	std::cout << std::dec << "};\n\n";
+}
+
+void printMapMapMap(const std::string &name, const std::string &mapName, std::map<char, std::map<char, std::map<char, std::string>>> &m, int base, int size)
+{
+	std::cout << "static const char **" << name << "[" << size << "] = {\n";
+	bool zero = false;
+	for (int i = base; i < (base + size); ++i)
+	{
+		int x = i;
+		if (m.find(x) == m.end())
+		{
+			if (x % 32 == 1315)
+			{
+				zero = false;
+				std::cout << "0, \n";
+			}
+			else
+			{
+				zero = true;
+				std::cout << "0, ";
+			}
+		}
+		else
+		{
+			if (zero) std::cout << "\n";
+			std::stringstream n;
+			n << mapName;
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)x;
+			std::cout << n.str() << ",\n";
+			zero = false;
+		}
+	}
+	if (zero) std::cout << "\n";
+	std::cout << "};\n\n";
+}
+
+void printMapMapMapMap(const std::string &name, const std::string &mapName, std::map<char, std::map<char, std::map<char, std::map<char, std::string>>>> &m, int base, int size)
+{
+	std::cout << "static const char ***" << name << "[" << size << "] = {\n";
+	bool zero = false;
+	for (int i = base; i < (base + size); ++i)
+	{
+		int x = i;
+		if (m.find(x) == m.end())
+		{
+			if (x % 32 == 1315)
+			{
+				zero = false;
+				std::cout << "0, \n";
+			}
+			else
+			{
+				zero = true;
+				std::cout << "0, ";
+			}
+		}
+		else
+		{
+			if (zero) std::cout << "\n";
+			std::stringstream n;
+			n << mapName;
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)x;
+			std::cout << n.str() << ",\n";
+			zero = false;
+		}
+	}
+	if (zero) std::cout << "\n";
+	std::cout << "};\n\n";
+}
+
+void generateMap(const std::string &file, const std::string &name, const std::vector<u32char> &map)
+{
+	std::map<char, std::string> m1;
+	std::map<char, std::map<char, std::string>> m2;
+	std::map<char, std::map<char, std::map<char, std::string>>> m3;
+	std::map<char, std::map<char, std::map<char, std::map<char, std::string>>>> m4;
+	for (u32char i = 0; i < 0x110000; ++i)
+	{
+		if (map[i] != i)
+		{
+			std::string from;
+			NLMISC::CUtfStringView::append(from, i);
+			std::string to;
+			NLMISC::CUtfStringView::append(to, map[i]);
+			// assert(from.size() == to.size());
+			if (from.length() == 1)
+			{
+				m1[from[0]] = to;
+			}
+			else if (from.length() == 2)
+			{
+				if (m2.find(from[0]) == m2.end())
+					m2[from[0]] = std::map<char, std::string>();
+				m2[from[0]][from[1]] = to;
+			}
+			else if (from.length() == 3)
+			{
+				if (m3.find(from[0]) == m3.end())
+					m3[from[0]] = std::map<char, std::map<char, std::string>>();
+				if (m3[from[0]].find(from[1]) == m3[from[0]].end())
+					m3[from[0]][from[1]] = std::map<char, std::string>();
+				m3[from[0]][from[1]][from[2]] = to;
+			}
+			else if (from.length() == 4)
+			{
+				if (m4.find(from[0]) == m4.end())
+					m4[from[0]] = std::map<char, std::map<char, std::map<char, std::string>>>();
+				if (m4[from[0]].find(from[1]) == m4[from[0]].end())
+					m4[from[0]][from[1]] = std::map<char, std::map<char, std::string>>();
+				if (m4[from[0]][from[1]].find(from[2]) == m4[from[0]][from[1]].end())
+					m4[from[0]][from[1]][from[2]] = std::map<char, std::string>();
+				m4[from[0]][from[1]][from[2]][from[3]] = to;
+			}
+		}
+	}
+	printStringMap("s_" + name, m1, false);
+	for (int i = 0; i < 256; ++i)
+	{
+		std::stringstream n;
+		n << "s_" << name;
+		if (m2.find(i) != m2.end())
+		{
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			printStringMap(n.str(), m2[i], true);
+		}
+		else if (m3.find(i) != m3.end())
+		{
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			for (int j = 0; j < 256; ++j)
+			{
+				if (m3[i].find(j) != m3[i].end())
+				{
+					std::stringstream nn;
+					nn << n.str();
+					nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
+					printStringMap(nn.str(), m3[i][j], true);
+				}
+			}
+		}
+		else if (m4.find(i) != m4.end())
+		{
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			for (int j = 0; j < 256; ++j)
+			{
+				if (m4[i].find(j) != m4[i].end())
+				{
+					std::stringstream nn;
+					nn << n.str();
+					nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
+					for (int k = 0; k < 256; ++k)
+					{
+						if (m4[i][j].find(k) != m4[i][j].end())
+						{
+							std::stringstream nnn;
+							nnn << nn.str();
+							nnn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)k;
+							printStringMap(nnn.str(), m4[i][j][k], true);
+						}
+					}
+				}
+			}
+		}
+	}
+	printMapMap("s_" + name + "Map", "s_" + name, m2, 0xC0, 32);
+
+	for (int i = 0; i < 256; ++i)
+	{
+		std::stringstream n;
+		n << "s_" << name << "Map";
+		std::stringstream nn;
+		nn << "s_" << name;
+		if (m3.find(i) != m3.end())
+		{
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			printMapMap(n.str(), nn.str(), m3[i], 0x80, 64);
+		}
+	}
+	printMapMapMap("s_" + name + "MapMap", "s_" + name + "Map", m3, 0xE0, 16);
+
+	for (int i = 0; i < 256; ++i)
+	{
+		std::stringstream n;
+		n << "s_" << name << "Map";
+		std::stringstream nn;
+		nn << "s_" << name;
+		std::stringstream nnn;
+		nnn << "s_" << name << "MapMap";
+		if (m4.find(i) != m4.end())
+		{
+			n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			nnn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
+			for (int j = 0; j < 256; ++j)
+			{
+				if (m4[i].find(j) != m4[i].end())
+				{
+					std::stringstream n2, nn2;
+					n2 << n.str() << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
+					nn2 << nn.str() << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
+					printMapMap(n2.str(), nn2.str(), m4[i][j], 0x80, 64);
+				}
+			}
+			printMapMapMap(nnn.str(), n.str(), m4[i], 0x80, 64);
+		}
+	}
+	printMapMapMapMap("s_" + name + "MapMapMap", "s_" + name + "MapMap", m4, 0xF0, 8);
+}
+
+int main (int argc, char **argv)
+{
+	std::ifstream fi("UnicodeData.txt");
+
+	std::vector<u32char> upper;
+	std::vector<u32char> lower;
+	std::vector<u32char> title;
+	std::vector<u32char> ci;
+
+	upper.resize(0x110000);
+	lower.resize(0x110000);
+	title.resize(0x110000);
+	ci.resize(0x110000);
+
+	for (u32char i = 0; i < 0x110000; ++i)
+	{
+		upper[i] = i;
+		lower[i] = i;
+		title[i] = i;
+		ci[i] = i;
+	}
+
+	std::string line;
+	while (std::getline(fi, line))
+	{
+		std::vector<std::string> cols;
+		NLMISC::explode(line, nlstr(";"), cols, false);
+		nlassert(cols.size() == 15);
+
+		u32char c = NLMISC::atoiInt64(cols[0].c_str(), 16);
+		u32char up = NLMISC::atoiInt64(cols[12].c_str(), 16);
+		u32char low = NLMISC::atoiInt64(cols[13].c_str(), 16);
+		u32char tit = NLMISC::atoiInt64(cols[14].c_str(), 16);
+		
+		if (up) upper[c] = up;
+		if (low) lower[c] = low;
+		if (tit) title[c] = tit;
+	}
+
+	std::vector<u32char> ref;
+	int rounds = 0;
+	for (;;)
+	{
+		ref = ci;
+
+		for (u32char i = 0; i < 0x110000; ++i)
+		{
+			ci[i] = title[ci[i]];
+		}
+
+		for (u32char i = 0; i < 0x110000; ++i)
+		{
+			ci[i] = upper[ci[i]];
+		}
+
+		for (u32char i = 0; i < 0x110000; ++i)
+		{
+			ci[i] = lower[ci[i]];
+		}
+
+		bool equal = true;
+		for (u32char i = 0; i < 0x110000; ++i)
+		{
+			if (ci[i] != ref[i])
+				equal = false;
+		}
+		++rounds;
+		std::cout << rounds << std::endl;
+		if (equal)
+			break;
+	}
+
+	for (u32char i = 0; i < 0x110000; ++i)
+	{
+		if (ci[i] != lower[i])
+			std::cout << i << std::endl;
+	}
+
+	generateMap("string_to_upper", "StringToUpper", upper);
+	//generateMap("string_to_lower", "StringToLower", lower);
+	//generateMap("string_to_title", "StringToTitle", title);
+	//generateMap("string_to_ci", "StringToCaseInsensitive", ci);
+
+	std::string test = nlstr("Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,");
+	std::string testUpper = NLMISC::toUpper(test);
+	std::string testLower = NLMISC::toLower(test);
+	std::string testUpper2 = NLMISC::toUpper(testLower);
+	std::string testLower2 = NLMISC::toLower(testUpper);
+	std::cout << test << std::endl;
+	std::cout << testUpper << std::endl;
+	std::cout << testLower << std::endl;
+	std::cout << testUpper2 << std::endl;
+	std::cout << testLower2 << std::endl;
+
+	int cci1 = NLMISC::compareCaseInsensitive("bAAAAfdsklj", "Cldsfjslkf");
+	int cci2 = NLMISC::compareCaseInsensitive("Cldsfjslkf", "bAAAAfdsklj");
+	int strc1 = strcmp(NLMISC::toLower("bAAAAfdsklj").c_str(), NLMISC::toLower("Cldsfjslkf").c_str());
+	int strc2 = strcmp(NLMISC::toLower("Cldsfjslkf").c_str(), NLMISC::toLower("bAAAAfdsklj").c_str());
+
+	int bcci1 = NLMISC::compareCaseInsensitive("bAAAAfdsklj", "AnlsqFDS");
+	int bcci2 = NLMISC::compareCaseInsensitive("AnlsqFDS", "bAAAAfdsklj");
+	int bstrc1 = strcmp(NLMISC::toLower("bAAAAfdsklj").c_str(), NLMISC::toLower("AnlsqFDS").c_str());
+	int bstrc2 = strcmp(NLMISC::toLower("AnlsqFDS").c_str(), NLMISC::toLower("bAAAAfdsklj").c_str());
+
+	std::vector<std::string> arr;
+	arr.push_back("AnlsqFDS");
+	arr.push_back("yozeRNZE");
+	arr.push_back("yOzeihfn");
+	arr.push_back("bAAAAfdsklj");
+	arr.push_back("Cldsfjslkf");
+	std::sort(arr.begin(), arr.end(), NLMISC::ltCaseInsensitive);
+	for (int i = 0; i < arr.size(); ++i)
+		std::cout << arr[i] << std::endl;
+
+	return EXIT_SUCCESS;
+}
+
+/* end of file */

nel/tools/misc/utf_generator/main.rc

@@ -0,0 +1,42 @@
+#include <windows.h>
+#include "config.h"
+
+IDI_MAIN_ICON ICON DISCARDABLE "gold_pill.ico"
+
+#ifdef _DEBUG
+#define NL_FILEEXT "_d"
+#else
+#define NL_FILEEXT ""
+#endif
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION NL_VERSION_RC
+ PRODUCTVERSION NL_VERSION_RC
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_APP
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "CompanyName", AUTHOR
+            VALUE "FileDescription", "NeL UTF Generator"
+            VALUE "FileVersion", NL_VERSION
+            VALUE "LegalCopyright", COPYRIGHT
+            VALUE "OriginalFilename", "nl_utf_generator" NL_FILEEXT ".exe"
+            VALUE "ProductName", "NeL Tools"
+            VALUE "ProductVersion", NL_PRODUCT_VERSION
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x9, 1200
+    END
+END

ryzom/client/src/client.cpp

@@ -451,6 +451,7 @@ int main(int argc, char **argv)
 	// delete all logs and displayers when we're not using logs macros anymore
 	destroyDebug();
 	CLog::releaseProcessName();
+	// CCoTask::releaseInstance();
 
 	// delete the Nel context
 	delete appContext;

ryzom/client/src/client_chat_manager.cpp

@@ -329,7 +329,7 @@ void CClientChatManager::tell( const string& receiverIn, const string& strIn )
 
 	// *** send str
 	CBitMemStream bms;
-	string msgType = "STRING:TELL";
+	const char *msgType = "STRING:TELL";
 	if( GenericMsgHeaderMngr.pushNameToStream(msgType,bms) )
 	{
 		bms.serial( receiver );
@@ -385,7 +385,7 @@ void CClientChatManager::tell( const string& receiverIn, const string& strIn )
 void CClientChatManager::filter( uint8 filter )
 {
 	CBitMemStream bms;
-	string msgType = "STRING:FILTER";
+	const char *msgType = "STRING:FILTER";
 	if( GenericMsgHeaderMngr.pushNameToStream(msgType,bms) )
 	{
 		bms.serial( filter );
@@ -417,7 +417,7 @@ void CClientChatManager::setChatMode(CChatGroup::TGroupType group, TChanID dynam
 	if (group != CChatGroup::team)
 	{
 		CBitMemStream bms;
-		string msgType = "STRING:CHAT_MODE";
+		const char *msgType = "STRING:CHAT_MODE";
 		if( GenericMsgHeaderMngr.pushNameToStream(msgType,bms) )
 		{
 			bms.serial( mode );

ryzom/client/src/commands.cpp

@@ -181,14 +181,14 @@ NLMISC_COMMAND(where, "Ask information on the position", "")
 	// Check parameters.
 	if(args.empty())
 	{	// Create the message and send.
-		const string msgName = "COMMAND:WHERE";
+		const char *msgName = "COMMAND:WHERE";
 		CBitMemStream out;
 		if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 		{
 			NetMngr.push(out);
 		}
 		else
-			nlwarning("command 'where': unknown message named '%s'", msgName.c_str());
+			nlwarning("command 'where': unknown message named '%s'", msgName);
 		return true;
 	}
 	return false;
@@ -1803,14 +1803,14 @@ NLMISC_COMMAND(usePreprogCombat, "use the specified combat preprog sentence", "<
 NLMISC_COMMAND(engage, "engage target in combat", "")
 {
 	// Create the message for the server to execute a phrase.
-	const string msgName = "COMBAT:ENGAGE";
+	const char *msgName = "COMBAT:ENGAGE";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("mainLoop : unknown message name : '%s'", msgName.c_str());
+		nlwarning("mainLoop : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1836,14 +1836,14 @@ NLMISC_COMMAND(disengage, "disengage from combat", "")
 NLMISC_COMMAND(leaveTeam, "leave team", "")
 {
 	// Create the message for the server to execute a phrase.
-	const string msgName = "TEAM:LEAVE";
+	const char *msgName = "TEAM:LEAVE";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("mainLoop : unknown message name : '%s'", msgName.c_str());
+		nlwarning("mainLoop : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1851,14 +1851,14 @@ NLMISC_COMMAND(leaveTeam, "leave team", "")
 NLMISC_COMMAND(joinTeam, "join the specified team", "")
 {
 	// Create the message for the server to execute a phrase.
-	const string msgName = "TEAM:JOIN";
+	const char *msgName = "TEAM:JOIN";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("mainLoop : unknown message name : '%s'", msgName.c_str());
+		nlwarning("mainLoop : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1866,14 +1866,14 @@ NLMISC_COMMAND(joinTeam, "join the specified team", "")
 NLMISC_COMMAND(joinTeamProposal, "propose to current target to join the team", "")
 {
 	// Create the message for the server to execute a phrase.
-	const string msgName = "TEAM:JOIN_PROPOSAL";
+	const char *msgName = "TEAM:JOIN_PROPOSAL";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("mainLoop : unknown message name : '%s'", msgName.c_str());
+		nlwarning("mainLoop : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1881,14 +1881,14 @@ NLMISC_COMMAND(joinTeamProposal, "propose to current target to join the team", "
 NLMISC_COMMAND(joinTeamDecline, "decline a join team proposal", "")
 {
 	// Create the message for the server to execute a phrase.
-	const string msgName = "TEAM:JOIN_PROPOSAL_DECLINE";
+	const char *msgName = "TEAM:JOIN_PROPOSAL_DECLINE";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("mainLoop : unknown message name : '%s'", msgName.c_str());
+		nlwarning("mainLoop : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1896,14 +1896,14 @@ NLMISC_COMMAND(joinTeamDecline, "decline a join team proposal", "")
 NLMISC_COMMAND(kickTeammate, "kick someone from your team", "")
 {
 	// Create the message for the server to execute a phrase.
-	const string msgName = "TEAM:KICK";
+	const char *msgName = "TEAM:KICK";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("mainLoop : unknown message name : '%s'", msgName.c_str());
+		nlwarning("mainLoop : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1913,14 +1913,14 @@ NLMISC_COMMAND(cancelCurrentSentence, "cancel the sentence being executed", "")
 	// no parameter needed
 
 	// Create the message for the server to cancel the phrase being executed
-	const string msgName = "SENTENCE:CANCEL_CURRENT";
+	const char *msgName = "SENTENCE:CANCEL_CURRENT";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
 		NetMngr.push(out);
 	}
 	else
-		nlwarning("command : unknown message name : '%s'", msgName.c_str());
+		nlwarning("command : unknown message name : '%s'", msgName);
 
 	return true;
 }
@@ -1952,7 +1952,7 @@ NLMISC_COMMAND(drop,"drop an item to the ground","<id>")
 	sint32 z = (sint32)UserEntity->pos().z * 1000;
 
 	CBitMemStream bms;
-	string msgType = "ITEM:DROP";
+	const char *msgType = "ITEM:DROP";
 	if( GenericMsgHeaderMngr.pushNameToStream(msgType,bms) )
 	{
 		bms.serial( itemId );
@@ -3769,7 +3769,7 @@ NLMISC_COMMAND( createPerso, "create a new character", "Parameters:\n-Character
 	fromString(args[4], level);
 
 	CBitMemStream bms;
-	string msgType = "CHEAT:CREATE_CHARACTER";
+	const char *msgType = "CHEAT:CREATE_CHARACTER";
 	if( GenericMsgHeaderMngr.pushNameToStream(msgType,bms) )
 	{
 		bms.serial( characterName );
@@ -3800,7 +3800,7 @@ NLMISC_COMMAND( add_role, "add role to character", "<Role( MeleeFighter, RangeFi
 	fromString(args[1], level);
 
 	CBitMemStream bms;
-	string msgType = "CHEAT:ADD_ROLE";
+	const char *msgType = "CHEAT:ADD_ROLE";
 	if( GenericMsgHeaderMngr.pushNameToStream(msgType,bms) )
 	{
 		bms.serialEnum( role );
@@ -4130,11 +4130,11 @@ NLMISC_COMMAND(browseRingAdmin, "Browse a HTML document with the ring web browse
 NLMISC_COMMAND(GUCreate, "create a guild", "<guild name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:CREATE";
+	const char *msgName = "GUILD:CREATE";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
-		string buf = args[0];
+		ucstring buf = ucstring::makeFromUtf8(args[0]); // FIXME: UTF-8 (serial)
 		out.serial( buf );
 		NetMngr.push(out);
 	}
@@ -4144,7 +4144,7 @@ NLMISC_COMMAND(GUCreate, "create a guild", "<guild name>")
 NLMISC_COMMAND(GUQuit, "quit a guild", "")
 {
 	if (args.size() != 0) return false;
-	const string msgName = "GUILD:QUIT";
+	const char *msgName = "GUILD:QUIT";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4156,7 +4156,7 @@ NLMISC_COMMAND(GUQuit, "quit a guild", "")
 NLMISC_COMMAND(GULeaveLeadership, "abandon leadership of a guild", "")
 {
 	if (args.size() != 0) return false;
-	const string msgName = "GUILD:ABANDON_LEADERSHIP";
+	const char *msgName = "GUILD:ABANDON_LEADERSHIP";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4167,7 +4167,7 @@ NLMISC_COMMAND(GULeaveLeadership, "abandon leadership of a guild", "")
 NLMISC_COMMAND(GULeaveOfficerTitle, "abandon officer title", "")
 {
 	if (args.size() != 0) return false;
-	const string msgName = "GUILD:ABANDON_OFFICER_TITLE";
+	const char *msgName = "GUILD:ABANDON_OFFICER_TITLE";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4179,7 +4179,7 @@ NLMISC_COMMAND(GULeaveOfficerTitle, "abandon officer title", "")
 NLMISC_COMMAND(GUNameOfficer, "name an officer", "<player name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:NAME_OFFICER";
+	const char *msgName = "GUILD:NAME_OFFICER";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4193,7 +4193,7 @@ NLMISC_COMMAND(GUNameOfficer, "name an officer", "<player name>")
 NLMISC_COMMAND(GUDismissOfficer, "dismiss an officer", "<player name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:DISMISS_OFFICER";
+	const char *msgName = "GUILD:DISMISS_OFFICER";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4207,7 +4207,7 @@ NLMISC_COMMAND(GUDismissOfficer, "dismiss an officer", "<player name>")
 NLMISC_COMMAND(GUKick, "kick a member", "<player name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:KICK_MEMBER";
+	const char *msgName = "GUILD:KICK_MEMBER";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4234,7 +4234,7 @@ NLMISC_COMMAND(GURefuse, "refuse an invitation", "")
 NLMISC_COMMAND(GUFriend, "invite a player to become a friend of the guild", "<player name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:FRIEND_INVITATION";
+	const char *msgName = "GUILD:FRIEND_INVITATION";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4248,7 +4248,7 @@ NLMISC_COMMAND(GUFriend, "invite a player to become a friend of the guild", "<pl
 NLMISC_COMMAND(GUFriendAccept, "accept to be a friend of a guild that invited you", "")
 {
 	if (args.size() != 0) return false;
-	const string msgName = "GUILD:ACCEPT_FRIEND_INVITATION";
+	const char *msgName = "GUILD:ACCEPT_FRIEND_INVITATION";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4260,7 +4260,7 @@ NLMISC_COMMAND(GUFriendAccept, "accept to be a friend of a guild that invited yo
 NLMISC_COMMAND(GUFriendRefuse, "refuse to be a friend of a guild that invited you", "")
 {
 	if (args.size() != 0) return false;
-	const string msgName = "GUILD:REFUSE_FRIEND_INVITATION";
+	const char *msgName = "GUILD:REFUSE_FRIEND_INVITATION";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4272,7 +4272,7 @@ NLMISC_COMMAND(GUFriendRefuse, "refuse to be a friend of a guild that invited yo
 NLMISC_COMMAND(GUSetSuccessor, "set the successor of the guild leader", "<player name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:SET_SUCCESSOR";
+	const char *msgName = "GUILD:SET_SUCCESSOR";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4286,7 +4286,7 @@ NLMISC_COMMAND(GUSetSuccessor, "set the successor of the guild leader", "<player
 NLMISC_COMMAND(GUInfos, "get information on a guild", "<guild name>")
 {
 	if (args.size() != 1) return false;
-	const string msgName = "GUILD:GET_INFOS";
+	const char *msgName = "GUILD:GET_INFOS";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4300,7 +4300,7 @@ NLMISC_COMMAND(GUInfos, "get information on a guild", "<guild name>")
 NLMISC_COMMAND(GUJournal, "get the guild journal", "")
 {
 	if (args.size() != 0) return false;
-	const string msgName = "GUILD:GET_LOG";
+	const char *msgName = "GUILD:GET_LOG";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{
@@ -4314,7 +4314,7 @@ NLMISC_COMMAND(buildingTeleport, "teleport to a building", "building index")
 	if (args.size() != 1) return false;
 	uint16 index;
 	fromString(args[0], index);
-	const string msgName = "GUILD:TELEPORT";
+	const char *msgName = "GUILD:TELEPORT";
 	CBitMemStream out;
 	if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 	{

ryzom/client/src/connection.cpp

@@ -1708,7 +1708,7 @@ public:
 		string sCharSumPath = getParam(Params, "charsum");
 		SCharacter3DSetup::setupCharacterSummaryFromDB(CS, sCharSumPath);
 		CS.Mainland = MainlandSelected;
-		CS.Name = sFirstName;
+		CS.Name = ucstring::makeFromUtf8(sFirstName); // FIXME: UTF-8 (serial)
 		//CS.Surname = sSurName;
 
 		// Create the message to send to the server from the character summary
@@ -1865,8 +1865,9 @@ string getTarget(CCtrlBase * /* ctrl */, const string &targetName)
 	return "";
 }
 
+#ifdef RYZOM_LUA_UCSTRING
 // ------------------------------------------------------------------------------------------------
-ucstring getUCTarget(CCtrlBase * /* ctrl */, const string &targetName) // TODO: UTF-8 Lua
+ucstring getUCTarget(CCtrlBase * /* ctrl */, const string &targetName)
 {
 	string sTmp = targetName;
 	std::vector<CInterfaceLink::CTargetInfo> targetsVector;
@@ -1886,6 +1887,7 @@ ucstring getUCTarget(CCtrlBase * /* ctrl */, const string &targetName) // TODO:
 		return ((elem->*(pRP->GetMethod.GetUCString))());
 	return ucstring(""); // TODO: UTF-8 Lua
 }
+#endif
 
 /*// Ask the server to rename a character
 // ------------------------------------------------------------------------------------------------
@@ -1957,7 +1959,11 @@ public:
 		string sDBLink = getParam(Params, "dblink");
 		CharNameValidDBLink = sDBLink;
 
+#ifdef RYZOM_LUA_UCSTRING
 		string sName = getUCTarget(NULL,sTarget).toUtf8(); // TODO: UTF-8 Lua
+#else
+		string sName = getTarget(NULL, sTarget);
+#endif
 
 		CInterfaceManager *pIM = CInterfaceManager::getInstance();
 		if (sName.empty())

ryzom/client/src/far_tp.cpp

@@ -1010,13 +1010,13 @@ void CFarTP::hookNextFarTPForEditor()
  */
 void CFarTP::requestReturnToPreviousSession(TSessionId rejectedSessionId)
 {
-	const string msgName = "CONNECTION:RET_MAINLAND";
+	const char *msgName = "CONNECTION:RET_MAINLAND";
 	CBitMemStream out;
 	nlverify(GenericMsgHeaderMngr.pushNameToStream(msgName, out));
 	out.serial(PlayerSelectedSlot);
 	out.serial(rejectedSessionId);
 	NetMngr.push(out);
-	nlinfo("%s sent", msgName.c_str());
+	nlinfo("%s sent", msgName);
 }
 
 /*

ryzom/client/src/interface_v3/action_handler_game.cpp

@@ -1166,7 +1166,7 @@ public:
 			game_exit_request = true;
 			ryzom_exit_request = true;
 
-			const string msgName = "CONNECTION:CLIENT_QUIT_REQUEST";
+			const char *msgName = "CONNECTION:CLIENT_QUIT_REQUEST";
 			CBitMemStream out;
 			nlverify(GenericMsgHeaderMngr.pushNameToStream(msgName, out));
 			bool bypassDisconnectionTimer = FarTP.isFastDisconnectGranted() && (!IsInRingSession); // no need on a ring shard, as it's very short
@@ -1489,7 +1489,7 @@ void beastOrder (const std::string &orderStr, const std::string &beastIndexStr,
 	else
 	{
 		// execute the order.
-		const string msgName = "ANIMALS:BEAST";
+		const char *msgName = "ANIMALS:BEAST";
 		CBitMemStream out;
 		if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 		{
@@ -1500,7 +1500,7 @@ void beastOrder (const std::string &orderStr, const std::string &beastIndexStr,
 			NetMngr.push(out);
 		}
 		else
-			nlwarning("<beastOrder> : unknown message name : '%s'.", msgName.c_str());
+			nlwarning("<beastOrder> : unknown message name : '%s'.", msgName);
 	}
 }
 
@@ -4530,7 +4530,7 @@ public:
 		if( sCustomPhrase.empty() )
 		{
 			// Create the message and send.
-			static const string msgName = "COMMAND:EMOTE";
+			static const char *msgName = "COMMAND:EMOTE";
 			CBitMemStream out;
 			if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 			{
@@ -4540,12 +4540,12 @@ public:
 				//nlinfo("impulseCallBack : %s %d %d sent", msgName.c_str(), (uint32)behavToSend, phraseNbToSend);
 			}
 			else
-				nlwarning("command 'emote': unknown message named '%s'.", msgName.c_str());
+				nlwarning("command 'emote': unknown message named '%s'.", msgName);
 		}
 		else
 		{
 			// Create the message and send.
-			static const string msgName = "COMMAND:CUSTOM_EMOTE";
+			static const char *msgName = "COMMAND:CUSTOM_EMOTE";
 			CBitMemStream out;
 			if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 			{
@@ -4575,7 +4575,7 @@ public:
 				//nlinfo("impulseCallBack : %s %d %s sent", msgName.c_str(), (uint32)behavToSend, sCustomPhrase.c_str());
 			}
 			else
-				nlwarning("command 'emote': unknown message named '%s'.", msgName.c_str());
+				nlwarning("command 'emote': unknown message named '%s'.", msgName);
 		}
 	}
 };

ryzom/client/src/interface_v3/action_handler_help.cpp

@@ -3707,17 +3707,17 @@ public:
 			sint n = PeopleInterraction.TeamList.getIndexFromName(selection->getEntityName());
 			if (n >= 0)
 			{
-				const string msgName = "TEAM:KICK";
+				const char *msgName = "TEAM:KICK";
 				CBitMemStream out;
 				if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 				{
 					uint8 teamMember = (uint8)n;
 					out.serialEnum(teamMember);
 					NetMngr.push(out);
-					//nlinfo("impulseCallBack : %s %d sent", msgName.c_str(), teamMember);
+					//nlinfo("impulseCallBack : %s %d sent", msgName, teamMember);
 				}
 				else
-					nlwarning("unknown message named '%s'.", msgName.c_str());
+					nlwarning("unknown message named '%s'.", msgName);
 			}
 		}
 	}

ryzom/client/src/interface_v3/action_handler_item.cpp

@@ -417,10 +417,10 @@ void CInterfaceItemEdition::CItemEditionWindow::validate()
 			if (textValid)
 			{
 				CBitMemStream out;
-				const string msgName = "EVENT:SET_ITEM_CUSTOM_TEXT";
+				const char *msgName = "EVENT:SET_ITEM_CUSTOM_TEXT";
 				if (!GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 				{
-					nlwarning ("don't know message name %s", msgName.c_str());
+					nlwarning ("don't know message name %s", msgName);
 				}
 				else
 				{
@@ -456,7 +456,7 @@ static void checkItemCommand(const CItemSheet *itemSheet);
 static void sendSwapItemMsg(const CDBCtrlSheet *pCSSrc, const CDBCtrlSheet *pCSDst, sint32 quantitySrc)
 {
 	CBitMemStream out;
-	const string sMsg = "ITEM:SWAP";
+	const char *sMsg = "ITEM:SWAP";
 	if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 	{
 		// Swap all the Src (quantity= quantitySrc) to dest
@@ -489,7 +489,7 @@ static void sendSwapItemMsg(const CDBCtrlSheet *pCSSrc, const CDBCtrlSheet *pCSD
 		//nlinfo("impulseCallBack : %s %d %d %d %d %d sent", sMsg.c_str(), srcInvId, srcSlotId, dstInvId, dstSlotId, quantity);
 	}
 	else
-		nlwarning(" unknown message name '%s'",sMsg.c_str());
+		nlwarning(" unknown message name '%s'",sMsg);
 }
 
 /** Display the popup to ask item quantity
@@ -567,7 +567,7 @@ static void openStackItem(CCtrlBase *pCaller, CDBCtrlSheet *pCSSrc, CDBCtrlSheet
 		CInterfaceManager	*pIM= CInterfaceManager::getInstance();
 
 		CBitMemStream out;
-		const string sMsg = "EXCHANGE:ADD";
+		const char *sMsg = "EXCHANGE:ADD";
 		if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 		{
 			// Swap all the Src (quantity= quantitySrc) to dest
@@ -580,7 +580,7 @@ static void openStackItem(CCtrlBase *pCaller, CDBCtrlSheet *pCSSrc, CDBCtrlSheet
 			//nlinfo("impulseCallBack : %s %d %d %d sent", sMsg.c_str(), srcSlotIndex, destSlotIndex, quantitySrc);
 		}
 		else
-			nlwarning(" unknown message name '%s'",sMsg.c_str());
+			nlwarning(" unknown message name '%s'",sMsg);
 	}
 
 //=====================================================================================================================
@@ -636,8 +636,8 @@ static void openStackItem(CCtrlBase *pCaller, CDBCtrlSheet *pCSSrc, CDBCtrlSheet
 
 		// send msg to server
 		CBitMemStream out;
-		const string sMsg = "EXCHANGE:REMOVE";
+		const char *sMsg = "EXCHANGE:REMOVE";
-		if(GenericMsgHeaderMngr.pushNameToStream(sMsg.c_str(), out))
+		if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 		{
 			// Swap all the Src (quantity= quantitySrc) to dest
 			uint16	slotIndex =	(uint16) exchangeSlot->getIndexInDB();
@@ -647,7 +647,7 @@ static void openStackItem(CCtrlBase *pCaller, CDBCtrlSheet *pCSSrc, CDBCtrlSheet
 			//nlinfo("impulseCallBack : %s %d sent", sMsg.c_str(), slotIndex);
 		}
 		else
-			nlwarning(" unknown message name '%s'",sMsg.c_str());
+			nlwarning(" unknown message name '%s'",sMsg);
 	}
 
 
@@ -1658,7 +1658,7 @@ REGISTER_ACTION_HANDLER( CHandlerDragNDrop, "drag_n_drop" );
 static void sendToServerEnchantMessage(uint8 invent, uint16 slot)
 {
 	CBitMemStream out;
-	const string sMsg = "ITEM:ENCHANT";
+	const char *sMsg = "ITEM:ENCHANT";
 
 	if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 	{
@@ -1667,7 +1667,7 @@ static void sendToServerEnchantMessage(uint8 invent, uint16 slot)
 		NetMngr.push(out);
 	}
 	else
-		nlinfo("unknown message %s", sMsg.c_str());
+		nlinfo("unknown message %s", sMsg);
 }
 
 // **********************************************************************************************************
@@ -2230,7 +2230,7 @@ static void sendMsgUseItem(uint16 slot)
 	if(!ClientCfg.Local)
 	{
 		CBitMemStream out;
-		const string sMsg = "ITEM:USE_ITEM";
+		const char *sMsg = "ITEM:USE_ITEM";
 
 		if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 		{
@@ -2238,7 +2238,7 @@ static void sendMsgUseItem(uint16 slot)
 			NetMngr.push(out);
 		}
 		else
-			nlinfo("unknown message %s", sMsg.c_str());
+			nlinfo("unknown message %s", sMsg);
 	}
 }
 
@@ -2248,7 +2248,7 @@ static void sendMsgStopUseXpCat( bool isRingCatalyser )
 	if(!ClientCfg.Local)
 	{
 		CBitMemStream out;
-		const string sMsg = "ITEM:STOP_USE_XP_CAT";
+		const char *sMsg = "ITEM:STOP_USE_XP_CAT";
 
 		if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 		{
@@ -2256,7 +2256,7 @@ static void sendMsgStopUseXpCat( bool isRingCatalyser )
 			NetMngr.push(out);
 		}
 		else
-			nlinfo("unknown message %s", sMsg.c_str());
+			nlinfo("unknown message %s", sMsg);
 	}
 }
 

ryzom/client/src/interface_v3/action_handler_outpost.cpp

@@ -471,7 +471,7 @@ public:
 			startAttackTime= node->getValue32();
 
 		// send a DECLARE_WAR_VALIDATE message to server
-		string	sMsg= "OUTPOST:DECLARE_WAR_VALIDATE";
+		const char *sMsg= "OUTPOST:DECLARE_WAR_VALIDATE";
 		CBitMemStream out;
 		if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 		{
@@ -483,7 +483,7 @@ public:
 		}
 		else
 		{
-			nlwarning("command : unknown message name : '%s'.", sMsg.c_str());
+			nlwarning("command : unknown message name : '%s'.", sMsg);
 		}
 
 	}
@@ -508,7 +508,7 @@ public:
 		// Send a msg to server
 		if(outpostSheet)
 		{
-			string	sMsg= "OUTPOST:SELECT";
+			const char *sMsg= "OUTPOST:SELECT";
 			CBitMemStream out;
 			if(GenericMsgHeaderMngr.pushNameToStream(sMsg, out))
 			{
@@ -518,7 +518,7 @@ public:
 			}
 			else
 			{
-				nlwarning("command : unknown message name : '%s'.", sMsg.c_str());
+				nlwarning("command : unknown message name : '%s'.", sMsg);
 			}
 		}
 	}

ryzom/client/src/interface_v3/bot_chat_page_create_guild.cpp

@@ -79,7 +79,7 @@ class CHandlerGuildCreate : public IActionHandler
 		uint64 icon = CGuildManager::iconMake((uint8)pCS->getGuildBack(), (uint8)pCS->getGuildSymbol(),
 								pCS->getInvertGuildSymbol(), pCS->getGuildColor1(), pCS->getGuildColor2());
 
-		const string msgName = "GUILD:CREATE";
+		const char *msgName = "GUILD:CREATE";
 		NLMISC::CBitMemStream out;
 		if(GenericMsgHeaderMngr.pushNameToStream(msgName, out))
 		{

ryzom/client/src/release.cpp

@@ -43,6 +43,7 @@
 #include "nel/3d/u_visual_collision_manager.h"
 #include "nel/3d/u_shape_bank.h"
 #include "nel/3d/stereo_hmd.h"
+#include "nel/3d/async_file_manager_3d.h"
 // Client
 #include "global.h"
 #include "release.h"
@@ -573,6 +574,8 @@ void release()
 		CLoginProgressPostThread::getInstance().step(CLoginStep(LoginStep_GameExit, "login_step_game_exit&play_time=" + toString((NLMISC::CTime::getLocalTime() - StartPlayTime) / 1000)));
 	}
 	
+	setCrashCallback(NULL);
+
 #ifdef RYZOM_BG_DOWNLOADER
 	CBGDownloaderAccess::getInstance().release();
 #endif
@@ -629,6 +632,9 @@ void release()
 		delete FXAA; FXAA = NULL;
 		CBloomEffect::releaseInstance();
 
+		// Release texture
+		endLoading();
+
 		// Release Scene, textcontexts, materials, ...
 		Driver->release();
 
@@ -701,6 +707,7 @@ void release()
 	CIXml::releaseLibXml();
 	CHttpCache::release();
 	CStrictTransportSecurity::release();
+	CAsyncFileManager3D::releaseInstance();
 
 #if FINAL_VERSION
 	// openURL ("http://ryzom.com/exit/");

ryzom/client/src/string_manager_client.cpp

@@ -127,13 +127,16 @@ namespace STRING_MANAGER
 
 				nlinfo("SM : Try to open the string cache : %s", _CacheFilename.c_str());
 
-				if (CFile::fileExists(_CacheFilename))
+				if (CFile::fileExists(_CacheFilename) && CFile::getFileSize(_CacheFilename))
 				{
 					// there is a cache file, check date reset it if needed
 					{
 						NLMISC::CIFile file(_CacheFilename);
 						file.setVersionException(false, false);
 						file.serialVersion(currentVersion);
+						if (file.getPos() + sizeof(_Timestamp) > file.getFileSize())
+							_Timestamp = ~timestamp;
+						else
 							file.serial(_Timestamp);
 					}
 

ryzom/client/src/string_manager_client.h

@@ -46,6 +46,7 @@ class CStringManagerClient
 public:
 	// Singleton pattern implementation
 	static CStringManagerClient	*instance();
+	static bool hasInstance() { return _Instance; }
 	static void	release(bool mustReleaseStaticArrays);
 
 	/** Prepare the string manager to use a persistent string cache.
@@ -388,6 +389,7 @@ public:
 	virtual ~IStringWaiterRemover()
 	{
 		// signal the string manager that this waiter is destroyed
+		if (CStringManagerClient::hasInstance())
 			CStringManagerClient::instance()->removeStringWaiter(this);
 	}
 };
@@ -406,6 +408,7 @@ public:
 	virtual ~IStringWaitCallback()
 	{
 		// signal the string manager that this waiter is destroyed
+		if (CStringManagerClient::hasInstance())
 			CStringManagerClient::instance()->removeStringWaiter(this);
 	}
 

ryzom/tools/leveldesign/georges_dll/georges_edit_doc.cpp

@@ -184,7 +184,7 @@ bool CGeorgesEditDocForm::initDocument (const char *dfnName, bool newElement)
 	}
 
 	// Set file name and title
-	std::string name2 = toLower(NLMISC::CFile::getFilenameWithoutExtension(dfnName));
+	std::string name2 = toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(dfnName));
 	SetPathName(nlUtf8ToTStr("*." + name2), FALSE);
 	SetTitle(nlUtf8ToTStr("New " + name2 + " form"));
 
@@ -547,7 +547,7 @@ BOOL CGeorgesEditDoc::OnOpenDocument(LPCTSTR lpszPathName)
 		{
 			// Check form
 			std::string ext = NLMISC::CFile::getExtension(tStrToUtf8(lpszPathName));
-			string extLower = toLower(ext);
+			string extLower = toLowerAscii(ext);
 
 			if (!extLower.empty ())
 			{

ryzom/tools/leveldesign/georges_dll/imagelist_ex.cpp

@@ -107,7 +107,7 @@ void CImageListEx::addResourceIcon (const char *filename)
 		
 			// Add in the map
 			std::string name = NLMISC::CFile::getFilenameWithoutExtension(filename);
-			_IconMapString.insert (std::map<string, int>::value_type (toLower(name), index));
+			_IconMapString.insert (std::map<string, int>::value_type (toLowerAscii(name), index));
 
 			// Release the icon
 			DestroyIcon (handle);
@@ -132,7 +132,7 @@ int CImageListEx::getImage (int resource) const
 
 int CImageListEx::getImage (const char *filename) const
 {
-	std::string name = toLower(NLMISC::CFile::getFilenameWithoutExtension(filename));
+	std::string name = toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(filename));
 	std::map<string, int>::const_iterator ite = _IconMapString.find (name);
 	if (ite == _IconMapString.end())
 		return -1;

ryzom/tools/leveldesign/mission_compiler_fe/mission_compiler_feDlg.cpp

@@ -239,8 +239,8 @@ BOOL CMissionCompilerFeDlg::OnInitDialog()
 			{
 				// We found the same filename : check the path
 				string srcPath = it->second;
-				srcPath  = toUpper(CPath::standardizeDosPath(srcPath));
+				srcPath  = toUpperAscii(CPath::standardizeDosPath(srcPath));
-				files[i] = toUpper(CPath::standardizeDosPath(files[i]));
+				files[i] = toUpperAscii(CPath::standardizeDosPath(files[i]));
 				if (srcPath != files[i])
 				{
 					::MessageBox(NULL, _T("Primitive path and working directory are not the same !"),

ryzom/tools/leveldesign/mission_compiler_lib/mission_compiler.cpp

@@ -189,7 +189,7 @@ GenderExtractor::GenderExtractor(const std::string & literal, const std::string&
 	const char * f = fs[level];
 	const char * h = hs[level];
 
-	_Identifier = toLower(identifier);
+	_Identifier = toLowerAscii(identifier);
 
 	std::string newPhrase;
 
@@ -657,7 +657,7 @@ bool CMissionCompiler::installCompiledMission(NLLIGO::CLigoConfig &ligoConfig, c
 				// use mission primitive instead
 				fileName = primFileName;
 			}
-			if (loadedPrimitives.find(toLower(fileName)) == loadedPrimitives.end())
+			if (loadedPrimitives.find(toLowerAscii(fileName)) == loadedPrimitives.end())
 			{
 				string fullFileName = CPath::lookup(fileName, false);
 				if (fullFileName.empty())
@@ -670,7 +670,7 @@ bool CMissionCompiler::installCompiledMission(NLLIGO::CLigoConfig &ligoConfig, c
 				if (loadXmlPrimitiveFile(*primDoc, fullFileName, ligoConfig))
 				{
 					// the primitive file is loaded correctly
-					loadedPrimitives.insert(make_pair(toLower(fileName), TLoadedPrimitive(primDoc, fullFileName)));
+					loadedPrimitives.insert(make_pair(toLowerAscii(fileName), TLoadedPrimitive(primDoc, fullFileName)));
 					CPrimitiveContext::instance().CurrentPrimitive = NULL;
 				}
 				else
@@ -679,7 +679,7 @@ bool CMissionCompiler::installCompiledMission(NLLIGO::CLigoConfig &ligoConfig, c
 					throw EParseException(NULL, toString("Can't read primitive file '%s'", fullFileName.c_str()).c_str());
 				}
 			}
-			TLoadedPrimitive &loadedPrim = loadedPrimitives[toLower(fileName)];
+			TLoadedPrimitive &loadedPrim = loadedPrimitives[toLowerAscii(fileName)];
 			CPrimitives *primDoc = loadedPrim.PrimDoc;
 
 			TPrimitiveSet scripts;
@@ -744,7 +744,7 @@ bool CMissionCompiler::installCompiledMission(NLLIGO::CLigoConfig &ligoConfig, c
 				fileName = primFileName;
 			}
 
-			TLoadedPrimitive &loadedPrim = loadedPrimitives[toLower(fileName)];
+			TLoadedPrimitive &loadedPrim = loadedPrimitives[toLowerAscii(fileName)];
 			CPrimitives *primDoc = loadedPrim.PrimDoc;
 			CPrimitiveContext::instance().CurrentPrimitive = primDoc;
 
@@ -1304,7 +1304,7 @@ retry:
 		_Suffixe = tokens.back();
 
 		// generate identifier
-		_PhraseId = toUpper(md.getMissionName()+"_"+_Suffixe);
+		_PhraseId = toUpperAscii(md.getMissionName()+"_"+_Suffixe);
 
 		set<string>	ps;
 		// select only unique params
@@ -1857,11 +1857,11 @@ void CMissionData::parseMissionHeader(NLLIGO::IPrimitive *prim)
 //	_MissionTitle.init(*this, prim, vs); 
 	_MissionDescriptionRaw = getPropertyArray(prim, "mission_description", false, false);
 //	_MissionDescription.init(*this, prim, vs);
-	_MonoInstance = toLower(getProperty(prim, "mono_instance", true, false)) == "true";
+	_MonoInstance = toBool(getProperty(prim, "mono_instance", true, false));
-	_RunOnce = toLower(getProperty(prim, "run_only_once", true, false)) == "true";
+	_RunOnce = toBool(getProperty(prim, "run_only_once", true, false));
-	_Replayable = toLower(getProperty(prim, "replayable", true, false)) == "true";
+	_Replayable = toBool(getProperty(prim, "replayable", true, false));
 	
-	_NeedValidation = toLower(getProperty(prim, "need_validation", true, false)) == "true";
+	_NeedValidation = toBool(getProperty(prim, "need_validation", true, false));
 	
 	_MissionAutoMenuRaw = getPropertyArray(prim, "phrase_auto_menu", false, false);
 
@@ -1872,15 +1872,15 @@ void CMissionData::parseMissionHeader(NLLIGO::IPrimitive *prim)
 	else if (s == "guild")
 		_Guild = true;
 	
-	_NotInJournal = NLMISC::toLower(getProperty(prim, "not_in_journal", false, false)) == "true";
+	_NotInJournal = NLMISC::toBool(getProperty(prim, "not_in_journal", false, false));
-	_AutoRemoveFromJournal = NLMISC::toLower(getProperty(prim, "auto_remove_from_journal", false, false)) == "true";
+	_AutoRemoveFromJournal = NLMISC::toBool(getProperty(prim, "auto_remove_from_journal", false, false));
 	_MissionCategory = getProperty(prim, "mission_category", false, false);
 	NLMISC::fromString(getProperty(prim, "player_replay_timer", true, false), _PlayerReplayTimer);
 	NLMISC::fromString(getProperty(prim, "global_replay_timer", true, false), _GlobalReplayTimer);
-	_NotProposed = NLMISC::toLower(getProperty(prim, "not_proposed", false, false)) == "true";
+	_NotProposed = NLMISC::toBool(getProperty(prim, "not_proposed", false, false));
-	_MissionAuto = NLMISC::toLower(getProperty(prim, "automatic", false, false)) == "true";
+	_MissionAuto = NLMISC::toBool(getProperty(prim, "automatic", false, false));
-	_NonAbandonnable = NLMISC::toLower(getProperty(prim, "non_abandonnable", false, false)) == "true";
+	_NonAbandonnable = NLMISC::toBool(getProperty(prim, "non_abandonnable", false, false));
-	_FailIfInventoryIsFull = NLMISC::toLower(getProperty(prim, "fail_if_inventory_is_full", false, false)) == "true";
+	_FailIfInventoryIsFull = NLMISC::toBool(getProperty(prim, "fail_if_inventory_is_full", false, false));
 	_MissionIcon = getProperty(prim, "mission_icon", false, false);
 
 	if (_MissionAuto)

ryzom/tools/leveldesign/mission_compiler_lib/step_content.cpp

@@ -471,7 +471,7 @@ public:
 
 		string s;
 		s = md.getProperty(prim, "hide_others", true, false);
-		_HideOthers = (NLMISC::toLower(s) == "true");
+		_HideOthers = NLMISC::toBool(s);
 	}
 
 	string genCode(CMissionData &md)
@@ -788,7 +788,7 @@ public:
 
 		string s;
 		s = md.getProperty(prim, "group", true, false);
-		_Group = (NLMISC::toLower(s) == "true");
+		_Group = NLMISC::toBool(s);
 
 		_Guild = md.getProperty(prim, "guild", false, true) == "true";
 		// Check: if _Guild is true then check if we are in a guild mission
@@ -866,7 +866,7 @@ public:
 		
 		string s;
 		s = md.getProperty(prim, "group", true, false);
-		_Group = (NLMISC::toLower(s) == "true");
+		_Group = NLMISC::toBool(s);
 
 		_Guild = md.getProperty(prim, "guild", false, true) == "true";
 		// Check: if _Guild is true then check if we are in a guild mission
@@ -1054,7 +1054,7 @@ public:
 
 		string s;
 		s = md.getProperty(prim, "group", true, false);
-		_Group = (NLMISC::toLower(s) == "true");
+		_Group = NLMISC::toBool(s);
 
 		IStepContent::init(md, prim);
 	}
@@ -1111,7 +1111,7 @@ public:
 
 		string s;
 		s = md.getProperty(prim, "group", true, false);
-		_Group = (NLMISC::toLower(s) == "true");
+		_Group = NLMISC::toBool(s);
 
 		IStepContent::init(md, prim);
 	}
@@ -1169,7 +1169,7 @@ public:
 
 		string s;
 		s = md.getProperty(prim, "group", true, false);
-		_Group = (NLMISC::toLower(s) == "true");
+		_Group = NLMISC::toBool(s);
 
 		IStepContent::init(md, prim);
 	}
@@ -1320,7 +1320,7 @@ public:
 		_WorldPosition = md.getProperty(prim, "world_position", true, false);
 		string s;
 		prim->getPropertyByName("once", s);
-		_Once = (NLMISC::toLower(s) == "true");
+		_Once = NLMISC::toBool(s);
 	}
 
 	string genCode(CMissionData &md)
@@ -3212,7 +3212,7 @@ public:
 	{
 		_GroupName = md.getProperty(prim, "group_to_escort", true, false);
 		string s = md.getProperty(prim, "save_all", true, false);
-		_SaveAll = (NLMISC::toLower(s) == "true");
+		_SaveAll = NLMISC::toBool(s);
 
 		CContentObjective::init(md, prim);
 	}

ryzom/tools/leveldesign/prim_export/main.cpp

@@ -527,7 +527,7 @@ public:
 		formDate = 0;
 
 		// In the map ?
-		string formShortName = NLMISC::toLower(CFile::getFilename (formName));
+		string formShortName = NLMISC::toLowerAscii(CFile::getFilename (formName));
 		map<string, CValue >::iterator ite = _FormMap.find (formShortName);
 		if (ite == _FormMap.end ())
 		{
@@ -614,7 +614,7 @@ void addPointPrimitive (CLandscape &landscape, const char *primFilename, uint32
 				if (point->getPropertyByName ("form", plantFilename))
 				{
 					// Add an extension
-					if (NLMISC::toLower(CFile::getExtension (plantFilename)) != "plant")
+					if (NLMISC::toLowerAscii(CFile::getExtension (plantFilename)) != "plant")
 						plantFilename += ".plant";
 
 					// Load this form
@@ -669,7 +669,7 @@ void addPointPrimitive (CLandscape &landscape, const char *primFilename, uint32
 								instance.Scale = CVector (scale, scale, scale);
 								instance.nParent = -1;
 								instance.Name = shape;
-								instance.InstanceName = NLMISC::toLower(CFile::getFilename (plantFilename));
+								instance.InstanceName = NLMISC::toLowerAscii(CFile::getFilename (plantFilename));
 
 								// Get the instance group ref
 								CIgContainer::CIG	&instances = igs.get (x, y);
@@ -826,7 +826,7 @@ int main (int argc, char**argv)
 				callback.progress ((float)i/(float)files.size ());
 
 				// Zonew ?
-				if (NLMISC::toLower(CFile::getExtension (files[i])) == "zonew")
+				if (NLMISC::toLowerAscii(CFile::getExtension (files[i])) == "zonew")
 				{
 					// Load it
 					try
@@ -890,7 +890,7 @@ int main (int argc, char**argv)
 			for (i=0; i<fileCount; i++)
 			{
 				// Primitive file ?
-				if (NLMISC::toLower(CFile::getExtension (files[i])) == "primitive")
+				if (NLMISC::toLowerAscii(CFile::getExtension (files[i])) == "primitive")
 				{
 					// Progress bar
 					nlinfo (files[i].c_str());

ryzom/tools/leveldesign/uni_conv/uni_conv.cpp

@@ -17,6 +17,7 @@
 
 #include <nel/misc/types_nl.h>
 #include <nel/misc/ucstring.h>
+#include <nel/misc/utf_string_view.h>
 #include <nel/misc/common.h>
 #include <nel/misc/sstring.h>
 #include <nel/misc/i18n.h>
@@ -76,21 +77,10 @@ int main(int argc, char *argv[])
 	ucstring	str;
 	CI18N::readTextFile(inputFile, str, false, false);
 
-	if (outMode == ASCII)
-	{
-		// remove any outof ascii char
-		ucstring temp;
-		for (uint i=0; i<str.size(); ++i)
-		{
-			if (str[i] < 256)
-				temp += str[i];
-		}
-		str = temp;
-	}
-
 	if (xmlSupport)
 	{
 		ucstring temp;
+		temp.reserve(str.size());
 		for (uint i=0; i<str.size(); ++i)
 		{
 			switch(str[i])
@@ -121,9 +111,16 @@ int main(int argc, char *argv[])
 		break;
 	case ASCII:
 		{
-			string s = str.toString();
+			std::string res;
+			res.reserve(str.size());
+			for (ucstring::const_iterator it(str.begin()), end(str.end()); it != end; ++it)
+			{
+				ucchar c = *it;
+				if (c < 0x80)
+					res += (char)c;
+			}
 			FILE *fp = nlfopen(outputFile, "wt");
-			fwrite(s.data(), s.size(), 1, fp);
+			fwrite(res.data(), res.size(), 1, fp);
 			fclose(fp);
 		}
 		break;

ryzom/tools/leveldesign/world_editor/world_editor/dialog_properties.cpp

@@ -3485,7 +3485,7 @@ BOOL CMyComboBox::PreTranslateMessage( MSG* pMsg )
 			_LastString.clear();
 
 		// Add this char
-		_LastString.push_back (tolower((TCHAR) pMsg->wParam));
+		_LastString.push_back((wchar_t)NLMISC::toLower((ucchar)pMsg->wParam));
 		search = true;
 
 		// New stroke time
@@ -3518,7 +3518,7 @@ BOOL CMyComboBox::PreTranslateMessage( MSG* pMsg )
 		{
 			// 0 final
 			text[511] = 0;
-			_LastString = tStrToUtf8(text);
+			_LastString = CUtfStringView(tStrToUtf8(text)).toWide();
 			search = true;
 		}
 	}
@@ -3538,11 +3538,11 @@ BOOL CMyComboBox::PreTranslateMessage( MSG* pMsg )
 		{
 			CString rString;
 			GetLBText (i, rString);
-			string tmp = toLower(tStrToUtf8(rString));
+			wstring tmp = CUtfStringView(toLower(tStrToUtf8(rString))).toWide();
 			uint size = std::min (_LastString.size(), tmp.size());
 			if (size > matchSize)
 			{
-				if (strncmp(tmp.c_str(), _LastString.c_str(), size) == 0)
+				if (wcsncmp(tmp.c_str(), _LastString.c_str(), size) == 0)
 				{
 					matchItem = i;
 					matchSize = size;

ryzom/tools/leveldesign/world_editor/world_editor/dialog_properties.h

@@ -60,7 +60,7 @@ public:
 	bool						loaded;
 
 private:
-	std::string					_LastString;
+	std::wstring				_LastString;
 	sint64						_LastStrokeTime;
 	std::vector<std::string>	_data;
 	void						reloadData();

ryzom/tools/leveldesign/world_editor/world_editor/imagelist_ex.cpp

@@ -110,7 +110,7 @@ void CImageListEx::addResourceIcon (const char *filename)
 			index = ImageList.Replace( index, handle);
 		
 			// Add in the map
-			std::string name = toLower(NLMISC::CFile::getFilenameWithoutExtension(filename));
+			std::string name = toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(filename));
 			_IconMapString.insert (std::map<string, int>::value_type (name, index));
 
 			// Release the icon
@@ -136,7 +136,7 @@ int CImageListEx::getImage (int resource) const
 
 int CImageListEx::getImage (const char *filename) const
 {
-	std::string name = toLower(NLMISC::CFile::getFilenameWithoutExtension(filename));
+	std::string name = toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(filename));
 	std::map<string, int>::const_iterator ite = _IconMapString.find (name);
 	if (ite == _IconMapString.end())
 		return -1;

ryzom/tools/leveldesign/world_editor/world_editor/name_dlg.cpp

@@ -167,7 +167,7 @@ void CNameDlg::updateSearchList()
 			m_nameFilter.UnlockBuffer();
 
 			// filter
-			if (NLMISC::toLower(ig).find(NLMISC::toLower(filter)) != std::string::npos)
+			if (NLMISC::toLowerAscii(ig).find(NLMISC::toLowerAscii(filter)) != std::string::npos)
 			{
 				m_listToName.insert(std::make_pair(j, i));
 				m_searchList.InsertString(j++, nlUtf8ToTStr(s));

ryzom/tools/leveldesign/world_editor/world_editor/world_editor.cpp

@@ -555,7 +555,7 @@ bool CWorldEditorApp::initPath (const std::string &filename, CSplashScreen &spla
 							string noRecurse;
 							if (CIXml::getPropertyString (noRecurse, search_path, "NO_RECURSE"))
 							{
-								if (toLower(noRecurse) == "true")
+								if (toBool(noRecurse))
 									recurse = false;
 							}
 
@@ -635,14 +635,14 @@ class CMainFrame *getMainFrame ()
 
 std::string standardizePath (const std::string &str)
 {
-	return NLMISC::toLower(NLMISC::CPath::standardizePath (str, true));
+	return NLMISC::toLowerAscii(NLMISC::CPath::standardizePath (str, true));
 }
 
 // ***************************************************************************
 
 std::string formatString (const std::string &str)
 {
-	string copy = NLMISC::toLower(str);
+	string copy = NLMISC::toLowerAscii(str);
 	return copy;
 }
 

ryzom/tools/leveldesign/world_editor/world_editor_fauna_graph_plugin/plugin.cpp

@@ -222,7 +222,7 @@ void CPlugin::drawFaunaGraph(CDisplay &display, const NLLIGO::IPrimitive &grp)
 				if (!grp.getChild(child, k)) continue;
 				std::string className;
 				if (!child->getPropertyByName("class", className)) continue;
-				if (NLMISC::toLower(className) != FAUNA_PLACE) continue;
+				if (NLMISC::toLowerAscii(className) != FAUNA_PLACE) continue;
 				if (_PluginAccess->isSelected(*child))
 				{
 					found = true;
@@ -239,7 +239,7 @@ void CPlugin::drawFaunaGraph(CDisplay &display, const NLLIGO::IPrimitive &grp)
 		if (!grp.getChild(child, k)) continue;
 		std::string className;
 		if (!child->getPropertyByName("class", className)) continue;
-		if (NLMISC::toLower(className) != FAUNA_PLACE) continue;
+		if (NLMISC::toLowerAscii(className) != FAUNA_PLACE) continue;
 		std::string indexStr;
 		int index;
 		if (!child->getPropertyByName("index", indexStr)) continue;

ryzom/tools/leveldesign/world_editor/world_editor_sound_plugin/sound_plugin.cpp

@@ -169,7 +169,7 @@ void CSoundPlugin::ReInit()
 	bool	fmodDriver = false;
 	try
 	{
-		fmodDriver = toLower(cf.getVar("DriverSound").asString())=="fmod";
+		fmodDriver = toLowerAscii(cf.getVar("DriverSound").asString())=="fmod"; // FIXME: Support all drivers
 	}
 	catch(...)
 	{

ryzom/tools/make_anim_by_race/main.cpp

@@ -134,7 +134,7 @@ void	makeAnimByRace(const std::string &animSetFile, const std::vector<string> &a
 		// get the possible anim file name (lowered)
 		static vector<string>	raceAnimNames;
 		raceAnimNames.clear();
-		buildRaceAnimNames(raceAnimNames, toLower(CFile::getFilename(animList[i])));
+		buildRaceAnimNames(raceAnimNames, toLowerAscii(CFile::getFilename(animList[i])));
 
 		// For each line of the animSet
 		uint	lastStructLine= 0;
@@ -142,7 +142,7 @@ void	makeAnimByRace(const std::string &animSetFile, const std::vector<string> &a
 		for(uint j=0;j<animSetText.size();)
 		{
 			string	line= animSetText[j];
-			string	lineLwr= toLower(line);
+			string	lineLwr= toLowerAscii(line);
 
 			// Find <LOG> TAg? => stop
 			if(line.find("<LOG>")!=string::npos)

ryzom/tools/make_anim_melee_impact/main.cpp

@@ -87,7 +87,7 @@ void CAnimCombatState::build(const string &line)
 void	makeAnimMeleeImpact(const std::string &animSetFile, const set<CAnimCombatSet> &combatAnimSets)
 {
 	// look if this animSetFile is in the combat list to patch
-	string	shortName= NLMISC::toLower(CFile::getFilenameWithoutExtension(animSetFile));
+	string	shortName= NLMISC::toLowerAscii(CFile::getFilenameWithoutExtension(animSetFile));
 	CAnimCombatSet	key;
 	key.Name= shortName;
 	set<CAnimCombatSet>::const_iterator	it= combatAnimSets.find(key);
@@ -125,7 +125,7 @@ void	makeAnimMeleeImpact(const std::string &animSetFile, const set<CAnimCombatSe
 		for(uint j=0;j<animSetText.size();j++)
 		{
 			string	line= animSetText[j];
-			string	lineLwr= toLower(line);
+			string	lineLwr= toLowerAscii(line);
 
 			// Find <LOG> TAg? => stop
 			if(line.find("<LOG>")!=string::npos)
@@ -405,7 +405,7 @@ To generate the anim.txt file, this code has to be inserted in the client, in
 						if(anim && anim3d)
 						{
 							// name
-							string	name= NLMISC::toLower(_AnimationSet->getAnimationName(anim->id()));
+							string	name= NLMISC::toLowerAscii(_AnimationSet->getAnimationName(anim->id()));
 							if(animName.empty())
 								animName= name;
 							else if(!extended)

ryzom/tools/patch_gen/patch_gen_common.cpp

@@ -459,9 +459,9 @@ void CPackageDescription::buildDefaultFileList()
 	std::vector<std::string> fileList;
 	NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList);
 	for (uint32 i=0;i<fileList.size();++i)
-		if (NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="bnp"
+		if (NLMISC::toLowerAscii(NLMISC::CFile::getExtension(fileList[i]))=="bnp"
-			|| NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="snp")
+			|| NLMISC::toLowerAscii(NLMISC::CFile::getExtension(fileList[i]))=="snp")
-			_Categories.addFile("main",NLMISC::toLower(NLMISC::CFile::getFilename(fileList[i])));
+			_Categories.addFile("main",NLMISC::toLowerAscii(NLMISC::CFile::getFilename(fileList[i])));
 
 	_Categories.addFile("unpacked","root.bnp");
 }

ryzom/tools/pd_parser/parser.cpp

@@ -99,16 +99,16 @@ string	xmlSpecialChars(string str)
 
 string	getFullStdPathNoExt(const string &path)
 {
-	string	dir = NLMISC::toLower(NLMISC::CFile::getPath(path));
+	string	dir = NLMISC::toLowerAscii(NLMISC::CFile::getPath(path));
-	string	file = NLMISC::toLower(NLMISC::CFile::getFilenameWithoutExtension(path));
+	string	file = NLMISC::toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(path));
 
 	return dir.empty() ? file : NLMISC::CPath::standardizePath(dir)+file;
 }
 
 string	getFullStdPath(const string &path)
 {
-	string	dir = NLMISC::toLower(NLMISC::CFile::getPath(path));
+	string	dir = NLMISC::toLowerAscii(NLMISC::CFile::getPath(path));
-	string	file = NLMISC::toLower(NLMISC::CFile::getFilename(path));
+	string	file = NLMISC::toLowerAscii(NLMISC::CFile::getFilename(path));
 
 	return dir.empty() ? file : NLMISC::CPath::standardizePath(dir)+file;
 }
@@ -258,7 +258,7 @@ bool	CDbNode::epilog()
 	setEnv("db", Name);
 
 	string	fullfile = getFullStdPathNoExt(MainFile.empty() ? Name : MainFile);
-	string	filename = NLMISC::toLower(NLMISC::CFile::getFilenameWithoutExtension(fullfile));
+	string	filename = NLMISC::toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(fullfile));
 
 	setEnv("filename", filename);
 	setEnv("fullfilename", fullfile);
@@ -788,14 +788,14 @@ void	CDbNode::generateLogContent()
 // get file path from this file
 string	CDbNode::getFileNoExtPath(const std::string& file)
 {
-	string	thisPath = NLMISC::CFile::getPath(NLMISC::toLower(getDbFile()));
+	string	thisPath = NLMISC::CFile::getPath(NLMISC::toLowerAscii(getDbFile()));
-	string	filePath = NLMISC::CFile::getPath(NLMISC::toLower(file));
+	string	filePath = NLMISC::CFile::getPath(NLMISC::toLowerAscii(file));
-	string	fileName = NLMISC::CFile::getFilename(NLMISC::toLower(file));
+	string	fileName = NLMISC::CFile::getFilename(NLMISC::toLowerAscii(file));
 
 	if (thisPath == filePath)
 		return CFile::getFilenameWithoutExtension(fileName);
 	else
-		return CPath::standardizePath(filePath)+CFile::getFilenameWithoutExtension(NLMISC::toLower(file));
+		return CPath::standardizePath(filePath)+CFile::getFilenameWithoutExtension(NLMISC::toLowerAscii(file));
 }
 
 
@@ -825,7 +825,7 @@ bool	CFileNode::generateProlog()
 	if (!db->Description.empty())
 		Hpp << db->Description << "\n";
 
-	string	filename = NLMISC::toLower(CFile::getFilenameWithoutExtension(Name));
+	string	filename = NLMISC::toLowerAscii(CFile::getFilenameWithoutExtension(Name));
 
 	setEnv("fullfilename", getFullStdPathNoExt(Name));
 	setEnv("filename", filename);
@@ -870,7 +870,7 @@ bool	CFileNode::generateProlog()
 	if (SeparatedFlag)
 	{
 		string	fullfile = getFullStdPathNoExt(db->MainFile.empty() ? db->Name : db->MainFile);
-		string	filename = NLMISC::toLower(NLMISC::CFile::getFilenameWithoutExtension(fullfile));
+		string	filename = NLMISC::toLowerAscii(NLMISC::CFile::getFilenameWithoutExtension(fullfile));
 		Hpp << "#include \"" << filename << ".h\"\n";
 		Hpp << "\n";
 	}
@@ -920,7 +920,7 @@ bool	CFileNode::generateEpilog()
 	CDbNode*	db = getDbNode();
 
 	string	fullfile = getFullStdPathNoExt(Name);
-	string	filename = NLMISC::toLower(CFile::getFilenameWithoutExtension(Name));
+	string	filename = NLMISC::toLowerAscii(CFile::getFilenameWithoutExtension(Name));
 
 	Hpp.indent();
 	Hpp << "\n} // End of " << db->Name <<"\n";
@@ -948,14 +948,14 @@ bool	CFileNode::generateEpilog()
 
 string	CFileNode::getFileNoExtPath(const string& file)
 {
-	string	thisPath = NLMISC::CFile::getPath(NLMISC::toLower(Name));
+	string	thisPath = NLMISC::CFile::getPath(NLMISC::toLowerAscii(Name));
-	string	filePath = NLMISC::CFile::getPath(NLMISC::toLower(file));
+	string	filePath = NLMISC::CFile::getPath(NLMISC::toLowerAscii(file));
-	string	fileName = NLMISC::CFile::getFilename(NLMISC::toLower(file));
+	string	fileName = NLMISC::CFile::getFilename(NLMISC::toLowerAscii(file));
 
 	if (thisPath == filePath)
 		return CFile::getFilenameWithoutExtension(fileName);
 	else
-		return CFile::getFilenameWithoutExtension(NLMISC::toLower(file));
+		return CFile::getFilenameWithoutExtension(NLMISC::toLowerAscii(file));
 }
 
 void	CFileNode::writeFile()
@@ -1301,7 +1301,7 @@ bool	CEnumNode::generateContent()
 	gen.add("{");
 	gen.add("return Unknown;");
 	gen.add("}");
-	gen.add("const std::map<std::string, "+Name+">::const_iterator\tit = _ValueMap.find(NLMISC::toLower(v));");
+	gen.add("const std::map<std::string, "+Name+">::const_iterator\tit = _ValueMap.find(NLMISC::toLowerAscii(v));");
 	gen.add("if (it == _ValueMap.end())");
 	gen.add("{");
 	gen.add("nlwarning(\""+Name+"::toString(): string '%s' is not matched, 'Unknown' enum value returned\", v.c_str());");
@@ -1318,7 +1318,7 @@ bool	CEnumNode::generateContent()
 	gen.add("for (i=0; i<"+toString(Values.size())+"; ++i)");
 	gen.add("{");
 	gen.add("_StrTable["+Name+"Convert[i].Value] = "+Name+"Convert[i].Name;");
-	gen.add("_ValueMap[NLMISC::toLower(std::string("+Name+"Convert[i].Name))] = "+Name+"Convert[i].Value;");
+	gen.add("_ValueMap[NLMISC::toLowerAscii(std::string("+Name+"Convert[i].Name))] = "+Name+"Convert[i].Value;");
 	gen.add("}");
 
 	gen.add("_Initialised = true;");

ryzom/tools/server/build_world_packed_col/build_world_packed_col.cpp

@@ -180,7 +180,7 @@ int main(int argc, char* argv[])
 			if (!zonePath.empty())
 			{
 				presentZonePathes.push_back(zonePath);
-				presentZoneNames.push_back(toLower(zoneNames[l]));
+				presentZoneNames.push_back(toLowerAscii(zoneNames[l]));
 			}
 		}
 		//

ryzom/tools/server/build_world_packed_col/village.cpp

@@ -266,7 +266,7 @@ void CIGInfo::load(TShapeCache &shapeCache)
 		for(uint k = 0; k < IG->getNumInstance(); ++k)
 		{
 			std::string shapeName = standardizeShapeName(IG->getShapeName(k));			
-			if (NLMISC::toLower(CFile::getExtension(shapeName)) == "pacs_prim")
+			if (NLMISC::toLowerAscii(CFile::getExtension(shapeName)) == "pacs_prim")
 			{
 				continue;
 			}

ryzom/tools/skill_extractor/skill_extractor.cpp

@@ -311,10 +311,10 @@ sint main( sint argc, char ** argv )
 				}
 				break;
 			case 1: // code
-				skill.Code = toUpper(string( ptr ));
+				skill.Code = toUpperAscii(string( ptr ));
 				break;		
 			case 2: // parent skill
-				skill.ParentSkill = toUpper(string( ptr ));
+				skill.ParentSkill = toUpperAscii(string( ptr ));
 				break;
 			case 3: // max skill value
 				NLMISC::fromString(std::string(ptr), skill.MaxValue);

ryzom/tools/translation_tools/extract_new_sheet_names.cpp

@@ -58,7 +58,7 @@ struct CSheetWordListBuilder : public IWordListBuilder
 
 	virtual bool	buildWordList(std::vector<string> &allWords, string workSheetFileName)
 	{
-		SheetExt= toLower(SheetExt);
+		SheetExt= toLowerAscii(SheetExt);
 
 		// verify the directory is correct
 		if(!CFile::isDirectory(SheetPath))
@@ -78,7 +78,7 @@ struct CSheetWordListBuilder : public IWordListBuilder
 		for(uint i=0;i<allFiles.size();i++)
 		{
 			string	fileNameWithoutExt= CFile::getFilenameWithoutExtension(allFiles[i]);
-			string	extension= toLower(CFile::getExtension(allFiles[i]));
+			string	extension= toLowerAscii(CFile::getExtension(allFiles[i]));
 			// bad extension?
 			if(extension!=SheetExt)
 				continue;
@@ -86,7 +86,7 @@ struct CSheetWordListBuilder : public IWordListBuilder
 			if(fileNameWithoutExt.empty()||fileNameWithoutExt[0]=='_')
 				continue;
 			// ok, add
-			allWords.push_back(toLower(fileNameWithoutExt));
+			allWords.push_back(toLowerAscii(fileNameWithoutExt));
 		}
 
 		return true;
@@ -168,7 +168,7 @@ struct CRegionPrimWordListBuilder : public IWordListBuilder
 					string primName;
 					if(placeRes[placeId]->getPropertyByName(listProp[cid], primName) && !primName.empty())
 					{
-						primName= toLower(primName);
+						primName= toLowerAscii(primName);
 						// avoid duplicate
 						if(allWordSet.insert(primName).second)
 						{