(svn r4105) - Feature: Add proper ISO-8859-15 <> LOCALCODE conversion. As the mess that is makefile can't properly support it at the moment, it is only available for MACOSX. Windows doesn't need FS conversion and I have no idea about OS/2 so it's disabled for them.

- CodeChange: Change the function GetCurrentLocale(). It returns the locale from some default environment-variables, plus a custom one defined as parameter. If all fail, it tries $LANG.

9 files changed, 152 insertions, 83 deletions

diff --git a/functions.h b/functions.h
index 2f325d911..cf558f3ae 100644
--- a/functions.h
+++ b/functions.h
@@ -250,6 +250,7 @@ void GameSizeChanged(void);
 bool FileExists(const char *filename);
 bool ReadLanguagePack(int index);
 void InitializeLanguagePacks(void);
+const char *GetCurrentLocale(const char *param);
 void *ReadFileToMem(const char *filename, size_t *lenp, size_t maxsize);
 int GetLanguageList(char **languages, int max);
 
diff --git a/hal.h b/hal.h
index d2abd52ce..f92f5c974 100644
--- a/hal.h
+++ b/hal.h
@@ -94,4 +94,12 @@ int CDECL compare_FiosItems(const void *a, const void *b);
 
 void CreateConsole(void);
 
+#if defined(WIN32) || defined(WIN64) || defined(__WATCOMC__)
+# define FS2OTTD(name) name
+# define OTTD2FS(name) name
+#else
+const char *FS2OTTD(const char *name);
+const char *OTTD2FS(const char *name);
+#endif
+
 #endif /* HAL_H */
diff --git a/misc_gui.c b/misc_gui.c
index 724e9599e..68245528c 100644
--- a/misc_gui.c
+++ b/misc_gui.c
@@ -1357,7 +1357,7 @@ static void SaveLoadDlgWndProc(Window *w, WindowEvent *e)
 		break;
 	case WE_TIMEOUT:
 		if (HASBIT(w->click_state, 11)) { /* Delete button clicked */
-			if (!FiosDelete(WP(w,querystr_d).text.buf)) {
+			if (!FiosDelete(OTTD2FS(WP(w,querystr_d).text.buf))) {
 				ShowErrorMessage(INVALID_STRING_ID, STR_4008_UNABLE_TO_DELETE_FILE, 0, 0);
 			} else {
 				BuildFileList();
diff --git a/os2.c b/os2.c
index 3b6c48e2c..aa3aee58c 100644
--- a/os2.c
+++ b/os2.c
@@ -108,7 +108,7 @@ FiosItem *FiosGetSavegameList(int *num, int mode)
 				fios->type = FIOS_TYPE_DIR;
 				fios->mtime = 0;
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
-				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", dirent->d_name);
+				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", FS2OTTD(dirent->d_name));
 				str_validate(fios->title);
 			}
 		}
@@ -150,7 +150,7 @@ FiosItem *FiosGetSavegameList(int *num, int mode)
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
 
 				*t = '\0'; // strip extension
-				ttd_strlcpy(fios->title, dirent->d_name, lengthof(fios->title));
+				ttd_strlcpy(fios->title, FS2OTTD(dirent->d_name), lengthof(fios->title));
 				str_validate(fios->title);
 			} else if (mode == SLD_LOAD_GAME || mode == SLD_LOAD_SCENARIO) {
 				if (strcasecmp(t, ".ss1") == 0 ||
@@ -237,7 +237,7 @@ FiosItem *FiosGetScenarioList(int *num, int mode)
 				fios->type = FIOS_TYPE_DIR;
 				fios->mtime = 0;
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
-				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", dirent->d_name);
+				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", FS2OTTD(dirent->d_name));
 				str_validate(fios->title);
 			}
 		}
@@ -278,7 +278,7 @@ FiosItem *FiosGetScenarioList(int *num, int mode)
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
 
 				*t = '\0'; // strip extension
-				ttd_strlcpy(fios->title, dirent->d_name, lengthof(fios->title));
+				ttd_strlcpy(fios->title, FS2OTTD(dirent->d_name), lengthof(fios->title));
 				str_validate(fios->title);
 			} else if (mode == SLD_LOAD_GAME || mode == SLD_LOAD_SCENARIO ||
 					mode == SLD_NEW_GAME) {
diff --git a/saveload.c b/saveload.c
index 653ff2402..3b304f445 100644
--- a/saveload.c
+++ b/saveload.c
@@ -19,6 +19,7 @@
 #include "openttd.h"
 #include "debug.h"
 #include "functions.h"
+#include "hal.h"
 #include "vehicle.h"
 #include "station.h"
 #include "thread.h"
@@ -1314,12 +1315,6 @@ extern bool AfterLoadGame(void);
 extern void BeforeSaveGame(void);
 extern bool LoadOldSaveGame(const char *file);
 
-#if defined(__APPLE__) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3)
-extern const char *convert_to_fs_charset(const char *filename);
-#else
-#define convert_to_fs_charset(str) (str)
-#endif
-
 /** Small helper function to close the to be loaded savegame an signal error */
 static inline SaveOrLoadResult AbortSaveLoad(void)
 {
@@ -1454,11 +1449,7 @@ SaveOrLoadResult SaveOrLoad(const char *filename, int mode)
 		return SL_OK;
 	}
 
-	if(mode == SL_SAVE) {
-		_sl.fh = fopen(convert_to_fs_charset(filename), "wb");
-	} else {
-		_sl.fh = fopen(filename, "rb");
-	}
+	_sl.fh = (mode == SL_SAVE) ? fopen(OTTD2FS(filename), "wb") : fopen(filename, "rb");
 	if (_sl.fh == NULL) {
 		DEBUG(misc, 0) ("[Sl] Cannot open savegame for saving/loading.");
 		return SL_ERROR;
diff --git a/screenshot.c b/screenshot.c
index 963c5cff3..b384a591e 100644
--- a/screenshot.c
+++ b/screenshot.c
@@ -7,17 +7,12 @@
 #include "strings.h"
 #include "table/strings.h"
 #include "gfx.h"
+#include "hal.h"
 #include "viewport.h"
 #include "player.h"
 #include "screenshot.h"
 #include "variables.h"
 
-#if defined(__APPLE__) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3)
-extern const char *convert_to_fs_charset(const char *filename);
-#else
-#define convert_to_fs_charset(str) (str)
-#endif
-
 char _screenshot_format_name[8];
 uint _num_screenshot_formats;
 uint _cur_screenshot_format;
@@ -79,7 +74,7 @@ static bool MakeBmpImage(const char *name, ScreenshotCallback *callb, void *user
 	if (pixelformat != 8)
 		return false;
 
-	f = fopen(convert_to_fs_charset(name), "wb");
+	f = fopen(OTTD2FS(name), "wb");
 	if (f == NULL) return false;
 
 	// each scanline must be aligned on a 32bit boundary
@@ -183,7 +178,7 @@ static bool MakePNGImage(const char *name, ScreenshotCallback *callb, void *user
 	if (pixelformat != 8)
 		return false;
 
-	f = fopen(convert_to_fs_charset(name), "wb");
+	f = fopen(OTTD2FS(name), "wb");
 	if (f == NULL) return false;
 
 	png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, (char *)name, png_my_error, png_my_warning);
@@ -294,7 +289,7 @@ static bool MakePCXImage(const char *name, ScreenshotCallback *callb, void *user
 	if (pixelformat != 8 || w == 0)
 		return false;
 
-	f = fopen(convert_to_fs_charset(name), "wb");
+	f = fopen(OTTD2FS(name), "wb");
 	if (f == NULL) return false;
 
 	memset(&pcx, 0, sizeof(pcx));
diff --git a/strings.c b/strings.c
index 9943966ae..f57de374e 100644
--- a/strings.c
+++ b/strings.c
@@ -1051,6 +1051,30 @@ bool ReadLanguagePack(int lang_index)
 	return true;
 }
 
+/** Determine the current charset based on the environment
+ * First check some default values, after this one we passed ourselves
+ * and if none exist return the value for $LANG
+ * @param environment variable to check conditionally if default ones are not
+ *        set. Pass NULL if you don't want additional checks.
+ * @return return string containing current charset, or NULL if not-determinable */
+const char *GetCurrentLocale(const char *param)
+{
+	const char *env;
+
+	env = getenv("LANGUAGE");
+	if (env != NULL) return env;
+
+	env = getenv("LC_ALL");
+	if (env != NULL) return env;
+
+	if (param != NULL) {
+		env = getenv(param);
+		if (env != NULL) return env;
+	}
+
+	return getenv("LANG");
+}
+
 // make a list of the available language packs. put the data in _dynlang struct.
 void InitializeLanguagePacks(void)
 {
@@ -1063,24 +1087,11 @@ void InitializeLanguagePacks(void)
 	LanguagePack hdr;
 	FILE *in;
 	char *files[32];
-	uint j;
 
 	char lang[] = "en";
-	static const char* env[] = {
-		"LANGUAGE",
-		"LC_ALL",
-		"LC_MESSAGES",
-		"LANG"
-	};
-
-	for (j = 0; j < lengthof(env); j++) {
-		const char* envlang = getenv(env[j]);
-		if (envlang != NULL) {
-			snprintf(lang, lengthof(lang), "%.2s", envlang);
-			break;
-		}
-	}
+	const char *env = GetCurrentLocale("LC_MESSAGES");
 
+	if (env != NULL) snprintf(lang, lengthof(lang), "%.2s", env);
 	n = GetLanguageList(files, lengthof(files));
 
 	def = -1;
diff --git a/unix.c b/unix.c
index 29ddec090..ffb65cc77 100644
--- a/unix.c
+++ b/unix.c
@@ -8,12 +8,15 @@
 #include "table/strings.h"
 #include "hal.h"
 #include "variables.h"
+#include "debug.h"
 
 #include <dirent.h>
 #include <unistd.h>
 #include <sys/stat.h>
 #include <time.h>
 #include <signal.h>
+#include <iconv.h>
+#include <errno.h>
 
 #ifdef USE_HOMEDIR
 #include <pwd.h>
@@ -118,8 +121,7 @@ FiosItem *FiosGetSavegameList(int *num, int mode)
 				fios->type = FIOS_TYPE_DIR;
 				fios->mtime = 0;
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
-				snprintf(fios->title, lengthof(fios->title),
-					"%s/ (Directory)", dirent->d_name);
+				snprintf(fios->title, lengthof(fios->title), "%s/ (Directory)", FS2OTTD(dirent->d_name));
 				str_validate(fios->title);
 			}
 		}
@@ -162,7 +164,7 @@ FiosItem *FiosGetSavegameList(int *num, int mode)
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
 
 				*t = '\0'; // strip extension
-				ttd_strlcpy(fios->title, dirent->d_name, lengthof(fios->title));
+				ttd_strlcpy(fios->title, FS2OTTD(dirent->d_name), lengthof(fios->title));
 				str_validate(fios->title);
 			} else if (mode == SLD_LOAD_GAME || mode == SLD_LOAD_SCENARIO) {
 				if (strcasecmp(t, ".ss1") == 0 ||
@@ -222,7 +224,7 @@ FiosItem *FiosGetScenarioList(int *num, int mode)
 				fios->type = FIOS_TYPE_DIR;
 				fios->mtime = 0;
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
-				snprintf(fios->title, lengthof(fios->title), "%s/ (Directory)", dirent->d_name);
+				snprintf(fios->title, lengthof(fios->title), "%s/ (Directory)", FS2OTTD(dirent->d_name));
 				str_validate(fios->title);
 			}
 		}
@@ -263,7 +265,7 @@ FiosItem *FiosGetScenarioList(int *num, int mode)
 				ttd_strlcpy(fios->name, dirent->d_name, lengthof(fios->name));
 
 				*t = '\0'; // strip extension
-				ttd_strlcpy(fios->title, dirent->d_name, lengthof(fios->title));
+				ttd_strlcpy(fios->title, FS2OTTD(dirent->d_name), lengthof(fios->title));
 				str_validate(fios->title);
 			} else if (mode == SLD_LOAD_GAME || mode == SLD_LOAD_SCENARIO ||
 					mode == SLD_NEW_GAME) {
@@ -604,42 +606,103 @@ void CSleep(int milliseconds)
 	#endif // __AMIGA__
 }
 
+// No proper makefile detection, so just force this for the time being
 #if defined(__APPLE__) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3)
-/* FYI: This is not thread-safe.
-Assumptions:
-	- the 'from' charset is ISO-8859-15
-	- the 'to' charset is either the same, or UTF-8
-NOTE: iconv was added in OSX 10.3. 10.2.x will still have the invalid char issues. There aren't any easy fix for this
-*/
-#include <iconv.h>
-#include <locale.h>
-const char *convert_to_fs_charset(const char *filename)
+# define WITH_ICONV
+#endif
+
+#ifdef WITH_ICONV
+
+#define INTERNALCODE "ISO-8859-15"
+
+/** Try and try to decipher the current locale from environmental
+ * variables. MacOSX is hardcoded, other OS's are dynamic. If no suitable
+ * locale can be found, don't do any conversion "" */
+static const char *GetLocalCode(void)
 {
-	static char statout[1024], statin[1024];
-	static iconv_t convd;
-	static bool alreadyInited;
-	char *outbuf = statout;
-	const char *inbuf = statin;
-	size_t inlen = strlen(filename), outlen = 1023;
-	size_t retval = 0;
-	if(inbuf == NULL)
-		inbuf = statin;
-
-	setlocale(LC_ALL, "C-UTF-8");
-	strcpy(statout, filename);
-	strcpy(statin, filename);
-	inbuf = strrchr(statin, '/');
-	outbuf = strrchr(statout, '/');
-	if(alreadyInited == false)
-	{
-		convd = iconv_open("UTF-8-MAC", "ISO-8859-15");
-		if(convd == (iconv_t)(-1))
-			return filename;
-		alreadyInited = true;
+#if defined(__APPLE__) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_3)
+	return "UTF-8-MAC";
+#else
+	/* Strip locale (eg en_US.UTF-8) to only have UTF-8 */
+	const char *locale = GetCurrentLocale("LC_CTYPE");
+	if (locale != NULL) locale = strchr(locale, '.');
+
+	return (locale == NULL) ? "" : locale + 1;
+#endif
+}
+
+/** FYI: This is not thread-safe.
+ * convert between locales, which from and which to is set in the calling
+ * functions OTTD2FS() and FS2OTTD(). You should NOT use this function directly
+ * NOTE: iconv was added in OSX 10.3. 10.2.x will still have the invalid char
+ * issues. There aren't any easy fix for this */
+static const char *convert_tofrom_fs(iconv_t convd, const char *name)
+{
+	static char buf[1024];
+	/* Work around buggy iconv implementation where inbuf is wrongly typed as
+	 * non-const. Correct implementation is at
+	 * http://www.opengroup.org/onlinepubs/007908799/xsh/iconv.html */
+#if defined (__GLIBC__) || defined (__GNU_LIBRARY__)
+	char *inbuf = (char*)name;
+#else
+	const char *inbuf = name;
+#endif
+
+	char *outbuf  = buf;
+	size_t outlen = sizeof(buf) - 1;
+	size_t inlen  = strlen(name);
+
+	ttd_strlcpy(outbuf, name, sizeof(buf));
+
+	iconv(convd, NULL, NULL, NULL, NULL);
+	if (iconv(convd, &inbuf, &inlen, &outbuf, &outlen) == (size_t)(-1)) {
+		DEBUG(misc, 0) ("[Iconv] Error converting '%s'. Errno %d", name, errno);
 	}
-	retval = iconv(convd, NULL, NULL, NULL, NULL);
-	inlen = iconv(convd, &inbuf, &inlen, &outbuf, &outlen);
+
+	*outbuf = '\0';
 	// FIX: invalid characters will abort conversion, but they shouldn't occur?
-	return statout;
+	return buf;
 }
-#endif
+
+/** Convert from OpenTTD's encoding to that of the local environment
+ * @param name pointer to a valid string that will be converted
+ * @return pointer to a new stringbuffer that contains the converted string */
+const char *OTTD2FS(const char *name)
+{
+	static iconv_t convd = (iconv_t)(-1);
+
+	if (convd == (iconv_t)(-1)) {
+		const char *env = GetLocalCode();
+		convd = iconv_open(env, INTERNALCODE);
+		if (convd == (iconv_t)(-1)) {
+			DEBUG(misc, 0) ("[iconv] Cannot convert from codeset '%s' to '%s'", INTERNALCODE, env);
+			return name;
+		}
+	}
+
+	return convert_tofrom_fs(convd, name);
+}
+
+/** Convert to OpenTTD's encoding from that of the local environment
+ * @param name pointer to a valid string that will be converted
+ * @return pointer to a new stringbuffer that contains the converted string */
+const char *FS2OTTD(const char *name)
+{
+	static iconv_t convd = (iconv_t)(-1);
+
+	if (convd == (iconv_t)(-1)) {
+		const char *env = GetLocalCode();
+		convd = iconv_open(INTERNALCODE, env);
+		if (convd == (iconv_t)(-1)) {
+			DEBUG(misc, 0) ("[iconv] Cannot convert from codeset '%s' to '%s'", INTERNALCODE, env);
+			return name;
+		}
+	}
+
+	return convert_tofrom_fs(convd, name);
+}
+
+#else
+const char *FS2OTTD(const char *name) {return name;}
+const char *OTTD2FS(const char *name) {return name;}
+#endif /* WITH_ICONV */
diff --git a/win32.c b/win32.c
index 096702cf0..0324e2bb0 100644
--- a/win32.c
+++ b/win32.c
@@ -686,7 +686,7 @@ FiosItem *FiosGetSavegameList(int *num, int mode)
 				fios->type = FIOS_TYPE_DIR;
 				fios->mtime = 0;
 				ttd_strlcpy(fios->name, fd.cFileName, lengthof(fios->name));
-				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", fd.cFileName);
+				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", FS2OTTD(fd.cFileName));
 				str_validate(fios->title);
 			}
 		} while (FindNextFile(h, &fd));
@@ -727,7 +727,7 @@ FiosItem *FiosGetSavegameList(int *num, int mode)
 				ttd_strlcpy(fios->name, fd.cFileName, lengthof(fios->name));
 
 				*t = '\0'; // strip extension
-				ttd_strlcpy(fios->title, fd.cFileName, lengthof(fios->title));
+				ttd_strlcpy(fios->title, FS2OTTD(fd.cFileName), lengthof(fios->title));
 				str_validate(fios->title);
 			} else if (mode == SLD_LOAD_GAME || mode == SLD_LOAD_SCENARIO) {
 				if (strcasecmp(t, ".ss1") == 0 ||
@@ -802,7 +802,7 @@ FiosItem *FiosGetScenarioList(int *num, int mode)
 				fios->type = FIOS_TYPE_DIR;
 				fios->mtime = 0;
 				ttd_strlcpy(fios->name, fd.cFileName, lengthof(fios->name));
-				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", fd.cFileName);
+				snprintf(fios->title, lengthof(fios->title), "%s\\ (Directory)", FS2OTTD(fd.cFileName));
 				str_validate(fios->title);
 			}
 		} while (FindNextFile(h, &fd));
@@ -842,7 +842,7 @@ FiosItem *FiosGetScenarioList(int *num, int mode)
 				ttd_strlcpy(fios->name, fd.cFileName, lengthof(fios->name));
 
 				*t = '\0'; // strip extension
-				ttd_strlcpy(fios->title, fd.cFileName, lengthof(fios->title));
+				ttd_strlcpy(fios->title, FS2OTTD(fd.cFileName), lengthof(fios->title));
 				str_validate(fios->title);
 			} else if (mode == SLD_LOAD_GAME || mode == SLD_LOAD_SCENARIO ||
 					mode == SLD_NEW_GAME) {