(svn r11981) -Fix [FS#1698]: Use unicode glyph mapping to fix up missing/shuffled sprites in original data files instead of shuffling or skipping

sprites directly. Some required glyphs were not loaded.
-Fix: Large capital U with grave (Ù) along with some other glyphs are broken in the original data files, so do no display them.

6 files changed, 73 insertions, 109 deletions

diff --git a/src/fontcache.cpp b/src/fontcache.cpp
index 7a5813c75..14c735c02 100644
--- a/src/fontcache.cpp
+++ b/src/fontcache.cpp
@@ -518,34 +518,38 @@ void SetUnicodeGlyph(FontSize size, uint32 key, SpriteID sprite)
 
 void InitializeUnicodeGlyphMap()
 {
-	FontSize size;
-	SpriteID base;
-	SpriteID sprite;
-	uint i;
-
-	for (size = FS_NORMAL; size != FS_END; size++) {
+	for (FontSize size = FS_NORMAL; size != FS_END; size++) {
 		/* Clear out existing glyph map if it exists */
 		if (_unicode_glyph_map[size] != NULL) {
-			for (i = 0; i < 256; i++) {
+			for (uint i = 0; i < 256; i++) {
 				if (_unicode_glyph_map[size][i] != NULL) free(_unicode_glyph_map[size][i]);
 			}
 			free(_unicode_glyph_map[size]);
 			_unicode_glyph_map[size] = NULL;
 		}
 
-		base = GetFontBase(size);
-		for (i = ASCII_LETTERSTART; i < 256; i++) {
-			sprite = base + i - ASCII_LETTERSTART;
+		SpriteID base = GetFontBase(size);
+
+		for (uint i = ASCII_LETTERSTART; i < 256; i++) {
+			SpriteID sprite = base + i - ASCII_LETTERSTART;
 			if (!SpriteExists(sprite)) continue;
 			SetUnicodeGlyph(size, i, sprite);
 			SetUnicodeGlyph(size, i + SCC_SPRITE_START, sprite);
 		}
-		for (i = 0; i < lengthof(_default_unicode_map); i++) {
-			sprite = base + _default_unicode_map[i].key - ASCII_LETTERSTART;
-			SetUnicodeGlyph(size, _default_unicode_map[i].code, sprite);
+
+		for (uint i = 0; i < lengthof(_default_unicode_map); i++) {
+			byte key = _default_unicode_map[i].key;
+			if (key == CLRA || key == CLRL) {
+				/* Clear the glyph. This happens if the glyph at this code point
+				 * is non-standard and should be accessed by an SCC_xxx enum
+				 * entry only. */
+				if (key == CLRA || size == FS_LARGE) {
+					SetUnicodeGlyph(size, _default_unicode_map[i].code, 0);
+				}
+			} else {
+				SpriteID sprite = base + key - ASCII_LETTERSTART;
+				SetUnicodeGlyph(size, _default_unicode_map[i].code, sprite);
+			}
 		}
 	}
 }
-
-
-
diff --git a/src/gfxinit.cpp b/src/gfxinit.cpp
index 67b126002..4d67a2544 100644
--- a/src/gfxinit.cpp
+++ b/src/gfxinit.cpp
@@ -199,95 +199,12 @@ void CheckExternalFiles()
 }
 
 
-static const SpriteID trg1idx[] = {
-	   0,    1, ///< Mouse cursor, ZZZ
-/* Medium font */
-	   2,   92, ///< ' ' till 'z'
-	SKIP,   36,
-	 160,  160, ///< Move Ÿ to the correct position
-	  98,   98, ///< Up arrow
-	 131,  133,
-	SKIP,    1, ///< skip currency sign
-	 135,  135,
-	SKIP,    1,
-	 137,  137,
-	SKIP,    1,
-	 139,  139,
-	 140,  140, ///< @todo Down arrow
-	 141,  141,
-	 142,  142, ///< @todo Check mark
-	 143,  143, ///< @todo Cross
-	 144,  144,
-	 145,  145, ///< @todo Right arrow
-	 146,  149,
-	 118,  122, ///< Transport markers
-	SKIP,    2,
-	 157,  157,
-	 114,  115, ///< Small up/down arrows
-	SKIP,    1,
-	 161,  225,
-/* Small font */
-	 226,  316, ///< ' ' till 'z'
-	SKIP,   36,
-	 384,  384, ///< Move Ÿ to the correct position
-	 322,  322, ///< Up arrow
-	 355,  357,
-	SKIP,    1, ///< skip currency sign
-	 359,  359,
-	SKIP,    1,
-	 361,  361,
-	SKIP,    1,
-	 363,  363,
-	 364,  364, ////< @todo Down arrow
-	 365,  366,
-	SKIP,    1,
-	 368,  368,
-	 369,  369, ///< @todo Right arrow
-	 370,  373,
-	SKIP,    7,
-	 381,  381,
-	SKIP,    3,
-	 385,  449,
-/* Big font */
-	 450,  540, ///< ' ' till 'z'
-	SKIP,   36,
-	 608,  608, ///< Move Ÿ to the correct position
-	SKIP,    1,
-	 579,  581,
-	SKIP,    1,
-	 583,  583,
-	SKIP,    5,
-	 589,  589,
-	SKIP,   15,
-	 605,  605,
-	SKIP,    3,
-	 609,  625,
-	SKIP,    1,
-	 627,  632,
-	SKIP,    1,
-	 634,  639,
-	SKIP,    1,
-	 641,  657,
-	SKIP,    1,
-	 659,  664,
-	SKIP,    2,
-	 667,  671,
-	SKIP,    1,
-	 673,  673,
-/* Graphics */
-	 674, 4792,
-	END
-};
-
 static void LoadSpriteTables()
 {
 	const FileList *files = _use_dos_palette ? &files_dos : &files_win;
 	uint i = FIRST_GRF_SLOT;
 
-	LoadGrfIndexed(files->basic[0].filename, trg1idx, i++);
-	DupSprite(  2, 130); // non-breaking space medium
-	DupSprite(226, 354); // non-breaking space tiny
-	DupSprite(450, 578); // non-breaking space large
+	LoadGrfFile(files->basic[0].filename, 0, i++);
 
 	/*
 	 * The second basic file always starts at the given location and does
diff --git a/src/newgrf_text.cpp b/src/newgrf_text.cpp
index c98c7e48b..870ad5021 100644
--- a/src/newgrf_text.cpp
+++ b/src/newgrf_text.cpp
@@ -293,6 +293,9 @@ char *TranslateTTDPatchCodes(const char *str)
 			case 0xB6: d += Utf8Encode(d, SCC_BUS); break;
 			case 0xB7: d += Utf8Encode(d, SCC_PLANE); break;
 			case 0xB8: d += Utf8Encode(d, SCC_SHIP); break;
+			case 0xB9: d += Utf8Encode(d, SCC_SUPERSCRIPT_M1); break;
+			case 0xBC: d += Utf8Encode(d, SCC_SMALLUPARROW); break;
+			case 0xBD: d += Utf8Encode(d, SCC_SMALLDOWNARROW); break;
 			default:
 				/* Validate any unhandled character */
 				if (!IsValidChar(c, CS_ALPHANUMERAL)) c = '?';
diff --git a/src/table/control_codes.h b/src/table/control_codes.h
index 5c1ae73e7..1af7befd7 100644
--- a/src/table/control_codes.h
+++ b/src/table/control_codes.h
@@ -116,18 +116,19 @@ enum StringControlCode {
 	 * These are mapped to the original glyphs */
 	SCC_LESSTHAN       = SCC_SPRITE_START + 0x3C,
 	SCC_GREATERTHAN    = SCC_SPRITE_START + 0x3E,
-	SCC_UPARROW        = SCC_SPRITE_START + 0x80,
-	SCC_SMALLUPARROW   = SCC_SPRITE_START + 0x90,
-	SCC_SMALLDOWNARROW = SCC_SPRITE_START + 0x91,
-	SCC_TRAIN          = SCC_SPRITE_START + 0x94,
-	SCC_LORRY          = SCC_SPRITE_START + 0x95,
-	SCC_BUS            = SCC_SPRITE_START + 0x96,
-	SCC_PLANE          = SCC_SPRITE_START + 0x97,
-	SCC_SHIP           = SCC_SPRITE_START + 0x98,
+	SCC_UPARROW        = SCC_SPRITE_START + 0xA0,
 	SCC_DOWNARROW      = SCC_SPRITE_START + 0xAA,
 	SCC_CHECKMARK      = SCC_SPRITE_START + 0xAC,
 	SCC_CROSS          = SCC_SPRITE_START + 0xAD,
 	SCC_RIGHTARROW     = SCC_SPRITE_START + 0xAF,
+	SCC_TRAIN          = SCC_SPRITE_START + 0xB4,
+	SCC_LORRY          = SCC_SPRITE_START + 0xB5,
+	SCC_BUS            = SCC_SPRITE_START + 0xB6,
+	SCC_PLANE          = SCC_SPRITE_START + 0xB7,
+	SCC_SHIP           = SCC_SPRITE_START + 0xB8,
+	SCC_SUPERSCRIPT_M1 = SCC_SPRITE_START + 0xB9,
+	SCC_SMALLUPARROW   = SCC_SPRITE_START + 0xBC,
+	SCC_SMALLDOWNARROW = SCC_SPRITE_START + 0xBD,
 };
 
 #endif /* CONTROL_CODES_H */
diff --git a/src/table/unicode.h b/src/table/unicode.h
index 01831fa2b..d6ed30f7b 100644
--- a/src/table/unicode.h
+++ b/src/table/unicode.h
@@ -6,6 +6,10 @@ struct DefaultUnicodeMapping {
 	byte key;   ///< Character index of sprite
 };
 
+enum {
+	CLRA = 0, ///< Identifier to clear all glyphs at this codepoint
+	CLRL = 1, ///< Identifier to clear glyphs for large font at this codepoint
+};
 
 /* Default unicode mapping table for sprite based glyphs.
  * This table allows us use unicode characters even though the glyphs don't
@@ -13,5 +17,40 @@ struct DefaultUnicodeMapping {
  * This is not used for FreeType rendering */
 
 static DefaultUnicodeMapping _default_unicode_map[] = {
+	{ 0x00A0, 0x20 }, /* Non-breaking space / Up arrow */
+	{ 0x00A4, CLRL }, /* Currency sign */
+	{ 0x00A6, CLRL }, /* Broken bar */
+	{ 0x00A7, CLRL }, /* Section sign */
+	{ 0x00A8, CLRL }, /* Diaeresis */
+	{ 0x00A9, CLRL }, /* Copyright sign */
+	{ 0x00AA, CLRA }, /* Feminine ordinal indicator / Down arrow */
+	{ 0x00AC, CLRA }, /* Not sign / Tick mark */
+	{ 0x00AD, 0x20 }, /* Soft hyphen / X mark */
+	{ 0x00AF, CLRA }, /* Macron / Right arrow */
+	{ 0x00B0, CLRL }, /* Degree sign */
+	{ 0x00B1, CLRL }, /* Plus-Minus sign */
+	{ 0x00B2, CLRL }, /* Superscript 2 */
+	{ 0x00B3, CLRL }, /* Superscript 3 */
+	{ 0x00B4, CLRA }, /* Acute accent / Train symbol */
+	{ 0x00B5, CLRA }, /* Micro sign / Truck symbol */
+	{ 0x00B6, CLRA }, /* Pilcrow sign / Bus symbol */
+	{ 0x00B7, CLRA }, /* Middle dot / Aircraft symbol */
+	{ 0x00B8, CLRA }, /* Cedilla / Ship symbol */
+	{ 0x00B9, CLRA }, /* Superscript 1 / Superscript -1 */
+	{ 0x00BA, CLRL }, /* Masculine ordinal indicator */
+	{ 0x00BC, CLRA }, /* One quarter / Small up arrow */
+	{ 0x00BD, CLRA }, /* One half / Small down arrow */
+	{ 0x00BE, CLRL }, /* Three quarters */
+	{ 0x00D0, CLRL }, /* Capital letter eth */
+	{ 0x00D7, CLRL }, /* Multiplication sign */
+	{ 0x00D8, CLRL }, /* Capital letter O with stroke */
+	{ 0x00D9, CLRL }, /* Capital letter U with grave */
+	{ 0x00DE, CLRL }, /* Capital letter thorn */
+	{ 0x00E6, CLRL }, /* Small letter ae */
+	{ 0x00F0, CLRL }, /* Small letter eth */
+	{ 0x00F7, CLRL }, /* Divison sign */
+	{ 0x00F8, CLRL }, /* Small letter o with stroke */
+	{ 0x00FE, CLRL }, /* Small letter thorn */
+	{ 0x0178, 0x9F }, /* Capital letter Y with diaeresis */
 	{ 0x010D, 0x63 }, /* Small letter c with caron */
 };
diff --git a/src/widget.cpp b/src/widget.cpp
index d13352f77..3bd140ea1 100644
--- a/src/widget.cpp
+++ b/src/widget.cpp
@@ -14,7 +14,7 @@
 #include "table/sprites.h"
 #include "table/strings.h"
 
-static const char *UPARROW   = "\xEE\x8A\x80";
+static const char *UPARROW   = "\xEE\x8A\xA0";
 static const char *DOWNARROW = "\xEE\x8A\xAA";
 
 static Point HandleScrollbarHittest(const Scrollbar *sb, int top, int bottom)