From 59519edd344b3e9fa7d8f9f768563e8533d3760b Mon Sep 17 00:00:00 2001 From: rubidium Date: Wed, 27 Oct 2010 16:42:20 +0000 Subject: (svn r21044) -Feature: XZ/LZMA2 savegame support. New default reduces savegame size by 10 to 30% with slightly more CPU usage. With maximum settings it reduces savegame size by 20 to 30%, but that takes 7 to 14 times longer. Map saving + downloading takes, on average, 5% less. --- config.lib | 83 ++++++++++++++++++++++++++- os/debian/control | 2 +- os/debian/rules | 2 +- projects/openttd_vs100.vcxproj | 16 +++--- projects/openttd_vs100.vcxproj.in | 16 +++--- projects/openttd_vs80.vcproj | 16 +++--- projects/openttd_vs80.vcproj.in | 16 +++--- projects/openttd_vs90.vcproj | 16 +++--- projects/openttd_vs90.vcproj.in | 16 +++--- readme.txt | 6 +- src/crashlog.cpp | 7 +++ src/saveload/saveload.cpp | 116 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 257 insertions(+), 55 deletions(-) diff --git a/config.lib b/config.lib index 3fe61befb..d9dab6cc7 100644 --- a/config.lib +++ b/config.lib @@ -68,6 +68,7 @@ set_default() { with_sdl="1" with_cocoa="1" with_zlib="1" + with_lzma="1" with_lzo2="1" with_png="1" enable_builtin_depend="1" @@ -139,6 +140,7 @@ set_default() { with_sdl with_cocoa with_zlib + with_lzma with_lzo2 with_png enable_builtin_depend @@ -327,6 +329,13 @@ detect_params() { --without-zlib) with_zlib="0";; --with-zlib=*) with_zlib="$optarg";; + --with-lzma) with_lzma="2";; + --without-lzma) with_lzma="0";; + --with-lzma=*) with_lzma="$optarg";; + --with-liblzma) with_lzma="2";; + --without-liblzma) with_lzma="0";; + --with-liblzma=*) with_lzma="$optarg";; + --with-lzo2) with_lzo2="2";; --without-lzo2) with_lzo2="0";; --with-lzo2=*) with_lzo2="$optarg";; @@ -746,9 +755,9 @@ check_params() { if [ "$with_zlib" = "0" ] || [ -z "$zlib" ]; then log 1 "WARNING: zlib was not detected or disabled" - log 1 "WARNING: OpenTTD doesn't require zlib, but it does mean many features (like" - log 1 "WARNING: loading most savegames/scenarios, joining most servers, loading" - log 1 "WARNING: heightmaps, using PNG or using fonts, ...) will be disabled." + log 1 "WARNING: OpenTTD doesn't require zlib, but it does mean that many features" + log 1 "WARNING: (like loading most old savegames/scenarios, loading heightmaps," + log 1 "WARNING: using PNG, or using fonts, ...) will be disabled." if [ "$pre_detect_with_zlib" = "0" ]; then log 1 "WARNING: We strongly suggest you to install zlib." else @@ -758,6 +767,23 @@ check_params() { fi fi + pre_detect_with_lzma=$with_lzma + detect_lzma + + if [ "$with_lzma" = "0" ] || [ -z "$lzma_config" ]; then + log 1 "WARNING: lzma was not detected or disabled" + log 1 "WARNING: OpenTTD doesn't require lzma, but it does mean that many features" + log 1 "WARNING: (like loading most savegames/scenarios and joining most servers)" + log 1 "WARNING: will be disabled." + if [ "$pre_detect_with_lzma" = "0" ]; then + log 1 "WARNING: We strongly suggest you to install liblzma." + log 1 "configure: error: no liblzma detected" + else + log 1 " If you want to compile without lzma use --without-lzma as parameter" + exit + fi + fi + pre_detect_with_lzo2=$with_lzo2 detect_lzo2 @@ -1437,6 +1463,18 @@ make_cflags_and_ldflags() { CFLAGS="$CFLAGS -DWITH_ZLIB" fi + if [ -n "$lzma_config" ]; then + CFLAGS="$CFLAGS -DWITH_LZMA" + CFLAGS="$CFLAGS `$lzma_config --cflags | tr '\n\r' ' '`" + + if [ "$enable_static" != "0" ]; then + CFLAGS="$CFLAGS -DLZMA_API_STATIC" + LIBS="$LIBS `$lzma_config --libs --static | tr '\n\r' ' '`" + else + LIBS="$LIBS `$lzma_config --libs | tr '\n\r' ' '`" + fi + fi + if [ "$with_lzo2" != "0" ]; then if [ "$enable_static" != "0" ] && [ "$os" != "OSX" ]; then LIBS="$LIBS $lzo2" @@ -2437,6 +2475,44 @@ detect_libtimidity() { detect_library "$with_libtimidity" "libtimidity" "libtimidity.a" "" "timidity.h" } +detect_lzma() { + # 0 means no, 1 is auto-detect, 2 is force + if [ "$with_lzma" = "0" ]; then + log 1 "checking liblzma... disabled" + + lzma_config="" + return 0 + fi + + if [ "$with_lzma" = "1" ] || [ "$with_lzma" = "" ] || [ "$with_lzma" = "2" ]; then + lzma_config="pkg-config liblzma" + else + lzma_config="$with_lzma" + fi + + version=`$lzma_config --modversion 2>/dev/null` + ret=$? + log 2 "executing $lzma_config --modversion" + log 2 " returned $version" + log 2 " exit code $ret" + + if [ -z "$version" ] || [ "$ret" != "0" ]; then + log 1 "checking liblzma... not found" + + # It was forced, so it should be found. + if [ "$with_lzma" != "1" ]; then + log 1 "configure: error: pkg-config liblzma couldn't be found" + log 1 "configure: error: you supplied '$with_lzma', but it seems invalid" + exit 1 + fi + + lzma_config="" + return 0 + fi + + log 1 "checking liblzma... found" +} + detect_png() { # 0 means no, 1 is auto-detect, 2 is force if [ "$with_png" = "0" ]; then @@ -3236,6 +3312,7 @@ showhelp() { echo " --with-cocoa enables COCOA video driver (OSX ONLY)" echo " --with-sdl[=sdl-config] enables SDL video driver support" echo " --with-zlib[=zlib.a] enables zlib support" + echo " --with-liblzma[=liblzma.a] enables liblzma support" echo " --with-liblzo2[=liblzo2.a] enables liblzo2 support" echo " --with-png[=libpng-config] enables libpng support" echo " --with-freetype[=freetype-config]" diff --git a/os/debian/control b/os/debian/control index d971a3899..de22a807a 100644 --- a/os/debian/control +++ b/os/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: Matthijs Kooijman Uploaders: Jordi Mallach DM-Upload-Allowed: yes -Build-Depends: debhelper (>= 7.0.50), libsdl-dev, zlib1g-dev, libpng-dev, libfreetype6-dev, libfontconfig-dev, libicu-dev, liblzo2-dev +Build-Depends: debhelper (>= 7.0.50), libsdl-dev, zlib1g-dev, libpng-dev, libfreetype6-dev, libfontconfig-dev, libicu-dev, liblzma-dev, liblzo2-dev Standards-Version: 3.8.4 Vcs-Browser: http://git.debian.org/?p=collab-maint/openttd.git Vcs-Git: git://git.debian.org/collab-maint/openttd.git diff --git a/os/debian/rules b/os/debian/rules index 5b852fbe1..92056a2bc 100755 --- a/os/debian/rules +++ b/os/debian/rules @@ -17,7 +17,7 @@ endif # to be explicit about the dependencies, in case we're not running in a # clean build root. override_dh_auto_configure: - ./configure $(CROSS) --prefix-dir=/usr --install-dir=debian/openttd --without-allegro --with-zlib --with-sdl --with-png --with-freetype --with-fontconfig --with-icu --with-liblzo2 --without-iconv --disable-strip CFLAGS="$(CFLAGS) -g" + ./configure $(CROSS) --prefix-dir=/usr --install-dir=debian/openttd --without-allegro --with-zlib --with-sdl --with-png --with-freetype --with-fontconfig --with-icu --with-liblzo2 --with-lzma --without-iconv --disable-strip CFLAGS="$(CFLAGS) -g" # Do some extra installation override_dh_auto_install: diff --git a/projects/openttd_vs100.vcxproj b/projects/openttd_vs100.vcxproj index 34173c190..b7f150282 100644 --- a/projects/openttd_vs100.vcxproj +++ b/projects/openttd_vs100.vcxproj @@ -102,7 +102,7 @@ Size true ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";WITH_ASSERT;%(PreprocessorDefinitions) + WIN32;NDEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";WITH_ASSERT;%(PreprocessorDefinitions) true Sync MultiThreaded @@ -131,7 +131,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true %(IgnoreSpecificDefaultLibraries) true @@ -156,7 +156,7 @@ /MP %(AdditionalOptions) Disabled ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";%(PreprocessorDefinitions) + WIN32;_DEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebug @@ -176,7 +176,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true LIBCMT.lib;%(IgnoreSpecificDefaultLibraries) true @@ -204,7 +204,7 @@ Size true ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;WITH_ASSERT;%(PreprocessorDefinitions) + WIN32;NDEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;WITH_ASSERT;%(PreprocessorDefinitions) true Sync MultiThreaded @@ -233,7 +233,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true %(IgnoreSpecificDefaultLibraries) true @@ -256,7 +256,7 @@ /MP %(AdditionalOptions) Disabled ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;%(PreprocessorDefinitions) + WIN32;_DEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebug @@ -280,7 +280,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true LIBCMT.lib;%(IgnoreSpecificDefaultLibraries) true diff --git a/projects/openttd_vs100.vcxproj.in b/projects/openttd_vs100.vcxproj.in index 686f19299..434c228c7 100644 --- a/projects/openttd_vs100.vcxproj.in +++ b/projects/openttd_vs100.vcxproj.in @@ -102,7 +102,7 @@ Size true ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";WITH_ASSERT;%(PreprocessorDefinitions) + WIN32;NDEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";WITH_ASSERT;%(PreprocessorDefinitions) true Sync MultiThreaded @@ -131,7 +131,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true %(IgnoreSpecificDefaultLibraries) true @@ -156,7 +156,7 @@ /MP %(AdditionalOptions) Disabled ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";%(PreprocessorDefinitions) + WIN32;_DEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebug @@ -176,7 +176,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true LIBCMT.lib;%(IgnoreSpecificDefaultLibraries) true @@ -204,7 +204,7 @@ Size true ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;WITH_ASSERT;%(PreprocessorDefinitions) + WIN32;NDEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;WITH_ASSERT;%(PreprocessorDefinitions) true Sync MultiThreaded @@ -233,7 +233,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true %(IgnoreSpecificDefaultLibraries) true @@ -256,7 +256,7 @@ /MP %(AdditionalOptions) Disabled ..\objs\langs;..\src\3rdparty\squirrel\include;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;%(PreprocessorDefinitions) + WIN32;_DEBUG;_CONSOLE;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR="OpenTTD";_SQ64;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebug @@ -280,7 +280,7 @@ 0x0809 - winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) + winmm.lib;ws2_32.lib;libpng.lib;zlibstat.lib;lzo2.lib;liblzma.lib;libfreetype2.lib;libicu.lib;%(AdditionalDependencies) true LIBCMT.lib;%(IgnoreSpecificDefaultLibraries) true diff --git a/projects/openttd_vs80.vcproj b/projects/openttd_vs80.vcproj index eb9d856b3..b2d4ff401 100644 --- a/projects/openttd_vs80.vcproj +++ b/projects/openttd_vs80.vcproj @@ -52,7 +52,7 @@ FavorSizeOrSpeed="2" OmitFramePointers="true" AdditionalIncludeDirectories="..\objs\langs;..\src\3rdparty\squirrel\include" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR=\"OpenTTD\";WITH_ASSERT" + PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;WIN32_ENABLE_DIRECTMUSIC_SUPPORT;WITH_ZLIB;WITH_LZO;WITH_LZMA;LZMA_API_STATIC;WITH_PNG;WITH_FREETYPE;WITH_ICU;ENABLE_NETWORK;ENABLE_AI;WITH_PERSONAL_DIR;PERSONAL_DIR=\"OpenTTD\";WITH_ASSERT" StringPooling="true" ExceptionHandling="1" RuntimeLibrary="0" @@ -87,7 +87,7 @@ /> #endif /* WITH_ICU */ +#ifdef WITH_LZMA +# include +#endif #ifdef WITH_LZO #include #endif @@ -208,6 +211,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const buffer += seprintf(buffer, last, " ICU: %s\n", buf); #endif /* WITH_ICU */ +#ifdef WITH_LZMA + buffer += seprintf(buffer, last, " LZMA: %s\n", lzma_version_string()); +#endif + #ifdef WITH_LZO buffer += seprintf(buffer, last, " LZO: %s\n", lzo_version_string()); #endif diff --git a/src/saveload/saveload.cpp b/src/saveload/saveload.cpp index 6f0bc055b..4e656c013 100644 --- a/src/saveload/saveload.cpp +++ b/src/saveload/saveload.cpp @@ -1944,6 +1944,106 @@ static void UninitWriteZlib() #endif /* WITH_ZLIB */ +/******************************************** + ********** START OF LZMA CODE ************** + ********************************************/ + +#if defined(WITH_LZMA) +#include + +/** + * Have a copy of an initialised LZMA stream. We need this as it's + * impossible to "re"-assign LZMA_STREAM_INIT to a variable, i.e. + * LZMA_STREAM_INIT can't be used to reset something. This var can. + */ +static const lzma_stream _lzma_init = LZMA_STREAM_INIT; +/** The current LZMA stream we're processing. */ +static lzma_stream _lzma; + +static bool InitReadLZMA(byte compression) +{ + _lzma = _lzma_init; + /* Allow saves up to 256 MB uncompressed */ + if (lzma_auto_decoder(&_lzma, 1 << 28, 0) != LZMA_OK) return false; + + _sl.bufsize = MEMORY_CHUNK_SIZE; + _sl.buf = _sl.buf_ori = MallocT(MEMORY_CHUNK_SIZE + MEMORY_CHUNK_SIZE); // also contains fread buffer + return true; +} + +static size_t ReadLZMA() +{ + _lzma.next_out = _sl.buf; + _lzma.avail_out = MEMORY_CHUNK_SIZE; + + do { + /* read more bytes from the file? */ + if (_lzma.avail_in == 0) { + _lzma.next_in = _sl.buf + MEMORY_CHUNK_SIZE; + _lzma.avail_in = fread(_sl.buf + MEMORY_CHUNK_SIZE, 1, MEMORY_CHUNK_SIZE, _sl.fh); + } + + /* inflate the data */ + lzma_ret r = lzma_code(&_lzma, LZMA_RUN); + if (r == LZMA_STREAM_END) break; + if (r != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "liblzma returned error code"); + } while (_lzma.avail_out != 0); + + return MEMORY_CHUNK_SIZE - _lzma.avail_out; +} + +static void UninitReadLZMA() +{ + lzma_end(&_lzma); + free(_sl.buf_ori); +} + +static bool InitWriteLZMA(byte compression) +{ + _lzma = _lzma_init; + if (lzma_easy_encoder(&_lzma, compression, LZMA_CHECK_CRC32) != LZMA_OK) return false; + + _sl.bufsize = MEMORY_CHUNK_SIZE; + _sl.buf = _sl.buf_ori = MallocT(MEMORY_CHUNK_SIZE); + return true; +} + +static void WriteLZMALoop(lzma_stream *lzma, byte *p, size_t len, lzma_action action) +{ + byte buf[MEMORY_CHUNK_SIZE]; // output buffer + size_t n; + lzma->next_in = p; + lzma->avail_in = len; + do { + lzma->next_out = buf; + lzma->avail_out = sizeof(buf); + + lzma_ret r = lzma_code(&_lzma, action); + + /* bytes were emitted? */ + if ((n = sizeof(buf) - lzma->avail_out) != 0) { + if (fwrite(buf, n, 1, _sl.fh) != 1) SlError(STR_GAME_SAVELOAD_ERROR_FILE_NOT_WRITEABLE); + } + if (r == LZMA_STREAM_END) break; + if (r != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "liblzma returned error code"); + } while (lzma->avail_in || !lzma->avail_out); +} + +static void WriteLZMA(size_t len) +{ + WriteLZMALoop(&_lzma, _sl.buf, len, LZMA_RUN); +} + +static void UninitWriteLZMA() +{ + /* flush any pending output. */ + if (_sl.fh) WriteLZMALoop(&_lzma, NULL, 0, LZMA_FINISH); + lzma_end(&_lzma); + free(_sl.buf_ori); +} + +#endif /* WITH_LZMA */ + /******************************************* ************* END OF CODE ***************** *******************************************/ @@ -1966,18 +2066,34 @@ struct SaveLoadFormat { byte max_compression; ///< the maximum compression level of this format }; +/** The different saveload formats known/understood by OpenTTD. */ static const SaveLoadFormat _saveload_formats[] = { #if defined(WITH_LZO) + /* Roughly 75% larger than zlib level 6 at only ~7% of the CPU usage. */ {"lzo", TO_BE32X('OTTD'), InitLZO, ReadLZO, UninitLZO, InitLZO, WriteLZO, UninitLZO, 0, 0, 0}, #else {"lzo", TO_BE32X('OTTD'), NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0}, #endif + /* Roughly 5 times larger at only 1% of the CPU usage over zlib level 6. */ {"none", TO_BE32X('OTTN'), InitNoComp, ReadNoComp, UninitNoComp, InitNoComp, WriteNoComp, UninitNoComp, 0, 0, 0}, #if defined(WITH_ZLIB) + /* After level 6 the speed reduction is significant (1.5x to 2.5x slower per level), but the reduction in filesize is + * fairly insignificant (~1% for each step). Lower levels become ~5-10% bigger by each level than level 6 while level + * 1 is "only" 3 times as fast. Level 0 results in uncompressed savegames at about 8 times the cost of "none". */ {"zlib", TO_BE32X('OTTZ'), InitReadZlib, ReadZlib, UninitReadZlib, InitWriteZlib, WriteZlib, UninitWriteZlib, 0, 6, 9}, #else {"zlib", TO_BE32X('OTTZ'), NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0}, #endif +#if defined(WITH_LZMA) + /* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves. + * Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower. + * The next significant reduction in file size is at level 4, but that is already 4 times slower. Level 3 is primarily 50% + * slower while not improving the filesize, while level 0 and 1 are faster, but don't reduce savegame size much. + * It's OTTX and not e.g. OTTL because liblzma is part of xz-utils and .tar.xz is prefered over .tar.lzma. */ + {"lzma", TO_BE32X('OTTX'), InitReadLZMA, ReadLZMA, UninitReadLZMA, InitWriteLZMA, WriteLZMA, UninitWriteLZMA, 0, 2, 9}, +#else + {"lzma", TO_BE32X('OTTX'), NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0}, +#endif }; /** -- cgit v1.2.3-70-g09d2