summaryrefslogtreecommitdiff
path: root/cmake/scripts/Regression.cmake
blob: 0b4e522d3e06dc29859b7d1c4c03b0643857513c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
cmake_minimum_required(VERSION 3.5)

#
# Runs a single regressoion test
#

if (NOT REGRESSION_TEST)
    message(FATAL_ERROR "Script needs REGRESSION_TEST defined (tip: use -DREGRESSION_TEST=..)")
endif (NOT REGRESSION_TEST)
if (NOT OPENTTD_EXECUTABLE)
    message(FATAL_ERROR "Script needs OPENTTD_EXECUTABLE defined (tip: use -DOPENTTD_EXECUTABLE=..)")
endif (NOT OPENTTD_EXECUTABLE)

if (NOT EXISTS ai/${REGRESSION_TEST}/test.sav)
    message(FATAL_ERROR "Regression test ${REGRESSION_TEST} does not exist (tip: check regression folder for the correct spelling)")
endif ()

# If editbin is given, copy the executable to a new folder, and change the
# subsystem to console. The copy is needed as multiple regressions can run
# at the same time.
if (EDITBIN_EXECUTABLE)
    execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${OPENTTD_EXECUTABLE} regression/${REGRESSION_TEST}.exe)
    set(OPENTTD_EXECUTABLE "regression/${REGRESSION_TEST}.exe")

    execute_process(COMMAND ${EDITBIN_EXECUTABLE} /nologo /subsystem:console ${OPENTTD_EXECUTABLE})
endif (EDITBIN_EXECUTABLE)

# Run the regression test
execute_process(COMMAND ${OPENTTD_EXECUTABLE}
                        -x
                        -c regression/regression.cfg
                        -g ai/${REGRESSION_TEST}/test.sav
                        -snull
                        -mnull
                        -vnull:ticks=30000
                        -d script=2
                        -d misc=9
                OUTPUT_VARIABLE REGRESSION_OUTPUT
                ERROR_VARIABLE REGRESSION_RESULT
                OUTPUT_STRIP_TRAILING_WHITESPACE
                ERROR_STRIP_TRAILING_WHITESPACE
)

if (REGRESSION_OUTPUT)
    message(FATAL_ERROR "Unexpected output: ${REGRESSION_OUTPUT}")
endif (REGRESSION_OUTPUT)

if (NOT REGRESSION_RESULT)
    message(FATAL_ERROR "Regression did not output anything; did the compilation fail?")
endif (NOT REGRESSION_RESULT)

# For some reason pointer can be printed as '0x(nil)', '0x0000000000000000', or '0x0x0'
string(REPLACE "0x(nil)" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REPLACE "0x0000000000000000" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REPLACE "0x0x0" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")

# Convert the output to a format that is expected (and more readable) by result.txt
string(REPLACE "\ndbg: [script]" "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REPLACE "\n " "\nERROR: " REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REPLACE "\nERROR: [1] " "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REPLACE "\n[P] " "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REGEX REPLACE "dbg: ([^\n]*)\n?" "" REGRESSION_RESULT "${REGRESSION_RESULT}")

# Read the expected result
file(READ ai/${REGRESSION_TEST}/result.txt REGRESSION_EXPECTED)

# Convert the string to a list
string(REPLACE "\n" ";" REGRESSION_RESULT "${REGRESSION_RESULT}")
string(REPLACE "\n" ";" REGRESSION_EXPECTED "${REGRESSION_EXPECTED}")

set(ARGC 0)
set(ERROR NO)

list(LENGTH REGRESSION_EXPECTED REGRESSION_EXPECTED_LENGTH)

# Compare the output
foreach(RESULT IN LISTS REGRESSION_RESULT)
    list(GET REGRESSION_EXPECTED ${ARGC} EXPECTED)

    if (NOT RESULT STREQUAL EXPECTED)
        message("${ARGC}: - ${EXPECTED}")
        message("${ARGC}: + ${RESULT}'")
        set(ERROR YES)
    endif (NOT RESULT STREQUAL EXPECTED)

    math(EXPR ARGC "${ARGC} + 1")
endforeach(RESULT)

if (NOT REGRESSION_EXPECTED_LENGTH EQUAL ARGC)
    math(EXPR MISSING "${REGRESSION_EXPECTED_LENGTH} - ${ARGC}")
    message("(${MISSING} more lines were expected than found)")
    set(ERROR YES)
endif (NOT REGRESSION_EXPECTED_LENGTH EQUAL ARGC)

if (ERROR)
    message(FATAL_ERROR "Regression failed")
endif (ERROR)