Codechange: move regression outside of bin and make it work via CMake script

The tst_stationlist savegame had to be changed to start the correct
AI. In the old setup, all regression AIs had the same name, which
made it impossible to run both regressions in parallel. With the new
setup this is possible.

Although all files are available to run the regression, it won't
really work till CMake is introduced (which will happen in a few
commits from here)

15 files changed, 125 insertions, 303 deletions

diff --git a/bin/ai/regression/completeness.sh b/bin/ai/regression/completeness.sh
deleted file mode 100755
index 46cee4ed3..000000000
--- a/bin/ai/regression/completeness.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/sh
-
-if ! [ -f ai/regression/completeness.sh ]; then
-	echo "Make sure you are in the root of OpenTTD before starting this script."
-	exit 1
-fi
-
-cat ai/regression/tst_*/main.nut | tr ';' '\n' | awk '
-/^function/ {
-	for (local in locals) {
-		delete locals[local]
-	}
-	if (match($0, "function Regression::Start") || match($0, "function Regression::Stop")) next
-	locals["this"] = "AIControllerSquirrel"
-}
-
-/local/ {
-	gsub(".*local", "local")
-	if (match($4, "^AI")) {
-		sub("\\(.*", "", $4)
-		locals[$2] = $4
-	}
-}
-
-/Valuate/ {
-	gsub(".*Valuate\\(", "")
-	gsub("\\).*", "")
-	gsub(",.*", "")
-	gsub("\\.", "::")
-	print $0
-}
-
-/\./ {
-	for (local in locals) {
-		if (match($0, local ".")) {
-			fname = substr($0, index($0, local "."))
-			sub("\\(.*", "", fname)
-			sub("\\.", "::", fname)
-			sub(local, locals[local], fname)
-			print fname
-			if (match(locals[local], "List")) {
-				sub(locals[local], "AIAbstractList", fname)
-				print fname
-			}
-		}
-	}
-	# We want to remove everything before the FIRST occurrence of AI.
-	# If we do not remove any other occurrences of AI from the string
-	# we will remove everything before the LAST occurrence of AI, so
-	# do some little magic to make it work the way we want.
-	sub("AI", "AXXXXY")
-	gsub("AI", "AXXXXX")
-	sub(".*AXXXXY", "AI")
-	if (match($0, "^AI") && match($0, ".")) {
-		sub("\\(.*", "", $0)
-		sub("\\.", "::", $0)
-		print $0
-	}
-}
-' | sed 's/	//g' | sort | uniq > tmp.in_regression
-
-grep 'DefSQ.*Method' ../src/script/api/ai/*.hpp.sq | grep -v 'AIError::' | grep -v 'AIAbstractList::Valuate' | grep -v '::GetClassName' | sed 's/^[^,]*, &//g;s/,[^,]*//g' | sort > tmp.in_api
-
-diff -u tmp.in_regression tmp.in_api | grep -v '^+++' | grep '^+' | sed 's/^+//'
-
-rm -f tmp.in_regression tmp.in_api
-
diff --git a/bin/ai/regression/run.sh b/bin/ai/regression/run.sh
deleted file mode 100755
index 7574b0b38..000000000
--- a/bin/ai/regression/run.sh
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/sh
-
-if ! [ -f ai/regression/run.sh ]; then
-	echo "Make sure you are in the root of OpenTTD before starting this script."
-	exit 1
-fi
-
-if [ -f scripts/game_start.scr ]; then
-	mv scripts/game_start.scr scripts/game_start.scr.regression
-fi
-
-params=""
-gdb=""
-if [ "$1" != "-r" ]; then
-	params="-snull -mnull -vnull:ticks=30000"
-fi
-if [ "$1" = "-g" ]; then
-	gdb="gdb --ex run --args "
-fi
-
-if [ -d "ai/regression/tst_$1" ]; then
-	tests="ai/regression/tst_$1"
-elif [ -d "ai/regression/tst_$2" ]; then
-	tests="ai/regression/tst_$2"
-else
-	tests=ai/regression/tst_*
-fi
-
-ret=0
-for tst in $tests; do
-	echo -n "Running $tst... "
-
-	# Make sure that only one info.nut is present for each test run. Otherwise openttd gets confused.
-	cp ai/regression/regression_info.nut $tst/info.nut
-
-	sav=$tst/test.sav
-	if ! [ -f $sav ]; then
-		sav=ai/regression/empty.sav
-	fi
-
-	if [ -n "$gdb" ]; then
-		$gdb ./openttd -x -c ai/regression/regression.cfg $params -g $sav
-	else
-		./openttd -x -c ai/regression/regression.cfg $params -g $sav -d script=2 -d misc=9 2>&1 | awk '{ gsub("0x(\\(nil\\)|0+)(x0)?", "0x00000000", $0); gsub("^dbg: \\[script\\]", "", $0); gsub("^ ", "ERROR: ", $0); gsub("ERROR: \\[1\\] ", "", $0); gsub("\\[P\\] ", "", $0); print $0; }' | grep -v '^dbg: \[.*\]' > $tst/tmp.regression
-	fi
-
-	if [ -z "$gdb" ]; then
-		res="`diff -ub $tst/result.txt $tst/tmp.regression`"
-		if [ -z "$res" ]; then
-			echo "passed!"
-		else
-			echo "failed! Difference:"
-			echo "$res"
-			ret=1
-		fi
-	fi
-
-	rm $tst/info.nut
-
-	if [ "$1" != "-k" ]; then
-		rm -f $tst/tmp.regression
-	fi
-done
-
-if [ -f scripts/game_start.scr.regression ]; then
-	mv scripts/game_start.scr.regression scripts/game_start.scr
-fi
-
-exit $ret
diff --git a/bin/ai/regression/run.vbs b/bin/ai/regression/run.vbs
deleted file mode 100644
index b4bdef4c1..000000000
--- a/bin/ai/regression/run.vbs
+++ /dev/null
@@ -1,152 +0,0 @@
-Option Explicit
-
-' This file is part of OpenTTD.
-' OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
-' OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-' See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
-
-Dim FSO
-Set FSO = CreateObject("Scripting.FileSystemObject")
-
-Function GetTestList()
-	Dim retests, i, tests, dir
-	Set retests = New RegExp
-	Set GetTestList = CreateObject("Scripting.Dictionary")
-
-	retests.Pattern = "ai/regression/tst_*"
-	retests.Global = True
-	For i = 0 To WScript.Arguments.Count - 1
-		Dim test
-		test = "ai/regression/tst_" & WScript.Arguments.Item(i)
-		If FSO.FolderExists(test) Then
-			retests.Pattern = test
-			Exit For
-		End If
-	Next
-
-	For Each dir In FSO.GetFolder("ai/regression/").SubFolders
-		Dim name
-		name = "ai/regression/" & dir.Name
-		If retests.Test(name) Then
-			GetTestList.Add name, name
-		End If
-	Next
-End Function
-
-Function GetParams()
-	GetParams = "-snull -mnull -vnull:ticks=30000"
-	If WScript.Arguments.Count = 0 Then Exit Function
-	If WScript.Arguments.Item(0) <> "-r" Then Exit Function
-	GetParams = ""
-End Function
-
-Sub FilterFile(filename)
-	Dim lines, filter, file
-
-	Set file = FSO.OpenTextFile(filename, 1)
-	If Not file.AtEndOfStream Then
-		lines = file.ReadAll
-	End If
-	file.Close
-
-	Set filter = New RegExp
-	filter.Global = True
-	filter.Multiline = True
-	filter.Pattern = "0x(\(nil\)|0+)(x0)?"
-	lines = filter.Replace(lines, "0x00000000")
-	filter.Pattern = "^dbg: \[script\]"
-	lines = filter.Replace(lines, "")
-	filter.Pattern = "^ "
-	lines = filter.Replace(lines, "ERROR: ")
-	filter.Pattern = "ERROR: \[1\] \[P\] "
-	lines = filter.Replace(lines, "")
-	filter.Pattern = "^dbg: .*\r\n"
-	lines = filter.Replace(lines, "")
-
-	Set file = FSO.OpenTextFile(filename, 2)
-	file.Write lines
-	file.Close
-End Sub
-
-Function CompareFiles(filename1, filename2)
-	Dim file, lines1, lines2
-	Set file = FSO.OpenTextFile(filename1, 1)
-	If Not file.AtEndOfStream Then
-		lines1 = file.ReadAll
-	End IF
-	file.Close
-	Set file = FSO.OpenTextFile(filename2, 1)
-	If Not file.AtEndOfStream Then
-		lines2 = file.ReadAll
-	End IF
-	file.Close
-	CompareFiles = (lines1 = lines2)
-End Function
-
-Function RunTest(test, params, ret)
-	Dim WshShell, oExec, sav, command
-	Set WshShell = CreateObject("WScript.Shell")
-
-	' Make sure that only one info.nut is present for each test run. Otherwise openttd gets confused.
-	FSO.CopyFile "ai/regression/regression_info.nut", test & "/info.nut"
-
-	sav = test & "/test.sav"
-	If Not FSO.FileExists(sav) Then
-		sav = "ai/regression/empty.sav"
-	End If
-
-	command = ".\openttd -x -c ai/regression/regression.cfg " & params & " -g " & sav & " -d script=2 -d misc=9"
-	' 2>&1 must be after >tmp.regression, else stderr is not redirected to the file
-	WshShell.Run "cmd /c " & command & " >"& test & "/tmp.regression 2>&1", 0, True
-
-	FilterFile test & "/tmp.regression"
-
-	If CompareFiles(test & "/result.txt", test & "/tmp.regression") Then
-		RunTest = "passed!"
-	Else
-		RunTest = "failed!"
-		ret = 1
-	End If
-
-	FSO.DeleteFile test & "/info.nut"
-
-	If WScript.Arguments.Count > 0 Then
-		If WScript.Arguments.Item(0) = "-k" Then
-			Exit Function
-		End If
-	End If
-
-	FSO.DeleteFile test & "/tmp.regression"
-End Function
-
-On Error Resume Next
-WScript.StdOut.WriteLine ""
-If Err.Number <> 0 Then
-	WScript.Echo "This script must be started with cscript."
-	WScript.Quit 1
-End If
-On Error Goto 0
-
-If Not FSO.FileExists("ai/regression/run.vbs") Then
-	WScript.Echo "Make sure you are in the root of OpenTTD before starting this script."
-	WScript.Quit 1
-End If
-
-If FSO.FileExists("scripts/game_start.scr") Then
-	FSO.MoveFile "scripts/game_start.scr", "scripts/game_start.scr.regression"
-End If
-
-Dim params, test, ret
-params = GetParams()
-ret = 0
-
-For Each test in GetTestList()
-	WScript.StdOut.Write "Running " & test & "... "
-	WScript.StdOut.WriteLine RunTest(test, params, ret)
-Next
-
-If FSO.FileExists("scripts/game_start.scr.regression") Then
-	FSO.MoveFile "scripts/game_start.scr.regression", "scripts/game_start.scr"
-End If
-
-WScript.Quit ret
diff --git a/bin/ai/regression/tst_stationlist/test.sav b/bin/ai/regression/tst_stationlist/test.sav
deleted file mode 100644
index ef551c74f..000000000
--- a/bin/ai/regression/tst_stationlist/test.sav
+++ /dev/null
diff --git a/cmake/scripts/Regression.cmake b/cmake/scripts/Regression.cmake
new file mode 100644
index 000000000..0b4e522d3
--- /dev/null
+++ b/cmake/scripts/Regression.cmake
@@ -0,0 +1,97 @@
+cmake_minimum_required(VERSION 3.5)
+
+#
+# Runs a single regressoion test
+#
+
+if (NOT REGRESSION_TEST)
+    message(FATAL_ERROR "Script needs REGRESSION_TEST defined (tip: use -DREGRESSION_TEST=..)")
+endif (NOT REGRESSION_TEST)
+if (NOT OPENTTD_EXECUTABLE)
+    message(FATAL_ERROR "Script needs OPENTTD_EXECUTABLE defined (tip: use -DOPENTTD_EXECUTABLE=..)")
+endif (NOT OPENTTD_EXECUTABLE)
+
+if (NOT EXISTS ai/${REGRESSION_TEST}/test.sav)
+    message(FATAL_ERROR "Regression test ${REGRESSION_TEST} does not exist (tip: check regression folder for the correct spelling)")
+endif ()
+
+# If editbin is given, copy the executable to a new folder, and change the
+# subsystem to console. The copy is needed as multiple regressions can run
+# at the same time.
+if (EDITBIN_EXECUTABLE)
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${OPENTTD_EXECUTABLE} regression/${REGRESSION_TEST}.exe)
+    set(OPENTTD_EXECUTABLE "regression/${REGRESSION_TEST}.exe")
+
+    execute_process(COMMAND ${EDITBIN_EXECUTABLE} /nologo /subsystem:console ${OPENTTD_EXECUTABLE})
+endif (EDITBIN_EXECUTABLE)
+
+# Run the regression test
+execute_process(COMMAND ${OPENTTD_EXECUTABLE}
+                        -x
+                        -c regression/regression.cfg
+                        -g ai/${REGRESSION_TEST}/test.sav
+                        -snull
+                        -mnull
+                        -vnull:ticks=30000
+                        -d script=2
+                        -d misc=9
+                OUTPUT_VARIABLE REGRESSION_OUTPUT
+                ERROR_VARIABLE REGRESSION_RESULT
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+                ERROR_STRIP_TRAILING_WHITESPACE
+)
+
+if (REGRESSION_OUTPUT)
+    message(FATAL_ERROR "Unexpected output: ${REGRESSION_OUTPUT}")
+endif (REGRESSION_OUTPUT)
+
+if (NOT REGRESSION_RESULT)
+    message(FATAL_ERROR "Regression did not output anything; did the compilation fail?")
+endif (NOT REGRESSION_RESULT)
+
+# For some reason pointer can be printed as '0x(nil)', '0x0000000000000000', or '0x0x0'
+string(REPLACE "0x(nil)" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REPLACE "0x0000000000000000" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REPLACE "0x0x0" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
+
+# Convert the output to a format that is expected (and more readable) by result.txt
+string(REPLACE "\ndbg: [script]" "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REPLACE "\n " "\nERROR: " REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REPLACE "\nERROR: [1] " "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REPLACE "\n[P] " "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REGEX REPLACE "dbg: ([^\n]*)\n?" "" REGRESSION_RESULT "${REGRESSION_RESULT}")
+
+# Read the expected result
+file(READ ai/${REGRESSION_TEST}/result.txt REGRESSION_EXPECTED)
+
+# Convert the string to a list
+string(REPLACE "\n" ";" REGRESSION_RESULT "${REGRESSION_RESULT}")
+string(REPLACE "\n" ";" REGRESSION_EXPECTED "${REGRESSION_EXPECTED}")
+
+set(ARGC 0)
+set(ERROR NO)
+
+list(LENGTH REGRESSION_EXPECTED REGRESSION_EXPECTED_LENGTH)
+
+# Compare the output
+foreach(RESULT IN LISTS REGRESSION_RESULT)
+    list(GET REGRESSION_EXPECTED ${ARGC} EXPECTED)
+
+    if (NOT RESULT STREQUAL EXPECTED)
+        message("${ARGC}: - ${EXPECTED}")
+        message("${ARGC}: + ${RESULT}'")
+        set(ERROR YES)
+    endif (NOT RESULT STREQUAL EXPECTED)
+
+    math(EXPR ARGC "${ARGC} + 1")
+endforeach(RESULT)
+
+if (NOT REGRESSION_EXPECTED_LENGTH EQUAL ARGC)
+    math(EXPR MISSING "${REGRESSION_EXPECTED_LENGTH} - ${ARGC}")
+    message("(${MISSING} more lines were expected than found)")
+    set(ERROR YES)
+endif (NOT REGRESSION_EXPECTED_LENGTH EQUAL ARGC)
+
+if (ERROR)
+    message(FATAL_ERROR "Regression failed")
+endif (ERROR)
diff --git a/bin/ai/regression/regression.cfg b/regression/regression.cfg
index 4b8a5f6cd..4b8a5f6cd 100644
--- a/bin/ai/regression/regression.cfg
+++ b/regression/regression.cfg
diff --git a/bin/ai/regression/regression_info.nut b/regression/regression/info.nut
index 758754cfe..758754cfe 100644
--- a/bin/ai/regression/regression_info.nut
+++ b/regression/regression/info.nut
diff --git a/bin/ai/regression/tst_regression/main.nut b/regression/regression/main.nut
index 65742d71b..65742d71b 100644
--- a/bin/ai/regression/tst_regression/main.nut
+++ b/regression/regression/main.nut
diff --git a/bin/ai/regression/tst_regression/require.nut b/regression/regression/require.nut
index d8dc4baa7..d8dc4baa7 100644
--- a/bin/ai/regression/tst_regression/require.nut
+++ b/regression/regression/require.nut
diff --git a/bin/ai/regression/tst_regression/result.txt b/regression/regression/result.txt
index 4cb74aaa5..4cb74aaa5 100644
--- a/bin/ai/regression/tst_regression/result.txt
+++ b/regression/regression/result.txt
diff --git a/bin/ai/regression/empty.sav b/regression/regression/test.sav
index cf97052c2..cf97052c2 100644
--- a/bin/ai/regression/empty.sav
+++ b/regression/regression/test.sav
diff --git a/regression/stationlist/info.nut b/regression/stationlist/info.nut
new file mode 100644
index 000000000..42e5072fd
--- /dev/null
+++ b/regression/stationlist/info.nut
@@ -0,0 +1,13 @@
+class StationList extends AIInfo {
+	function GetAuthor()      { return "OpenTTD NoAI Developers Team"; }
+	function GetName()        { return "StationList"; }
+	function GetShortName()   { return "REGS"; }
+	function GetDescription() { return "This runs stationlist-tests on some commands. On the same map the result should always be the same."; }
+	function GetVersion()     { return 1; }
+	function GetAPIVersion()  { return "1.11"; }
+	function GetDate()        { return "2007-03-18"; }
+	function CreateInstance() { return "StationList"; }
+}
+
+RegisterAI(StationList());
+
diff --git a/bin/ai/regression/tst_stationlist/main.nut b/regression/stationlist/main.nut
index 2f00ea1d6..b639c5404 100644
--- a/bin/ai/regression/tst_stationlist/main.nut
+++ b/regression/stationlist/main.nut
@@ -1,9 +1,9 @@
-class Regression extends AIController {
+class StationList extends AIController {
 	function Start();
 };
 
 
-function Regression::StationList()
+function StationList::StationList()
 {
 	local list = AIStationList(AIStation.STATION_BUS_STOP + AIStation.STATION_TRUCK_STOP);
 
@@ -27,7 +27,7 @@ function Regression::StationList()
 	}
 };
 
-function Regression::StationList_Cargo()
+function StationList::StationList_Cargo()
 {
 	print("");
 	print("--StationList_Cargo--");
@@ -44,7 +44,7 @@ function Regression::StationList_Cargo()
 	}
 };
 
-function Regression::StationList_CargoPlanned()
+function StationList::StationList_CargoPlanned()
 {
 	print("");
 	print("--StationList_CargoPlanned--");
@@ -58,7 +58,7 @@ function Regression::StationList_CargoPlanned()
 	}
 };
 
-function Regression::StationList_CargoPlannedByFrom()
+function StationList::StationList_CargoPlannedByFrom()
 {
 	print("");
 	print("--StationList_CargoPlannedByFrom--");
@@ -68,7 +68,7 @@ function Regression::StationList_CargoPlannedByFrom()
 	}
 };
 
-function Regression::StationList_CargoPlannedByVia()
+function StationList::StationList_CargoPlannedByVia()
 {
 	print("");
 	print("--StationList_CargoPlannedByVia--");
@@ -78,7 +78,7 @@ function Regression::StationList_CargoPlannedByVia()
 	}
 };
 
-function Regression::StationList_CargoPlannedViaByFrom()
+function StationList::StationList_CargoPlannedViaByFrom()
 {
 	print("");
 	print("--StationList_CargoPlannedViaByFrom--");
@@ -88,7 +88,7 @@ function Regression::StationList_CargoPlannedViaByFrom()
 	}
 };
 
-function Regression::StationList_CargoPlannedFromByVia()
+function StationList::StationList_CargoPlannedFromByVia()
 {
 	print("");
 	print("--StationList_CargoPlannedFromByVia--");
@@ -98,7 +98,7 @@ function Regression::StationList_CargoPlannedFromByVia()
 	}
 };
 
-function Regression::StationList_CargoWaiting()
+function StationList::StationList_CargoWaiting()
 {
 	print("");
 	print("--StationList_CargoWaiting--");
@@ -112,7 +112,7 @@ function Regression::StationList_CargoWaiting()
 	}
 };
 
-function Regression::StationList_CargoWaitingByFrom()
+function StationList::StationList_CargoWaitingByFrom()
 {
 	print("");
 	print("--StationList_CargoWaitingByFrom--");
@@ -122,7 +122,7 @@ function Regression::StationList_CargoWaitingByFrom()
 	}
 };
 
-function Regression::StationList_CargoWaitingByVia()
+function StationList::StationList_CargoWaitingByVia()
 {
 	print("");
 	print("--StationList_CargoWaitingByVia--");
@@ -132,7 +132,7 @@ function Regression::StationList_CargoWaitingByVia()
 	}
 };
 
-function Regression::StationList_CargoWaitingViaByFrom()
+function StationList::StationList_CargoWaitingViaByFrom()
 {
 	print("");
 	print("--StationList_CargoWaitingViaByFrom--");
@@ -142,7 +142,7 @@ function Regression::StationList_CargoWaitingViaByFrom()
 	}
 };
 
-function Regression::StationList_CargoWaitingFromByVia()
+function StationList::StationList_CargoWaitingFromByVia()
 {
 	print("");
 	print("--StationList_CargoWaitingFromByVia--");
@@ -152,7 +152,7 @@ function Regression::StationList_CargoWaitingFromByVia()
 	}
 };
 
-function Regression::StationList_Vehicle()
+function StationList::StationList_Vehicle()
 {
 	local list = AIStationList_Vehicle(12);
 
@@ -196,7 +196,7 @@ function Regression::StationList_Vehicle()
 	}
 }
 
-function Regression::Start()
+function StationList::Start()
 {
 	StationList();
 	StationList_Cargo();
diff --git a/bin/ai/regression/tst_stationlist/result.txt b/regression/stationlist/result.txt
index 20e594766..20e594766 100644
--- a/bin/ai/regression/tst_stationlist/result.txt
+++ b/regression/stationlist/result.txt
diff --git a/regression/stationlist/test.sav b/regression/stationlist/test.sav
new file mode 100644
index 000000000..959f77638
--- /dev/null
+++ b/regression/stationlist/test.sav