Change: [Linkgraph] Pause the game when linkgraph jobs lag (#6470)

Check if the job is still running two date fract ticks before it is due
to join, and if so pause the game until its done.
When loading a game, check if the game would block immediately due to
a job which is scheduled to be joined within two date fract ticks,
and if so pause the game until its done.
This avoids the main thread being blocked on a thread join, which appears
to the user as if the game is unresponsive, as the UI does not repaint
and cannot be interacted with.
Show if pause is due to link graph job in status bar, update network
messages.
This does not apply for network clients.

11 files changed, 98 insertions, 6 deletions

diff --git a/src/lang/english.txt b/src/lang/english.txt
index 7d7af8d85..04399d9a4 100644
--- a/src/lang/english.txt
+++ b/src/lang/english.txt
@@ -780,6 +780,7 @@ STR_SMALLMAP_TOOLTIP_ENABLE_ALL_CARGOS                          :{BLACK}Display
 STR_STATUSBAR_TOOLTIP_SHOW_LAST_NEWS                            :{BLACK}Show last message or news report
 STR_STATUSBAR_COMPANY_NAME                                      :{SILVER}- -  {COMPANY}  - -
 STR_STATUSBAR_PAUSED                                            :{YELLOW}*  *  PAUSED  *  *
+STR_STATUSBAR_PAUSED_LINK_GRAPH                                 :{ORANGE}*  *  PAUSED (waiting for link graph update) *  *
 STR_STATUSBAR_AUTOSAVE                                          :{RED}AUTOSAVE
 STR_STATUSBAR_SAVING_GAME                                       :{RED}*  *  SAVING GAME  *  *
 
@@ -2217,11 +2218,13 @@ STR_NETWORK_SERVER_MESSAGE_GAME_STILL_PAUSED_1                  :Game still paus
 STR_NETWORK_SERVER_MESSAGE_GAME_STILL_PAUSED_2                  :Game still paused ({STRING}, {STRING})
 STR_NETWORK_SERVER_MESSAGE_GAME_STILL_PAUSED_3                  :Game still paused ({STRING}, {STRING}, {STRING})
 STR_NETWORK_SERVER_MESSAGE_GAME_STILL_PAUSED_4                  :Game still paused ({STRING}, {STRING}, {STRING}, {STRING})
+STR_NETWORK_SERVER_MESSAGE_GAME_STILL_PAUSED_5                  :Game still paused ({STRING}, {STRING}, {STRING}, {STRING}, {STRING})
 STR_NETWORK_SERVER_MESSAGE_GAME_UNPAUSED                        :Game unpaused ({STRING})
 STR_NETWORK_SERVER_MESSAGE_GAME_REASON_NOT_ENOUGH_PLAYERS       :number of players
 STR_NETWORK_SERVER_MESSAGE_GAME_REASON_CONNECTING_CLIENTS       :connecting clients
 STR_NETWORK_SERVER_MESSAGE_GAME_REASON_MANUAL                   :manual
 STR_NETWORK_SERVER_MESSAGE_GAME_REASON_GAME_SCRIPT              :game script
+STR_NETWORK_SERVER_MESSAGE_GAME_REASON_LINK_GRAPH               :waiting for link graph update
 ############ End of leave-in-this-order
 STR_NETWORK_MESSAGE_CLIENT_LEAVING                              :leaving
 STR_NETWORK_MESSAGE_CLIENT_JOINED                               :*** {RAW_STRING} has joined the game
diff --git a/src/linkgraph/linkgraphjob.cpp b/src/linkgraph/linkgraphjob.cpp
index 2d7b407da..c66ddeac7 100644
--- a/src/linkgraph/linkgraphjob.cpp
+++ b/src/linkgraph/linkgraphjob.cpp
@@ -37,7 +37,8 @@ LinkGraphJob::LinkGraphJob(const LinkGraph &orig) :
 		 * This is on purpose. */
 		link_graph(orig),
 		settings(_settings_game.linkgraph),
-		join_date(_date + _settings_game.linkgraph.recalc_time)
+		join_date(_date + _settings_game.linkgraph.recalc_time),
+		job_completed(false)
 {
 }
 
diff --git a/src/linkgraph/linkgraphjob.h b/src/linkgraph/linkgraphjob.h
index cd7ece4b1..ab5e07fb1 100644
--- a/src/linkgraph/linkgraphjob.h
+++ b/src/linkgraph/linkgraphjob.h
@@ -13,6 +13,7 @@
 #include "../thread.h"
 #include "linkgraph.h"
 #include <list>
+#include <atomic>
 
 class LinkGraphJob;
 class Path;
@@ -61,6 +62,7 @@ protected:
 	Date join_date;                   ///< Date when the job is to be joined.
 	NodeAnnotationVector nodes;       ///< Extra node data necessary for link graph calculation.
 	EdgeAnnotationMatrix edges;       ///< Extra edge data necessary for link graph calculation.
+	std::atomic<bool> job_completed;  ///< Is the job still running. This is accessed by multiple threads and reads may be stale.
 
 	void EraseFlows(NodeID from);
 	void JoinThread();
@@ -265,7 +267,7 @@ public:
 	 * settings have to be brutally const-casted in order to populate them.
 	 */
 	LinkGraphJob() : settings(_settings_game.linkgraph),
-			join_date(INVALID_DATE) {}
+			join_date(INVALID_DATE), job_completed(false) {}
 
 	LinkGraphJob(const LinkGraph &orig);
 	~LinkGraphJob();
@@ -273,10 +275,17 @@ public:
 	void Init();
 
 	/**
+	 * Check if job has actually finished.
+	 * This is allowed to spuriously return an incorrect value.
+	 * @return True if job has actually finished.
+	 */
+	inline bool IsJobCompleted() const { return this->job_completed.load(std::memory_order_acquire); }
+
+	/**
 	 * Check if job is supposed to be finished.
 	 * @return True if job should be finished by now, false if not.
 	 */
-	inline bool IsFinished() const { return this->join_date <= _date; }
+	inline bool IsScheduledToBeJoined() const { return this->join_date <= _date; }
 
 	/**
 	 * Get the date when the job should be finished.
diff --git a/src/linkgraph/linkgraphschedule.cpp b/src/linkgraph/linkgraphschedule.cpp
index 964744509..2638b77ea 100644
--- a/src/linkgraph/linkgraphschedule.cpp
+++ b/src/linkgraph/linkgraphschedule.cpp
@@ -14,6 +14,7 @@
 #include "mcf.h"
 #include "flowmapper.h"
 #include "../framerate_type.h"
+#include "../command_func.h"
 
 #include "../safeguards.h"
 
@@ -49,13 +50,24 @@ void LinkGraphSchedule::SpawnNext()
 }
 
 /**
+ * Check if the next job is supposed to be finished, but has not yet completed.
+ * @return True if job should be finished by now but is still running, false if not.
+ */
+bool LinkGraphSchedule::IsJoinWithUnfinishedJobDue() const
+{
+	if (this->running.empty()) return false;
+	const LinkGraphJob *next = this->running.front();
+	return next->IsScheduledToBeJoined() && !next->IsJobCompleted();
+}
+
+/**
  * Join the next finished job, if available.
  */
 void LinkGraphSchedule::JoinNext()
 {
 	if (this->running.empty()) return;
 	LinkGraphJob *next = this->running.front();
-	if (!next->IsFinished()) return;
+	if (!next->IsScheduledToBeJoined()) return;
 	this->running.pop_front();
 	LinkGraphID id = next->LinkGraphIndex();
 	delete next; // implicitly joins the thread
@@ -75,6 +87,18 @@ void LinkGraphSchedule::JoinNext()
 	for (uint i = 0; i < lengthof(instance.handlers); ++i) {
 		instance.handlers[i]->Run(*job);
 	}
+
+	/*
+	 * Readers of this variable in another thread may see an out of date value.
+	 * However this is OK as this will only happen just as a job is completing,
+	 * and the real synchronisation is provided by the thread join operation.
+	 * In the worst case the main thread will be paused for longer than
+	 * strictly necessary before joining.
+	 * This is just a hint variable to avoid performing the join excessively
+	 * early and blocking the main thread.
+	 */
+
+	job->job_completed.store(true, std::memory_order_release);
 }
 
 /**
@@ -136,6 +160,42 @@ LinkGraphSchedule::~LinkGraphSchedule()
 }
 
 /**
+ * Pause the game if in 2 _date_fract ticks, we would do a join with the next
+ * link graph job, but it is still running.
+ * The check is done 2 _date_fract ticks early instead of 1, as in multiplayer
+ * calls to DoCommandP are executed after a delay of 1 _date_fract tick.
+ * If we previously paused, unpause if the job is now ready to be joined with.
+ */
+void StateGameLoop_LinkGraphPauseControl()
+{
+	if (_pause_mode & PM_PAUSED_LINK_GRAPH) {
+		/* We are paused waiting on a job, check the job every tick. */
+		if (!LinkGraphSchedule::instance.IsJoinWithUnfinishedJobDue()) {
+			DoCommandP(0, PM_PAUSED_LINK_GRAPH, 0, CMD_PAUSE);
+		}
+	} else if (_pause_mode == PM_UNPAUSED &&
+			_date_fract == LinkGraphSchedule::SPAWN_JOIN_TICK - 2 &&
+			_date % _settings_game.linkgraph.recalc_interval == _settings_game.linkgraph.recalc_interval / 2 &&
+			LinkGraphSchedule::instance.IsJoinWithUnfinishedJobDue()) {
+		/* Perform check two _date_fract ticks before we would join, to make
+		 * sure it also works in multiplayer. */
+		DoCommandP(0, PM_PAUSED_LINK_GRAPH, 1, CMD_PAUSE);
+	}
+}
+
+/**
+ * Pause the game on load if we would do a join with the next link graph job,
+ * but it is still running, and it would not be caught by a call to
+ * StateGameLoop_LinkGraphPauseControl().
+ */
+void AfterLoad_LinkGraphPauseControl()
+{
+	if (LinkGraphSchedule::instance.IsJoinWithUnfinishedJobDue()) {
+		_pause_mode |= PM_PAUSED_LINK_GRAPH;
+	}
+}
+
+/**
  * Spawn or join a link graph job or compress a link graph if any link graph is
  * due to do so.
  */
diff --git a/src/linkgraph/linkgraphschedule.h b/src/linkgraph/linkgraphschedule.h
index 62ca2b0c1..6a6dff697 100644
--- a/src/linkgraph/linkgraphschedule.h
+++ b/src/linkgraph/linkgraphschedule.h
@@ -55,6 +55,7 @@ public:
 	static void Clear();
 
 	void SpawnNext();
+	bool IsJoinWithUnfinishedJobDue() const;
 	void JoinNext();
 	void SpawnAll();
 	void ShiftDates(int interval);
@@ -76,4 +77,7 @@ public:
 	void Unqueue(LinkGraph *lg) { this->schedule.remove(lg); }
 };
 
+void StateGameLoop_LinkGraphPauseControl();
+void AfterLoad_LinkGraphPauseControl();
+
 #endif /* LINKGRAPHSCHEDULE_H */
diff --git a/src/misc_cmd.cpp b/src/misc_cmd.cpp
index 63dfb1525..2b32c06bf 100644
--- a/src/misc_cmd.cpp
+++ b/src/misc_cmd.cpp
@@ -150,6 +150,7 @@ CommandCost CmdPause(TileIndex tile, DoCommandFlag flags, uint32 p1, uint32 p2,
 		case PM_PAUSED_ERROR:
 		case PM_PAUSED_NORMAL:
 		case PM_PAUSED_GAME_SCRIPT:
+		case PM_PAUSED_LINK_GRAPH:
 			break;
 
 		case PM_PAUSED_JOIN:
diff --git a/src/network/network.cpp b/src/network/network.cpp
index a100b6b95..da341f253 100644
--- a/src/network/network.cpp
+++ b/src/network/network.cpp
@@ -342,7 +342,8 @@ void NetworkHandlePauseChange(PauseMode prev_mode, PauseMode changed_mode)
 		case PM_PAUSED_NORMAL:
 		case PM_PAUSED_JOIN:
 		case PM_PAUSED_GAME_SCRIPT:
-		case PM_PAUSED_ACTIVE_CLIENTS: {
+		case PM_PAUSED_ACTIVE_CLIENTS:
+		case PM_PAUSED_LINK_GRAPH: {
 			bool changed = ((_pause_mode == PM_UNPAUSED) != (prev_mode == PM_UNPAUSED));
 			bool paused = (_pause_mode != PM_UNPAUSED);
 			if (!paused && !changed) return;
@@ -355,6 +356,7 @@ void NetworkHandlePauseChange(PauseMode prev_mode, PauseMode changed_mode)
 				if ((_pause_mode & PM_PAUSED_JOIN) != PM_UNPAUSED)           SetDParam(++i, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_CONNECTING_CLIENTS);
 				if ((_pause_mode & PM_PAUSED_GAME_SCRIPT) != PM_UNPAUSED)    SetDParam(++i, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_GAME_SCRIPT);
 				if ((_pause_mode & PM_PAUSED_ACTIVE_CLIENTS) != PM_UNPAUSED) SetDParam(++i, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_NOT_ENOUGH_PLAYERS);
+				if ((_pause_mode & PM_PAUSED_LINK_GRAPH) != PM_UNPAUSED)     SetDParam(++i, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_LINK_GRAPH);
 				str = STR_NETWORK_SERVER_MESSAGE_GAME_STILL_PAUSED_1 + i;
 			} else {
 				switch (changed_mode) {
@@ -362,6 +364,7 @@ void NetworkHandlePauseChange(PauseMode prev_mode, PauseMode changed_mode)
 					case PM_PAUSED_JOIN:           SetDParam(0, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_CONNECTING_CLIENTS); break;
 					case PM_PAUSED_GAME_SCRIPT:    SetDParam(0, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_GAME_SCRIPT); break;
 					case PM_PAUSED_ACTIVE_CLIENTS: SetDParam(0, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_NOT_ENOUGH_PLAYERS); break;
+					case PM_PAUSED_LINK_GRAPH:     SetDParam(0, STR_NETWORK_SERVER_MESSAGE_GAME_REASON_LINK_GRAPH); break;
 					default: NOT_REACHED();
 				}
 				str = paused ? STR_NETWORK_SERVER_MESSAGE_GAME_PAUSED : STR_NETWORK_SERVER_MESSAGE_GAME_UNPAUSED;
diff --git a/src/openttd.cpp b/src/openttd.cpp
index 9cc5d2f20..e4fcf40e9 100644
--- a/src/openttd.cpp
+++ b/src/openttd.cpp
@@ -1355,6 +1355,10 @@ static void CheckCaches()
  */
 void StateGameLoop()
 {
+	if (!_networking || _network_server) {
+		StateGameLoop_LinkGraphPauseControl();
+	}
+
 	/* don't execute the state loop during pause */
 	if (_pause_mode != PM_UNPAUSED) {
 		PerformanceMeasurer::Paused(PFE_GAMELOOP);
diff --git a/src/openttd.h b/src/openttd.h
index 61cff2456..6568881c4 100644
--- a/src/openttd.h
+++ b/src/openttd.h
@@ -62,6 +62,7 @@ enum PauseMode : byte {
 	PM_PAUSED_ERROR          = 1 << 3, ///< A game paused because a (critical) error
 	PM_PAUSED_ACTIVE_CLIENTS = 1 << 4, ///< A game paused for 'min_active_clients'
 	PM_PAUSED_GAME_SCRIPT    = 1 << 5, ///< A game paused by a game script
+	PM_PAUSED_LINK_GRAPH     = 1 << 6, ///< A game paused due to the link graph schedule lagging
 
 	/** Pause mode bits when paused for network reasons. */
 	PMB_PAUSED_NETWORK = PM_PAUSED_ACTIVE_CLIENTS | PM_PAUSED_JOIN,
diff --git a/src/saveload/linkgraph_sl.cpp b/src/saveload/linkgraph_sl.cpp
index 29b685cf7..aa3a2ff4b 100644
--- a/src/saveload/linkgraph_sl.cpp
+++ b/src/saveload/linkgraph_sl.cpp
@@ -11,6 +11,7 @@
 #include "../linkgraph/linkgraph.h"
 #include "../linkgraph/linkgraphjob.h"
 #include "../linkgraph/linkgraphschedule.h"
+#include "../network/network.h"
 #include "../settings_internal.h"
 #include "saveload.h"
 
@@ -245,6 +246,10 @@ void AfterLoadLinkGraphs()
 	}
 
 	LinkGraphSchedule::instance.SpawnAll();
+
+	if (!_networking || _network_server) {
+		AfterLoad_LinkGraphPauseControl();
+	}
 }
 
 /**
diff --git a/src/statusbar_gui.cpp b/src/statusbar_gui.cpp
index a437f9fa8..f97d0d431 100644
--- a/src/statusbar_gui.cpp
+++ b/src/statusbar_gui.cpp
@@ -163,7 +163,8 @@ struct StatusBarWindow : Window {
 				} else if (_do_autosave) {
 					DrawString(r.left + WD_FRAMERECT_LEFT, r.right - WD_FRAMERECT_RIGHT, text_top, STR_STATUSBAR_AUTOSAVE, TC_FROMSTRING, SA_HOR_CENTER);
 				} else if (_pause_mode != PM_UNPAUSED) {
-					DrawString(r.left + WD_FRAMERECT_LEFT, r.right - WD_FRAMERECT_RIGHT, text_top, STR_STATUSBAR_PAUSED, TC_FROMSTRING, SA_HOR_CENTER);
+					StringID msg = (_pause_mode & PM_PAUSED_LINK_GRAPH) ? STR_STATUSBAR_PAUSED_LINK_GRAPH : STR_STATUSBAR_PAUSED;
+					DrawString(r.left + WD_FRAMERECT_LEFT, r.right - WD_FRAMERECT_RIGHT, text_top, msg, TC_FROMSTRING, SA_HOR_CENTER);
 				} else if (this->ticker_scroll < TICKER_STOP && _statusbar_news_item != nullptr && _statusbar_news_item->string_id != 0) {
 					/* Draw the scrolling news text */
 					if (!DrawScrollingStatusText(_statusbar_news_item, ScaleGUITrad(this->ticker_scroll), r.left + WD_FRAMERECT_LEFT, r.right - WD_FRAMERECT_RIGHT, r.top + WD_FRAMERECT_TOP, r.bottom)) {