summaryrefslogtreecommitdiff
path: root/systemd
blob: c45b64b171a4ba587c1fb66b5905c324ddb761f3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370

########################################################
# Please file all bug reports, patches, and feature
# requests under:
#      https://sourceforge.net/p/logwatch/_list/tickets
# Help requests and discusion can be filed under:
#      https://sourceforge.net/p/logwatch/discussion/
########################################################

########################################################
## Copyright (c) 2016 Orion Poplawski
## Covered under the included MIT/X-Consortium License:
##    http://www.opensource.org/licenses/mit-license.php
## All modifications and contributions by other persons to
## this script are assumed to have been donated to the
## Logwatch project and thus assume the above copyright
## and licensing terms.  If you want to make contributions
## under your own copyright or a different license this
## must be explicitly stated in the contribution an the
## Logwatch project reserves the right to not accept such
## contributions.  If you have made significant
## contributions to this script and want to claim
## copyright please contact logwatch-devel@lists.sourceforge.net.
#########################################################

use strict;
my $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0;
my $Ignore_failed = $ENV{'ignore_failed'} || "";
my $Ignore_leftover = $ENV{'ignore_leftover'} || "";
my %ConfigError;
my %Activated;
my %Closed;
my %Failed;
my %Deactivated;
my %LeftOver;
my $Reexecuted = 0;
my %Reloaded;
my %Skipped;
my %Slice;
my %Slow;
my %Started;
my %Target;
my $TimeChanged = 0;
my $LastTarget;
my %UserSession;
my %OtherList;

# Failure will generate multiple messages like:
# Feb  5 16:37:50 hostname systemd: ansible-pull.service: main process exited, code=exited, status=2/INVALIDARGUMENT
# Feb  5 16:37:50 hostname systemd: Failed to start Run ansible-pull on boot.
# Feb  5 16:37:50 hostname systemd: Unit ansible-pull.service entered failed state.
# Feb  5 16:37:50 hostname systemd: ansible-pull.service failed.

while (defined(my $ThisLine = <STDIN>)) {
   chomp($ThisLine);
   if ($ThisLine =~ /^(Activat|Deactivat|Mount|Unmount|Reload|Start|Stopp)ing / or
       # These events will be caught with the Unit X entered failed state message
       $ThisLine =~ /^Failed to start / or
       $ThisLine =~ /: Failed with result / or
       $ThisLine =~ /Failed at step / or
       $ThisLine =~ / failed\.$/ or
       $ThisLine =~ /([Cc]ontrol|[Mm]ain|[Mm]ount) process exited, code=(exited|killed|dumped),? status=/ or
       # Informational
       $ThisLine =~ /^Closed .*[\. ][Ss]ockets?\.$/ or
       $ThisLine =~ /^Closed .* [Ss]cheduler\.$/ or
       $ThisLine =~ /^Closed .* [Ww]atch\.$/ or
       $ThisLine =~ /^Closed (?:Multimedia|Sound) System\.$/ or
       $ThisLine =~ /^Closed udev / or
       $ThisLine =~ /: Consumed .* CPU time\.$/ or
       $ThisLine =~ /^\S+: Succeeded\.$/ or
       $ThisLine =~ /^\S+\.service: Current command vanished from the unit file, execution of the command list won't be resumed\.$/ or
       # crond will never restart process when it is restarted
       $ThisLine =~ /^crond\.service: Found left-over process \d+ \(.*\) in control group while starting unit\. Ignoring\.$/ or
       $ThisLine =~ /^Received SIGINT\./ or
       $ThisLine =~ /^Deactivated / or
       $ThisLine =~ /^Detected (architecture|virtualization) / or
       $ThisLine =~ /^Found device / or
       $ThisLine =~ /Got automount request for \/proc\// or
       $ThisLine =~ /^Inserted module / or
       $ThisLine =~ /^Listening on / or
       $ThisLine =~ /^Mounted / or
       $ThisLine =~ /^Relabelled / or
       $ThisLine =~ /^Reloading\.$/ or         # Happens on each boot at switch root
       $ThisLine =~ /^RTC configured in / or
       $ThisLine =~ /^Running in initial RAM disk\.$/ or
       $ThisLine =~ /^selinux: avc: *received policyload notice/ or
       $ThisLine =~ /^Set hostname to / or
       $ThisLine =~ /^(?:Set up|Unset) automount Arbitrary Executable File Formats File System Automount Point\.$/ or
       $ThisLine =~ /^Shutting down\.$/ or
       $ThisLine =~ /^Startup finished in / or
       $ThisLine =~ /^Stopped / or
       $ThisLine =~ /^Switching root\.$/ or
       $ThisLine =~ /: Succeeded\.$/ or
       $ThisLine =~ /^Successfully loaded SELinux policy in / or
       $ThisLine =~ /^Finished .*\.$/ or
       $ThisLine =~ /already active, refusing\./ or
       $ThisLine =~ /^\S+: Triggering OnFailure= dependencies\.$/ or
       $ThisLine =~ /^Failed to propagate agent release message:/ or
       $ThisLine =~ /: Supervising process .* which is not our child\. We'll most likely not notice when it exits\.$/ or
       $ThisLine =~ /: Got notification message from PID \d+, but reception is disabled\./ or
       $ThisLine =~ /: Got notification message from PID \d+, but reception only permitted for main PID \d+/ or
       $ThisLine =~ /Cannot find unit for notify message of PID \d+/ or
       $ThisLine =~ /^systemd .* running in system mode/ or
       # This is preceeded by a more descriptive message
       $ThisLine =~ /^This usually indicates unclean termination of a previous run, or service implementation deficiencies\.$/ or
       $ThisLine =~ /Transaction is destructive\./ or
       $ThisLine =~ /^Unit .* is bound to inactive unit .*\. Stopping, too\./ or
       $ThisLine =~ /Unit (.* is )?not needed anymore\. Stopping\./ or
       $ThisLine =~ /State '(stop-sigterm|stop-final-sigterm)' timed out\. Killing\./ or
       $ThisLine =~ /: Start(-pre)? operation timed out\. Terminating\./ or
       $ThisLine =~ /hold-?off time over, scheduling restart\./ or
       $ThisLine =~ /Service has no hold-off time.*, scheduling restart\./ or
       $ThisLine =~ /Scheduled restart job, restart counter is at .*\./ or
       $ThisLine =~ /Stopping timed out\. Killing\./ or
       $ThisLine =~ /^Timed out waiting for/ or
       $ThisLine =~ /: Watchdog timeout/ or
       $ThisLine =~ /^Dependency failed for / or
       $ThisLine =~ /Processes still around after .*SIGKILL\./ or
       $ThisLine =~ /^Unmounted / or
       $ThisLine =~ /: Unit is bound to inactive unit / or
       $ThisLine =~ /[Hh]ardware watchdog / or
       # This is now trapped as a Slow Start error
       #$ThisLine =~ /PID file .* not readable \(yet\?\) after start/ or
       $ThisLine =~ /Failed to read PID from file / or
       # Units can depend on files that do not exist
       $ThisLine =~ /(Cannot add dependency|Failed to enqueue OnFailure=) job(:? for unit .*)?, ignoring: Unit (:?.* (failed to load: No such file or directory|not found))\.$/ or
       # https://bugs.freedesktop.org/show_bug.cgi?id=90386
       $ThisLine =~ /Dev(ice)? .* appeared twice with different sysfs paths .* and / or
       # Inactive units are sometimes reloaded
       $ThisLine =~ /^Unit .* cannot be reloaded because it is inactive\.$/ or
       $ThisLine =~ /^.*: Unit cannot be reloaded because it is inactive\.$/ or
       $ThisLine =~ / is not active\.$/ or
       $ThisLine =~ /^Condition check resulted in .* being skipped\.$/ or
       $ThisLine =~ /^Failed to parse swap priority "[^"]+", ignoring: Numerical result out of range$/ or
       # https://bugzilla.redhat.com/show_bug.cgi?id=1306452 
       $ThisLine =~ /^[^ ]*\.mount: Directory \/[^ ]* to mount over is not empty, mounting anyway\.$/ or
       # A known issue - reported by multiple distributions
       $ThisLine =~ /^user\@\d+\.service: Failed at step CGROUP spawning \/usr\/lib\/systemd\/systemd: No such file or directory$/ or
       $ThisLine =~ /^Received SIGRTMIN\+2[01] from PID \d+ \((?:plymouthd|n\/a)\)\.$/ or
       # https://bugzilla.redhat.com/show_bug.cgi?id=1072368
       $ThisLine =~ /^Received SIGRTMIN\+24 from PID \d+ \((?:kill|n\/a)\)\.$/ or
       $ThisLine =~ /: Killing process \d+ \(.*\) with signal SIG.+\.$/ or
       $ThisLine =~ /^Removed slice / or
       $ThisLine =~ /^pam_unix\(systemd-user:session\): session (?:opened|closed) for user/ or
       $ThisLine =~ /Adding .* random time\.$/ or
       # These happen on every shutdown - downgraded to debug message in systemd v235
       # https://github.com/systemd/systemd/issues/6777
       $ThisLine =~ /^Failed to propagate agent release message: (?:Connection reset by peer|Transport endpoint is not connected)/ or
       $ThisLine =~ /^cgroup compatibility translation between legacy and unified hierarchy settings activated\. See cgroup-compat debug messages for details\.$/ or
       $ThisLine =~ /^.*\.socket: Socket service .* already active/
   ) {
      # Ignore these
   } elsif (my ($service,$reason) = ($ThisLine =~ /^Configuration file ([^ ]*) is ([^.]*)\./)) {
      $ConfigError{$reason}{$service}++;
   } elsif (my ($service,$reason) = ($ThisLine =~ /^\[(.*)\] (Support for option .* has been removed) and it is ignored/)) {
      $ConfigError{$reason}{$service}++;
   } elsif (my ($service,$reason) = ($ThisLine =~ /^\[?([^\]:]+(?::\d+)?)[\]:]? (Unknown .* in section '.*')/)) {
      $ConfigError{$reason}{$service}++;
   } elsif (my ($service,$reason) = ($ThisLine =~ /^[^:]+\/([^\/:]+)\.service:\d+: (\S+= references path below legacy directory|\.include directives are deprecated)[, ]/)) {
      $ConfigError{$reason}{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Unit (.*) entered failed state\.$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^(.*): Failed to execute command/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^(.*): Unit entered failed state\.$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^(.*) failed with error code \d+\.$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^(.*): Start request repeated too quickly\.$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^(.*): Job .* timed out\.$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^(.*): Job .* failed with result .*$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Failed (unmounting .*)\.$/)) {
      $Failed{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Failed to (listen on .*)\.$/)) {
      $Failed{$service}++;
   } elsif (my ($target) = ($ThisLine =~ /^Reached target (.*)\.$/)) {
      $Target{$target}++;
      $LastTarget = $target;
   } elsif (my ($session, $user) = ($ThisLine =~ /^Started Session (\d+) of user (.*)\.$/)) {
      $UserSession{$user}->{$session}++;
   } elsif (my ($service) = ($ThisLine =~ /^Activated (.*)\.$/)) {
      $Activated{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Started (.*)\.$/)) {
      $Started{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Reloaded (.*)\.$/)) {
      $Reloaded{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Deactivated (.*)\.$/)) {
      $Deactivated{$service}++;
   } elsif (my ($service) = ($ThisLine =~ /^Closed (.*)\.$/)) {
      $Closed{$service}++;
   } elsif ($ThisLine eq "Reexecuting.") {
      $Reexecuted++ if $Detail;
   } elsif ($ThisLine =~ /^Time has been changed$/) {
      $TimeChanged++;
   } elsif (my ($slice) = ($ThisLine =~ /^Created slice (.*)\.$/)) {
      $Slice{$slice}++;
   } elsif (my ($name) = ($ThisLine =~ /^Condition check resulted in (.*) being skipped\.$/)) {
      $Skipped{$name}++ if $Detail;
   } elsif (my ($pidfile) = ($ThisLine =~ /^PID file (.*) not readable \(yet\?\) after start\.$/)) {
      $Slow{$pidfile}++;
   } elsif (my ($pidfile) = ($ThisLine =~ /: Can't open PID file (.*) \(yet\?\) after start:/)) {
      $Slow{$pidfile}++;
   } elsif (my ($Service, $Exe) = ($ThisLine =~ /^(.*): Found left-over process \d+ \((.*)\) in control group while starting unit\. Ignoring\.$/)) {
      $LeftOver{"$Service:$Exe"}++ unless "$Service:$Exe" =~/^$Ignore_leftover$/i;
   } else {
      $OtherList{$ThisLine}++;
   }
}

if (keys %ConfigError) {
    print "Configuration errors:\n";
    foreach my $reason (sort {$a cmp $b} keys %ConfigError) {
	my $tot = 0;
	print "   $reason";
	foreach my $service (sort {$a cmp $b} keys %{$ConfigError{$reason}}) {
	    $tot += $ConfigError{$reason}{$service};
	    if ($Detail >= 10) {
		print "\n      $service: $ConfigError{$reason}{$service} Time(s)";
	    }
	}
	if ($Detail < 10) {
	    print ": $tot Time(s)"
	}
	print "\n";
    }
    print "\n";
}

# Because we set Failed in multiple locations, cleanup once here
foreach my $item (keys %Failed) {
   delete $Failed{$item} if ($item =~ /^$Ignore_failed$/i);
}

if (keys %Failed) {
   print "ERROR: Failed state:\n";
   foreach my $item (sort {$a cmp $b} keys %Failed) {
      print "        $item: $Failed{$item} Time(s)\n";
   }
   print "\n";
}

# Detail >= 1
if ($Reexecuted) {
   print "Reexecuted systemd: $Reexecuted Time(s)\n\n";
}

if (keys %Skipped) {
   print "Condition check resulted in the following being skipped:\n";
   foreach my $item (sort {$a cmp $b} keys %Skipped) {
      print "        $item: $Skipped{$item} Time(s)\n";
   }
   print "\n";
}

if (keys %LeftOver) {
   print "Warning: Found left-over process in control group while starting unit:\n";
   foreach my $item (sort {$a cmp $b} keys %LeftOver) {
      my ($service, $exe) = split(":", $item);
      print "        $service($exe): $LeftOver{$item} Time(s)\n";
   }
   print "\n";
}

if (keys %Target && $Detail > 3) {
   print "Reached target $LastTarget: $Target{$LastTarget} Time(s)";
   if ($Detail > 10) {
      print ", and:\n";
      foreach my $target (sort {$a cmp $b} keys %Target) {
         print "    $target: $Target{$target} Time(s)\n";
      }
   } else {
      print "\n";
   }
   print "\n";
}

if (keys %Started && $Detail > 3) {
   print "Started:\n";
   foreach my $started (sort {$a cmp $b} keys %Started) {
      print "    $started: $Started{$started} Time(s)\n";
   }
   print "\n";
}

if (keys %Closed && $Detail > 3) {
   print "Closed:\n";
   foreach my $closed (sort {$a cmp $b} keys %Closed) {
      print "    $closed: $Closed{$closed} Time(s)\n";
   }
   print "\n";
}

if (keys %Activated && $Detail > 3) {
   print "Activated:\n";
   foreach my $item (sort {$a cmp $b} keys %Activated) {
      print "    $item: $Activated{$item} Time(s)\n";
   }
   print "\n";
}

if (keys %Slow && $Detail > 3) {
   print "Slow to start:\n";
   foreach my $pidfile (sort {$a cmp $b} keys %Slow) {
      print "    $pidfile: $Slow{$pidfile} Time(s)\n";
   }
   print "\n";
}

if (keys %Reloaded && $Detail > 5) {
   print "Reloaded:\n";
   foreach my $item (sort {$a cmp $b} keys %Reloaded) {
      print "    $item: $Reloaded{$item} Time(s)\n";
   }
   print "\n";
}

if (keys %Deactivated && $Detail > 5) {
   print "Deactivated:\n";
   foreach my $item (sort {$a cmp $b} keys %Deactivated) {
      print "    $item: $Deactivated{$item} Time(s)\n";
   }
   print "\n";
}

if ($TimeChanged && $Detail > 5) {
   print "Time Changed: $TimeChanged Time(s)\n\n";
}

if (keys %UserSession && $Detail > 3) {
   print "User Sessions:\n";
   foreach my $user (sort {$a cmp $b} keys %UserSession) {
      print "    $user: ";
      if ($Detail >= 10) {
         foreach my $session (sort {$a cmp $b} keys %{$UserSession{$user}}) {
            print " $session";
         }
         print "\n";
      } else {
         print scalar(keys %{$UserSession{$user}}) . ": Time(s)\n";
      }
      print "\n";
   }
}

if (keys %Slice && $Detail > 5) {
   print "Slices created:\n";
   foreach my $slice (sort {$a cmp $b} keys %Slice) {
      print "    $slice: $Slice{$slice} Time(s)\n";
   }
    print "\n";
}

if (keys %OtherList) {
   print "\n\n**Unmatched Entries**\n";
   foreach my $line (sort {$a cmp $b} keys %OtherList) {
      print "   $line: $OtherList{$line} Time(s)\n";
   }
}

exit(0);

# vi: shiftwidth=3 tabstop=3 syntax=perl et
# Local Variables:
# mode: perl
# perl-indent-level: 3
# indent-tabs-mode: nil
# End: