| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/monitor.cc |
| Date: | 2026-08-30 02:40:36 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 92 | 373 | 24.7% |
| Branches: | 62 | 554 | 11.2% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System. | ||
| 3 | * | ||
| 4 | * This module forks a watchdog process that listens on | ||
| 5 | * a pipe and prints a stackstrace into syslog, when cvmfs | ||
| 6 | * fails. | ||
| 7 | * | ||
| 8 | * Also, it handles getting and setting the maximum number of file descriptors. | ||
| 9 | */ | ||
| 10 | |||
| 11 | |||
| 12 | #include "monitor.h" | ||
| 13 | |||
| 14 | #include <errno.h> | ||
| 15 | #include <execinfo.h> | ||
| 16 | #include <poll.h> | ||
| 17 | #include <pthread.h> | ||
| 18 | #include <signal.h> | ||
| 19 | #include <sys/resource.h> | ||
| 20 | #include <sys/types.h> | ||
| 21 | #ifdef __APPLE__ | ||
| 22 | #include <sys/ucontext.h> | ||
| 23 | #else | ||
| 24 | #include <ucontext.h> | ||
| 25 | #endif | ||
| 26 | #include <sys/uio.h> | ||
| 27 | #include <sys/wait.h> | ||
| 28 | #include <syslog.h> | ||
| 29 | #include <time.h> | ||
| 30 | #include <unistd.h> | ||
| 31 | |||
| 32 | #include <cassert> | ||
| 33 | #include <cstdio> | ||
| 34 | #include <cstdlib> | ||
| 35 | #include <cstring> | ||
| 36 | #include <map> | ||
| 37 | #include <set> | ||
| 38 | #include <string> | ||
| 39 | #include <vector> | ||
| 40 | |||
| 41 | #if defined(CVMFS_FUSE_MODULE) | ||
| 42 | #include "cvmfs.h" | ||
| 43 | #endif | ||
| 44 | #include "util/capabilities.h" | ||
| 45 | #include "util/exception.h" | ||
| 46 | #include "util/logging.h" | ||
| 47 | #include "util/posix.h" | ||
| 48 | #include "util/smalloc.h" | ||
| 49 | #include "util/string.h" | ||
| 50 | |||
| 51 | // Used for address offset calculation | ||
| 52 | #if defined(CVMFS_FUSE_MODULE) | ||
| 53 | extern loader::CvmfsExports *g_cvmfs_exports; | ||
| 54 | #endif | ||
| 55 | |||
| 56 | using namespace std; // NOLINT | ||
| 57 | |||
| 58 | Watchdog *Watchdog::instance_ = NULL; | ||
| 59 | |||
| 60 | int Watchdog::g_suppressed_signals[] = { | ||
| 61 | SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGBUS, SIGFPE, | ||
| 62 | SIGUSR1, SIGSEGV, SIGUSR2, SIGTERM, SIGXCPU, SIGXFSZ}; | ||
| 63 | |||
| 64 | int Watchdog::g_crash_signals[] = {SIGQUIT, SIGILL, SIGABRT, SIGFPE, | ||
| 65 | SIGSEGV, SIGBUS, SIGPIPE, SIGXFSZ}; | ||
| 66 | |||
| 67 | 49 | Watchdog *Watchdog::Create(FnOnExit on_exit, | |
| 68 | bool needs_read_environ, | ||
| 69 | WatchdogState *saved_state) { | ||
| 70 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | assert(instance_ == NULL); |
| 71 |
1/2✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | instance_ = new Watchdog(on_exit); |
| 72 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (saved_state != NULL) |
| 73 | ✗ | instance_->RestoreState(saved_state); | |
| 74 | else | ||
| 75 | 49 | instance_->Fork(needs_read_environ); | |
| 76 | 49 | return instance_; | |
| 77 | } | ||
| 78 | |||
| 79 | |||
| 80 | /** | ||
| 81 | * Uses an external shell and gdb to create a full stack trace of the dying | ||
| 82 | * process. The same shell is used to force-quit the client afterwards. | ||
| 83 | */ | ||
| 84 | ✗ | string Watchdog::GenerateStackTrace(pid_t pid) { | |
| 85 | int retval; | ||
| 86 | ✗ | string result = ""; | |
| 87 | |||
| 88 | // Get capability to ptrace the dead main cvmfs2 process. | ||
| 89 | // This is often necessary because the main process can have its own | ||
| 90 | // elevated capability which would otherwise block ptrace. | ||
| 91 | ✗ | if (!ObtainSysPtraceCapability()) { | |
| 92 | ✗ | result += "failed to gain ptrace capability... still give it a try\n"; | |
| 93 | } | ||
| 94 | |||
| 95 | // run gdb and attach to the dying process | ||
| 96 | int fd_stdin; | ||
| 97 | int fd_stdout; | ||
| 98 | int fd_stderr; | ||
| 99 | ✗ | vector<string> argv; | |
| 100 | ✗ | argv.push_back("-p"); | |
| 101 | ✗ | argv.push_back(StringifyInt(pid)); | |
| 102 | ✗ | pid_t gdb_pid = 0; | |
| 103 | ✗ | const bool double_fork = false; | |
| 104 | ✗ | retval = ExecuteBinary(&fd_stdin, | |
| 105 | &fd_stdout, | ||
| 106 | &fd_stderr, | ||
| 107 | #ifdef __APPLE__ | ||
| 108 | "lldb", | ||
| 109 | #else | ||
| 110 | "gdb", | ||
| 111 | #endif | ||
| 112 | argv, | ||
| 113 | double_fork, | ||
| 114 | &gdb_pid); | ||
| 115 | ✗ | assert(retval); | |
| 116 | |||
| 117 | |||
| 118 | // Skip the gdb startup output | ||
| 119 | ✗ | ReadUntilGdbPrompt(fd_stdout); | |
| 120 | |||
| 121 | // Send stacktrace command to gdb | ||
| 122 | #ifdef __APPLE__ | ||
| 123 | const string gdb_cmd = "bt all\n" | ||
| 124 | "quit\n"; | ||
| 125 | #else | ||
| 126 | const string gdb_cmd = "thread apply all bt\n" | ||
| 127 | ✗ | "quit\n"; | |
| 128 | #endif | ||
| 129 | // The execve can have failed, which can't be detected in ExecuteBinary. | ||
| 130 | // Instead, writing to the pipe will fail. | ||
| 131 | ✗ | const ssize_t nbytes = write(fd_stdin, gdb_cmd.data(), gdb_cmd.length()); | |
| 132 | ✗ | if ((nbytes < 0) || (static_cast<unsigned>(nbytes) != gdb_cmd.length())) { | |
| 133 | ✗ | result += "failed to start gdb/lldb (" + StringifyInt(nbytes) | |
| 134 | ✗ | + " bytes " | |
| 135 | "written, errno " | ||
| 136 | ✗ | + StringifyInt(errno) + ")\n"; | |
| 137 | ✗ | return result; | |
| 138 | } | ||
| 139 | |||
| 140 | // Read the stack trace from the stdout of our gdb process | ||
| 141 | #ifdef __APPLE__ | ||
| 142 | // lldb has one more prompt | ||
| 143 | result += ReadUntilGdbPrompt(fd_stdout); | ||
| 144 | #endif | ||
| 145 | ✗ | result += ReadUntilGdbPrompt(fd_stdout) + "\n\n"; | |
| 146 | |||
| 147 | // Check for output on stderr | ||
| 148 | ✗ | string result_err; | |
| 149 | ✗ | Block2Nonblock(fd_stderr); | |
| 150 | char cbuf; | ||
| 151 | ✗ | while (read(fd_stderr, &cbuf, 1) == 1) | |
| 152 | ✗ | result_err.push_back(cbuf); | |
| 153 | ✗ | if (!result_err.empty()) | |
| 154 | ✗ | result += "\nError output:\n" + result_err + "\n"; | |
| 155 | |||
| 156 | // Close the connection to the terminated gdb process | ||
| 157 | ✗ | close(fd_stderr); | |
| 158 | ✗ | close(fd_stdout); | |
| 159 | ✗ | close(fd_stdin); | |
| 160 | |||
| 161 | // Make sure gdb has terminated (wait for it for a short while) | ||
| 162 | ✗ | unsigned int timeout = 15; | |
| 163 | int statloc; | ||
| 164 | ✗ | while (timeout > 0 && waitpid(gdb_pid, &statloc, WNOHANG) != gdb_pid) { | |
| 165 | ✗ | --timeout; | |
| 166 | ✗ | SafeSleepMs(1000); | |
| 167 | } | ||
| 168 | |||
| 169 | // when the timeout expired, gdb probably hangs... we need to kill it | ||
| 170 | ✗ | if (timeout == 0) { | |
| 171 | ✗ | result += "gdb did not exit as expected. sending SIGKILL... "; | |
| 172 | ✗ | result += (kill(gdb_pid, SIGKILL) != 0) ? "failed\n" : "okay\n"; | |
| 173 | } | ||
| 174 | |||
| 175 | ✗ | return result; | |
| 176 | } | ||
| 177 | |||
| 178 | |||
| 179 | ✗ | pid_t Watchdog::GetPid() { | |
| 180 | ✗ | if (instance_ != NULL) { | |
| 181 | ✗ | return instance_->watchdog_pid_; | |
| 182 | } | ||
| 183 | ✗ | return getpid(); | |
| 184 | } | ||
| 185 | |||
| 186 | /** | ||
| 187 | * Log a string to syslog and into the crash dump file. | ||
| 188 | * We expect ideally nothing to be logged, so that file is created on demand. | ||
| 189 | */ | ||
| 190 | ✗ | void Watchdog::LogEmergency(string msg) { | |
| 191 | char ctime_buffer[32]; | ||
| 192 | |||
| 193 | ✗ | if (!crash_dump_path_.empty()) { | |
| 194 | ✗ | FILE *fp = fopen(crash_dump_path_.c_str(), "a"); | |
| 195 | ✗ | if (fp) { | |
| 196 | ✗ | const time_t now = time(NULL); | |
| 197 | ✗ | msg += "\nTimestamp: " + string(ctime_r(&now, ctime_buffer)); | |
| 198 | ✗ | if (fwrite(&msg[0], 1, msg.length(), fp) != msg.length()) { | |
| 199 | ✗ | msg += " (failed to report into crash dump file " + crash_dump_path_ | |
| 200 | ✗ | + ")"; | |
| 201 | } else { | ||
| 202 | ✗ | msg += "\n Crash logged also on file: " + crash_dump_path_ + "\n"; | |
| 203 | } | ||
| 204 | ✗ | fclose(fp); | |
| 205 | } else { | ||
| 206 | ✗ | msg += " (failed to open crash dump file " + crash_dump_path_ + ")"; | |
| 207 | } | ||
| 208 | } | ||
| 209 | ✗ | LogCvmfs(kLogMonitor, kLogSyslogErr, "%s", msg.c_str()); | |
| 210 | } | ||
| 211 | |||
| 212 | /** | ||
| 213 | * Reads from the file descriptor until the specific gdb prompt is reached or | ||
| 214 | * the pipe gets broken. | ||
| 215 | * | ||
| 216 | * @param fd_pipe the file descriptor of the pipe to be read | ||
| 217 | * @return the data read from the pipe | ||
| 218 | */ | ||
| 219 | ✗ | string Watchdog::ReadUntilGdbPrompt(int fd_pipe) { | |
| 220 | #ifdef __APPLE__ | ||
| 221 | static const string gdb_prompt = "(lldb)"; | ||
| 222 | #else | ||
| 223 | ✗ | static const string gdb_prompt = "\n(gdb) "; | |
| 224 | #endif | ||
| 225 | |||
| 226 | ✗ | string result; | |
| 227 | char mini_buffer; | ||
| 228 | int chars_io; | ||
| 229 | ✗ | unsigned int ring_buffer_pos = 0; | |
| 230 | |||
| 231 | // read from stdout of gdb until gdb prompt occurs --> (gdb) | ||
| 232 | while (1) { | ||
| 233 | ✗ | chars_io = read(fd_pipe, &mini_buffer, 1); | |
| 234 | |||
| 235 | // in case something goes wrong... | ||
| 236 | ✗ | if (chars_io <= 0) | |
| 237 | ✗ | break; | |
| 238 | |||
| 239 | ✗ | result += mini_buffer; | |
| 240 | |||
| 241 | // find the gdb_promt in the stdout data | ||
| 242 | ✗ | if (mini_buffer == gdb_prompt[ring_buffer_pos]) { | |
| 243 | ✗ | ++ring_buffer_pos; | |
| 244 | ✗ | if (ring_buffer_pos == gdb_prompt.size()) { | |
| 245 | ✗ | break; | |
| 246 | } | ||
| 247 | } else { | ||
| 248 | ✗ | ring_buffer_pos = 0; | |
| 249 | } | ||
| 250 | } | ||
| 251 | |||
| 252 | ✗ | return result; | |
| 253 | } | ||
| 254 | |||
| 255 | |||
| 256 | /** | ||
| 257 | * Generates useful information from the backtrace log in the pipe. | ||
| 258 | */ | ||
| 259 | ✗ | string Watchdog::ReportStacktrace() { | |
| 260 | CrashData crash_data; | ||
| 261 | ✗ | if (!pipe_watchdog_->TryRead<CrashData>(&crash_data)) { | |
| 262 | ✗ | return "failed to read crash data (" + StringifyInt(errno) + ")"; | |
| 263 | } | ||
| 264 | |||
| 265 | ✗ | string debug = "--\n"; | |
| 266 | ✗ | debug += "Signal: " + StringifyInt(crash_data.signal); | |
| 267 | ✗ | debug += ", errno: " + StringifyInt(crash_data.sys_errno); | |
| 268 | ✗ | debug += ", version: " + string(CVMFS_VERSION); | |
| 269 | ✗ | debug += ", PID: " + StringifyInt(crash_data.pid) + "\n"; | |
| 270 | ✗ | debug += "Executable path: " + exe_path_ + "\n"; | |
| 271 | |||
| 272 | ✗ | debug += GenerateStackTrace(crash_data.pid); | |
| 273 | |||
| 274 | // Give the dying process the finishing stroke | ||
| 275 | ✗ | if (kill(crash_data.pid, SIGKILL) != 0) { | |
| 276 | ✗ | debug += "Failed to kill cvmfs client! ("; | |
| 277 | ✗ | switch (errno) { | |
| 278 | ✗ | case EINVAL: | |
| 279 | ✗ | debug += "invalid signal"; | |
| 280 | ✗ | break; | |
| 281 | ✗ | case EPERM: | |
| 282 | ✗ | debug += "permission denied"; | |
| 283 | ✗ | break; | |
| 284 | ✗ | case ESRCH: | |
| 285 | ✗ | debug += "no such process"; | |
| 286 | ✗ | break; | |
| 287 | ✗ | default: | |
| 288 | ✗ | debug += "unknown error " + StringifyInt(errno); | |
| 289 | } | ||
| 290 | ✗ | debug += ")\n\n"; | |
| 291 | } | ||
| 292 | |||
| 293 | ✗ | return debug; | |
| 294 | } | ||
| 295 | |||
| 296 | |||
| 297 | ✗ | void Watchdog::ReportSignalAndContinue(int sig, siginfo_t *siginfo, | |
| 298 | void * /* context */) { | ||
| 299 | ✗ | LogCvmfs(kLogMonitor, kLogSyslogErr, | |
| 300 | "watchdog: received unexpected signal %d from PID %d / UID %d", sig, | ||
| 301 | siginfo->si_pid, siginfo->si_uid); | ||
| 302 | } | ||
| 303 | |||
| 304 | |||
| 305 | ✗ | void Watchdog::SendTrace(int sig, siginfo_t *siginfo, void *context) { | |
| 306 | ✗ | const int send_errno = errno; | |
| 307 | ✗ | if (platform_spinlock_trylock(&Me()->lock_handler_) != 0) { | |
| 308 | // Concurrent call, wait for the first one to exit the process | ||
| 309 | ✗ | while (true) { | |
| 310 | } | ||
| 311 | } | ||
| 312 | |||
| 313 | // Set the original signal handler for the raised signal in | ||
| 314 | // SIGQUIT (watchdog process will raise SIGQUIT) | ||
| 315 | ✗ | (void)sigaction(SIGQUIT, &(Me()->old_signal_handlers_[sig]), NULL); | |
| 316 | |||
| 317 | // Inform the watchdog that CernVM-FS crashed | ||
| 318 | ✗ | if (!Me()->pipe_watchdog_->Write(ControlFlow::kProduceStacktrace)) { | |
| 319 | ✗ | _exit(1); | |
| 320 | } | ||
| 321 | |||
| 322 | // Send crash information to the watchdog | ||
| 323 | CrashData crash_data; | ||
| 324 | ✗ | crash_data.signal = sig; | |
| 325 | ✗ | crash_data.sys_errno = send_errno; | |
| 326 | ✗ | crash_data.pid = getpid(); | |
| 327 | ✗ | if (!Me()->pipe_watchdog_->Write<CrashData>(crash_data)) { | |
| 328 | ✗ | _exit(1); | |
| 329 | } | ||
| 330 | |||
| 331 | // Do not die before the stack trace was generated | ||
| 332 | // kill -SIGQUIT <pid> will finish this | ||
| 333 | ✗ | int counter = 0; | |
| 334 | while (true) { | ||
| 335 | ✗ | SafeSleepMs(100); | |
| 336 | // quit anyway after 30 seconds | ||
| 337 | ✗ | if (++counter == 300) { | |
| 338 | ✗ | LogCvmfs(kLogCvmfs, kLogSyslogErr, "stack trace generation failed"); | |
| 339 | // Last attempt to log something useful | ||
| 340 | #if defined(CVMFS_FUSE_MODULE) | ||
| 341 | ✗ | LogCvmfs(kLogCvmfs, kLogSyslogErr, "Signal %d, errno %d", sig, | |
| 342 | send_errno); | ||
| 343 | void *addr[kMaxBacktrace]; | ||
| 344 | // Note: this doesn't work due to the signal stack on OS X (it works on | ||
| 345 | // Linux). Since anyway lldb is supposed to produce the backtrace, we | ||
| 346 | // consider it more important to protect cvmfs against stack overflows. | ||
| 347 | ✗ | const int num_addr = backtrace(addr, kMaxBacktrace); | |
| 348 | ✗ | char **symbols = backtrace_symbols(addr, num_addr); | |
| 349 | ✗ | string backtrace = "Backtrace (" + StringifyInt(num_addr) | |
| 350 | ✗ | + " symbols):\n"; | |
| 351 | ✗ | for (int i = 0; i < num_addr; ++i) | |
| 352 | ✗ | backtrace += string(symbols[i]) + "\n"; | |
| 353 | ✗ | LogCvmfs(kLogCvmfs, kLogSyslogErr, "%s", backtrace.c_str()); | |
| 354 | ✗ | LogCvmfs(kLogCvmfs, kLogSyslogErr, "address of g_cvmfs_exports: %p", | |
| 355 | &g_cvmfs_exports); | ||
| 356 | #endif | ||
| 357 | |||
| 358 | ✗ | _exit(1); | |
| 359 | } | ||
| 360 | } | ||
| 361 | |||
| 362 | _exit(1); | ||
| 363 | } | ||
| 364 | |||
| 365 | |||
| 366 | /** | ||
| 367 | * Sets the signal handlers of the current process according to the ones | ||
| 368 | * defined in the given SigactionMap. | ||
| 369 | * | ||
| 370 | * @param signal_handlers a map of SIGNAL -> struct sigaction | ||
| 371 | * @return a SigactionMap containing the old handlers | ||
| 372 | */ | ||
| 373 | 49 | Watchdog::SigactionMap Watchdog::SetSignalHandlers( | |
| 374 | const SigactionMap &signal_handlers) { | ||
| 375 | 49 | SigactionMap old_signal_handlers; | |
| 376 | 49 | SigactionMap::const_iterator i = signal_handlers.begin(); | |
| 377 | 49 | const SigactionMap::const_iterator iend = signal_handlers.end(); | |
| 378 |
2/2✓ Branch 1 taken 637 times.
✓ Branch 2 taken 49 times.
|
686 | for (; i != iend; ++i) { |
| 379 | struct sigaction old_signal_handler; | ||
| 380 |
1/2✗ Branch 3 not taken.
✓ Branch 4 taken 637 times.
|
637 | if (sigaction(i->first, &i->second, &old_signal_handler) != 0) { |
| 381 | ✗ | PANIC(NULL); | |
| 382 | } | ||
| 383 |
1/2✓ Branch 2 taken 637 times.
✗ Branch 3 not taken.
|
637 | old_signal_handlers[i->first] = old_signal_handler; |
| 384 | } | ||
| 385 | |||
| 386 | 98 | return old_signal_handlers; | |
| 387 | } | ||
| 388 | |||
| 389 | |||
| 390 | /** | ||
| 391 | * Fork the watchdog process and put it on hold until Spawn() is called. | ||
| 392 | */ | ||
| 393 | 49 | void Watchdog::Fork(bool needs_read_environ) { | |
| 394 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | Pipe<kPipeWatchdogPid> pipe_pid; |
| 395 | 98 | pipe_watchdog_ = std::unique_ptr<Pipe<kPipeWatchdog> >( | |
| 396 |
2/4✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
|
98 | new Pipe<kPipeWatchdog>()); |
| 397 | 98 | pipe_listener_ = std::unique_ptr<Pipe<kPipeWatchdogSupervisor> >( | |
| 398 |
2/4✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
|
98 | new Pipe<kPipeWatchdogSupervisor>()); |
| 399 | |||
| 400 | pid_t pid; | ||
| 401 | int statloc; | ||
| 402 |
1/3✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 49 times.
|
49 | switch (pid = fork()) { |
| 403 | ✗ | case -1: | |
| 404 | ✗ | PANIC(NULL); | |
| 405 | ✗ | case 0: | |
| 406 | // Double fork to avoid zombie | ||
| 407 | ✗ | switch (fork()) { | |
| 408 | ✗ | case -1: | |
| 409 | ✗ | _exit(1); | |
| 410 | ✗ | case 0: { | |
| 411 | ✗ | pipe_watchdog_->CloseWriteFd(); | |
| 412 | ✗ | Daemonize(); | |
| 413 |
4/8✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 49 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 49 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 49 times.
|
49 | if ((geteuid() != 0) && SetuidCapabilityPermitted()) { |
| 414 | ✗ | const std::vector<cap_value_t> nocaps; | |
| 415 | ✗ | if (on_exit_) { | |
| 416 | // Reduce to minimum capabilities, which unfortunately is | ||
| 417 | // still quite powerful. | ||
| 418 | // CAP_SYS_ADMIN is needed to unmount, and CAP_SYS_PTRACE | ||
| 419 | // is needed when needs_read_environ is true because then | ||
| 420 | // the main Fuse process has elevated capabilities and | ||
| 421 | // ptrace (needed for collecting a stack trace) is not | ||
| 422 | // allowed on a process with more capabilities. | ||
| 423 | ✗ | if (needs_read_environ) { | |
| 424 | const std::vector<cap_value_t> reservecaps = {CAP_SYS_ADMIN, | ||
| 425 | ✗ | CAP_SYS_PTRACE}; | |
| 426 | ✗ | const std::vector<cap_value_t> inheritcaps = {CAP_SYS_PTRACE}; | |
| 427 | ✗ | if (!ClearPermittedCapabilities(reservecaps, inheritcaps)) | |
| 428 | ✗ | PANIC(kLogStderr | kLogSyslogErr, | |
| 429 | "Failed to reduce watchdog process capabilities"); | ||
| 430 | ✗ | } else { | |
| 431 | ✗ | const std::vector<cap_value_t> reservecaps = {CAP_SYS_ADMIN}; | |
| 432 | ✗ | if (!ClearPermittedCapabilities(reservecaps, nocaps)) | |
| 433 | ✗ | PANIC(kLogStderr | kLogSyslogErr, | |
| 434 | "Failed to reduce watchdog process capabilities"); | ||
| 435 | } | ||
| 436 | } else { | ||
| 437 | // Only need to be able to do the stack trace, and the | ||
| 438 | // main process needs no extra capabilities, so we can | ||
| 439 | // drop all capabilities. | ||
| 440 | ✗ | if (!ClearPermittedCapabilities(nocaps, nocaps)) | |
| 441 | ✗ | PANIC(kLogStderr | kLogSyslogErr, | |
| 442 | "Failed to clear watchdog process capabilities"); | ||
| 443 | } | ||
| 444 | } | ||
| 445 | // send the watchdog PID to the supervisee | ||
| 446 | 49 | const pid_t watchdog_pid = getpid(); | |
| 447 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | pipe_pid.Write(watchdog_pid); |
| 448 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | pipe_pid.CloseWriteFd(); |
| 449 | // Close all unused file descriptors | ||
| 450 | // close also usyslog, only get it back if necessary | ||
| 451 | // string usyslog_save = GetLogMicroSyslog(); | ||
| 452 |
1/3✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
49 | const string debuglog_save = GetLogDebugFile(); |
| 453 |
2/4✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 49 times.
✗ Branch 6 not taken.
|
49 | SetLogDebugFile(""); |
| 454 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | const string usyslog_save = GetLogMicroSyslog(); |
| 455 |
2/4✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 49 times.
✗ Branch 6 not taken.
|
49 | SetLogMicroSyslog(""); |
| 456 | // Gracefully close the syslog before closing all fds. The next call | ||
| 457 | // to syslog will reopen it. | ||
| 458 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | closelog(); |
| 459 | // Let's keep stdin, stdout, stderr open at /dev/null (daemonized) | ||
| 460 | // in order to prevent accidental outputs from messing with another | ||
| 461 | // file descriptor | ||
| 462 | 49 | std::set<int> preserve_fds; | |
| 463 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | preserve_fds.insert(0); |
| 464 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | preserve_fds.insert(1); |
| 465 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | preserve_fds.insert(2); |
| 466 |
1/2✓ Branch 3 taken 49 times.
✗ Branch 4 not taken.
|
49 | preserve_fds.insert(pipe_watchdog_->GetReadFd()); |
| 467 |
1/2✓ Branch 3 taken 49 times.
✗ Branch 4 not taken.
|
49 | preserve_fds.insert(pipe_listener_->GetWriteFd()); |
| 468 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | CloseAllFildes(preserve_fds); |
| 469 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | SetLogMicroSyslog(usyslog_save); // no-op if usyslog not used |
| 470 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | SetLogDebugFile(debuglog_save); // no-op if debug log not used |
| 471 | |||
| 472 |
2/4✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 49 times.
|
49 | if (WaitForSupervisee()) |
| 473 | ✗ | Supervise(); | |
| 474 | |||
| 475 |
1/2✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | pipe_watchdog_->CloseReadFd(); |
| 476 |
1/2✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | pipe_listener_->CloseWriteFd(); |
| 477 | 49 | exit(0); | |
| 478 | } | ||
| 479 | ✗ | default: | |
| 480 | ✗ | _exit(0); | |
| 481 | } | ||
| 482 | 49 | default: | |
| 483 |
1/2✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | pipe_watchdog_->CloseReadFd(); |
| 484 |
1/2✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | pipe_listener_->CloseWriteFd(); |
| 485 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | pipe_pid.CloseWriteFd(); |
| 486 |
2/4✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 49 times.
|
49 | if (waitpid(pid, &statloc, 0) != pid) |
| 487 | ✗ | PANIC(NULL); | |
| 488 |
2/4✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 49 times.
|
49 | if (!WIFEXITED(statloc) || WEXITSTATUS(statloc)) |
| 489 | ✗ | PANIC(NULL); | |
| 490 | } | ||
| 491 | |||
| 492 | // retrieve the watchdog PID from the pipe | ||
| 493 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | pipe_pid.Read(&watchdog_pid_); |
| 494 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | pipe_pid.CloseReadFd(); |
| 495 | 49 | } | |
| 496 | |||
| 497 | |||
| 498 | 49 | bool Watchdog::WaitForSupervisee() { | |
| 499 | // We want broken pipes not to raise a signal but handle the error in the | ||
| 500 | // read/write code | ||
| 501 | 49 | platform_sighandler_t rv_sig = signal(SIGPIPE, SIG_IGN); | |
| 502 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | assert(rv_sig != SIG_ERR); |
| 503 | |||
| 504 | // The watchdog is not supposed to receive signals. If it does, report it. | ||
| 505 | struct sigaction sa; | ||
| 506 | 49 | memset(&sa, 0, sizeof(sa)); | |
| 507 | 49 | sa.sa_sigaction = ReportSignalAndContinue; | |
| 508 | 49 | sa.sa_flags = SA_SIGINFO; | |
| 509 | 49 | sigfillset(&sa.sa_mask); | |
| 510 | |||
| 511 | 49 | SigactionMap signal_handlers; | |
| 512 |
2/2✓ Branch 0 taken 637 times.
✓ Branch 1 taken 49 times.
|
686 | for (size_t i = 0; i < sizeof(g_suppressed_signals) / sizeof(int); i++) { |
| 513 |
1/2✓ Branch 1 taken 637 times.
✗ Branch 2 not taken.
|
637 | signal_handlers[g_suppressed_signals[i]] = sa; |
| 514 | } | ||
| 515 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | SetSignalHandlers(signal_handlers); |
| 516 | |||
| 517 | 49 | ControlFlow::Flags control_flow = ControlFlow::kUnknown; | |
| 518 | |||
| 519 |
3/4✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 48 times.
✓ Branch 5 taken 1 times.
|
49 | if (!pipe_watchdog_->TryRead(&control_flow)) { |
| 520 |
1/2✓ Branch 1 taken 48 times.
✗ Branch 2 not taken.
|
48 | LogCvmfs(kLogMonitor, kLogDebug, "supervisee canceled watchdog"); |
| 521 | 48 | return false; | |
| 522 | } | ||
| 523 | |||
| 524 |
1/4✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
1 | switch (control_flow) { |
| 525 | 1 | case ControlFlow::kQuit: | |
| 526 | 1 | return false; | |
| 527 | ✗ | case ControlFlow::kQuitWithExit: | |
| 528 | ✗ | if (on_exit_) | |
| 529 | ✗ | on_exit_(false); | |
| 530 | ✗ | return false; | |
| 531 | ✗ | case ControlFlow::kSupervise: | |
| 532 | ✗ | break; | |
| 533 | ✗ | default: | |
| 534 | ✗ | LogEmergency("Internal error: invalid control flow"); | |
| 535 | ✗ | return false; | |
| 536 | } | ||
| 537 | |||
| 538 | size_t size; | ||
| 539 | ✗ | pipe_watchdog_->Read(&size); | |
| 540 | ✗ | crash_dump_path_.resize(size); | |
| 541 | ✗ | if (size > 0) { | |
| 542 | ✗ | pipe_watchdog_->Read(&crash_dump_path_[0], size); | |
| 543 | |||
| 544 | ✗ | const int retval = chdir(GetParentPath(crash_dump_path_).c_str()); | |
| 545 | ✗ | if (retval != 0) { | |
| 546 | ✗ | LogEmergency(std::string("Cannot change to crash dump directory: ") | |
| 547 | ✗ | + crash_dump_path_); | |
| 548 | ✗ | return false; | |
| 549 | } | ||
| 550 | ✗ | crash_dump_path_ = GetFileName(crash_dump_path_); | |
| 551 | } | ||
| 552 | ✗ | return true; | |
| 553 | 49 | } | |
| 554 | |||
| 555 | |||
| 556 | /** | ||
| 557 | * Set up the signal handling and kick off the supervision. | ||
| 558 | */ | ||
| 559 | ✗ | void Watchdog::Spawn(const std::string &crash_dump_path) { | |
| 560 | // lower restrictions for ptrace | ||
| 561 | ✗ | if (!platform_allow_ptrace(watchdog_pid_)) { | |
| 562 | ✗ | LogCvmfs(kLogMonitor, kLogSyslogWarn, | |
| 563 | "failed to allow ptrace() for watchdog (PID: %d). " | ||
| 564 | "Post crash stacktrace might not work", | ||
| 565 | watchdog_pid_); | ||
| 566 | } | ||
| 567 | |||
| 568 | // Extra stack for signal handlers | ||
| 569 | ✗ | const int stack_size = kSignalHandlerStacksize; // 2 MB | |
| 570 | ✗ | sighandler_stack_.ss_sp = smalloc(stack_size); | |
| 571 | ✗ | sighandler_stack_.ss_size = stack_size; | |
| 572 | ✗ | sighandler_stack_.ss_flags = 0; | |
| 573 | ✗ | if (sigaltstack(&sighandler_stack_, NULL) != 0) | |
| 574 | ✗ | PANIC(NULL); | |
| 575 | |||
| 576 | // define our crash signal handler | ||
| 577 | struct sigaction sa; | ||
| 578 | ✗ | memset(&sa, 0, sizeof(sa)); | |
| 579 | ✗ | sa.sa_sigaction = SendTrace; | |
| 580 | ✗ | sa.sa_flags = SA_SIGINFO | SA_ONSTACK; | |
| 581 | ✗ | sigfillset(&sa.sa_mask); | |
| 582 | |||
| 583 | ✗ | SigactionMap signal_handlers; | |
| 584 | ✗ | for (size_t i = 0; i < sizeof(g_crash_signals) / sizeof(int); i++) { | |
| 585 | ✗ | signal_handlers[g_crash_signals[i]] = sa; | |
| 586 | } | ||
| 587 | ✗ | old_signal_handlers_ = SetSignalHandlers(signal_handlers); | |
| 588 | |||
| 589 | ✗ | pipe_terminate_ = std::unique_ptr<Pipe<kPipeThreadTerminator> >( | |
| 590 | ✗ | new Pipe<kPipeThreadTerminator>()); | |
| 591 | ✗ | const int retval = pthread_create(&thread_listener_, NULL, | |
| 592 | MainWatchdogListener, this); | ||
| 593 | ✗ | assert(retval == 0); | |
| 594 | |||
| 595 | ✗ | if (spawned_) { | |
| 596 | // This happens after a reload, when the watchdog process is | ||
| 597 | // already running so we can exit here. | ||
| 598 | ✗ | return; | |
| 599 | } | ||
| 600 | |||
| 601 | ✗ | pipe_watchdog_->Write(ControlFlow::kSupervise); | |
| 602 | ✗ | const size_t path_size = crash_dump_path.size(); | |
| 603 | ✗ | pipe_watchdog_->Write(path_size); | |
| 604 | ✗ | if (path_size > 0) { | |
| 605 | ✗ | pipe_watchdog_->Write(crash_dump_path.data(), path_size); | |
| 606 | } | ||
| 607 | |||
| 608 | ✗ | spawned_ = true; | |
| 609 | } | ||
| 610 | |||
| 611 | |||
| 612 | ✗ | void *Watchdog::MainWatchdogListener(void *data) { | |
| 613 | ✗ | Watchdog *watchdog = static_cast<Watchdog *>(data); | |
| 614 | ✗ | LogCvmfs(kLogMonitor, kLogDebug, "starting watchdog listener"); | |
| 615 | |||
| 616 | ✗ | if ((getuid() != 0) && SetuidCapabilityPermitted()) { | |
| 617 | // Drop all capabilities, none are needed in the listener | ||
| 618 | ✗ | const std::vector<cap_value_t> nocaps; | |
| 619 | ✗ | if (!ClearPermittedCapabilities(nocaps, nocaps)) | |
| 620 | ✗ | PANIC(kLogStderr | kLogSyslogErr, | |
| 621 | "Failed to clear watchdog listener capabilities"); | ||
| 622 | } | ||
| 623 | |||
| 624 | struct pollfd watch_fds[2]; | ||
| 625 | ✗ | watch_fds[0].fd = watchdog->pipe_listener_->GetReadFd(); | |
| 626 | ✗ | watch_fds[0].events = 0; // Only check for POLL[ERR,HUP,NVAL] in revents | |
| 627 | ✗ | watch_fds[0].revents = 0; | |
| 628 | ✗ | watch_fds[1].fd = watchdog->pipe_terminate_->GetReadFd(); | |
| 629 | ✗ | watch_fds[1].events = POLLIN | POLLPRI; | |
| 630 | ✗ | watch_fds[1].revents = 0; | |
| 631 | while (true) { | ||
| 632 | ✗ | const int retval = poll(watch_fds, 2, -1); | |
| 633 | ✗ | if (retval < 0) { | |
| 634 | ✗ | continue; | |
| 635 | } | ||
| 636 | |||
| 637 | // Terminate I/O thread | ||
| 638 | ✗ | if (watch_fds[1].revents) | |
| 639 | ✗ | break; | |
| 640 | |||
| 641 | ✗ | if (watch_fds[0].revents) { | |
| 642 | ✗ | if ((watch_fds[0].revents & POLLERR) || (watch_fds[0].revents & POLLHUP) | |
| 643 | ✗ | || (watch_fds[0].revents & POLLNVAL)) { | |
| 644 | ✗ | LogCvmfs(kLogMonitor, kLogDebug | kLogSyslogErr, | |
| 645 | "watchdog disappeared, disabling stack trace reporting " | ||
| 646 | "(revents: %d / %d|%d|%d)", | ||
| 647 | ✗ | watch_fds[0].revents, POLLERR, POLLHUP, POLLNVAL); | |
| 648 | ✗ | watchdog->SetSignalHandlers(watchdog->old_signal_handlers_); | |
| 649 | ✗ | PANIC(kLogDebug | kLogSyslogErr, "watchdog disappeared, aborting"); | |
| 650 | } | ||
| 651 | ✗ | PANIC(NULL); | |
| 652 | } | ||
| 653 | } | ||
| 654 | |||
| 655 | ✗ | LogCvmfs(kLogMonitor, kLogDebug, "stopping watchdog listener"); | |
| 656 | ✗ | return NULL; | |
| 657 | } | ||
| 658 | |||
| 659 | |||
| 660 | ✗ | void Watchdog::Supervise() { | |
| 661 | ✗ | ControlFlow::Flags control_flow = ControlFlow::kUnknown; | |
| 662 | |||
| 663 | ✗ | if (!pipe_watchdog_->TryRead<ControlFlow::Flags>(&control_flow)) { | |
| 664 | ✗ | LogEmergency("watchdog: unexpected termination (" | |
| 665 | ✗ | + StringifyInt(control_flow) + ")"); | |
| 666 | ✗ | if (on_exit_) | |
| 667 | ✗ | on_exit_(true /* crashed */); | |
| 668 | } else { | ||
| 669 | ✗ | switch (control_flow) { | |
| 670 | ✗ | case ControlFlow::kProduceStacktrace: | |
| 671 | ✗ | LogEmergency(ReportStacktrace()); | |
| 672 | ✗ | if (on_exit_) | |
| 673 | ✗ | on_exit_(true /* crashed */); | |
| 674 | ✗ | break; | |
| 675 | |||
| 676 | ✗ | case ControlFlow::kQuitWithExit: | |
| 677 | ✗ | if (on_exit_) | |
| 678 | ✗ | on_exit_(false /* crashed */); | |
| 679 | ✗ | break; | |
| 680 | |||
| 681 | ✗ | case ControlFlow::kQuit: | |
| 682 | ✗ | break; | |
| 683 | |||
| 684 | ✗ | default: | |
| 685 | ✗ | LogEmergency("watchdog: unexpected error"); | |
| 686 | ✗ | break; | |
| 687 | } | ||
| 688 | } | ||
| 689 | } | ||
| 690 | |||
| 691 | |||
| 692 | /** | ||
| 693 | * Save the state of the watchdog listener thread before reload. | ||
| 694 | */ | ||
| 695 | ✗ | void Watchdog::SaveState(WatchdogState *saved_state) { | |
| 696 | ✗ | saved_state->spawned = spawned_; | |
| 697 | ✗ | saved_state->pid = watchdog_pid_; | |
| 698 | ✗ | if (spawned_) { | |
| 699 | ✗ | saved_state->watchdog_write_fd = pipe_watchdog_->GetWriteFd(); | |
| 700 | ✗ | saved_state->listener_read_fd = pipe_listener_->GetReadFd(); | |
| 701 | } | ||
| 702 | } | ||
| 703 | |||
| 704 | |||
| 705 | /** | ||
| 706 | * Restore the state of the watchdog listener reload | ||
| 707 | */ | ||
| 708 | ✗ | void Watchdog::RestoreState(WatchdogState *saved_state) { | |
| 709 | ✗ | watchdog_pid_ = saved_state->pid; | |
| 710 | ✗ | if (!saved_state->spawned) { | |
| 711 | ✗ | return; | |
| 712 | } | ||
| 713 | ✗ | pipe_watchdog_ = std::unique_ptr<Pipe<kPipeWatchdog> >( | |
| 714 | ✗ | new Pipe<kPipeWatchdog>(-1, saved_state->watchdog_write_fd)); | |
| 715 | ✗ | pipe_listener_ = std::unique_ptr<Pipe<kPipeWatchdogSupervisor> >( | |
| 716 | ✗ | new Pipe<kPipeWatchdogSupervisor>(saved_state->listener_read_fd, -1)); | |
| 717 | ✗ | spawned_ = true; | |
| 718 | } | ||
| 719 | |||
| 720 | |||
| 721 | 49 | Watchdog::Watchdog(FnOnExit on_exit) | |
| 722 | 49 | : spawned_(false) | |
| 723 | 49 | , maintenance_mode_(false) | |
| 724 |
1/2✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
|
49 | , exe_path_(string(platform_getexepath())) |
| 725 | 49 | , watchdog_pid_(0) | |
| 726 | 98 | , on_exit_(on_exit) { | |
| 727 | 49 | const int retval = platform_spinlock_init(&lock_handler_, 0); | |
| 728 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | assert(retval == 0); |
| 729 | 49 | memset(&sighandler_stack_, 0, sizeof(sighandler_stack_)); | |
| 730 | 49 | } | |
| 731 | |||
| 732 | |||
| 733 | 1 | Watchdog::~Watchdog() { | |
| 734 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
|
1 | if (spawned_) { |
| 735 | // Reset signal handlers | ||
| 736 | ✗ | signal(SIGQUIT, SIG_DFL); | |
| 737 | ✗ | signal(SIGILL, SIG_DFL); | |
| 738 | ✗ | signal(SIGABRT, SIG_DFL); | |
| 739 | ✗ | signal(SIGFPE, SIG_DFL); | |
| 740 | ✗ | signal(SIGSEGV, SIG_DFL); | |
| 741 | ✗ | signal(SIGBUS, SIG_DFL); | |
| 742 | ✗ | signal(SIGPIPE, SIG_DFL); | |
| 743 | ✗ | signal(SIGXFSZ, SIG_DFL); | |
| 744 | ✗ | free(sighandler_stack_.ss_sp); | |
| 745 | ✗ | sighandler_stack_.ss_size = 0; | |
| 746 | |||
| 747 | // The watchdog listener thread exits on any message received | ||
| 748 | ✗ | pipe_terminate_->Write(ControlFlow::kQuit); | |
| 749 | ✗ | pthread_join(thread_listener_, NULL); | |
| 750 | ✗ | pipe_terminate_->Close(); | |
| 751 | } | ||
| 752 | |||
| 753 |
1/2✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
|
1 | if (!maintenance_mode_) { |
| 754 | // Shutdown the watchdog except when doing a reload | ||
| 755 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
|
1 | if (on_exit_) { |
| 756 | ✗ | pipe_watchdog_->Write(ControlFlow::kQuitWithExit); | |
| 757 | } else { | ||
| 758 | 1 | pipe_watchdog_->Write(ControlFlow::kQuit); | |
| 759 | } | ||
| 760 | 1 | pipe_watchdog_->CloseWriteFd(); | |
| 761 | 1 | pipe_listener_->CloseReadFd(); | |
| 762 | } else { | ||
| 763 | // Release the references to the watchdog pipes without closing them | ||
| 764 | ✗ | pipe_watchdog_.release(); | |
| 765 | ✗ | pipe_listener_.release(); | |
| 766 | } | ||
| 767 | |||
| 768 | 1 | platform_spinlock_destroy(&lock_handler_); | |
| 769 | 1 | LogCvmfs(kLogMonitor, kLogDebug, "monitor stopped"); | |
| 770 | 1 | instance_ = NULL; | |
| 771 | 1 | } | |
| 772 |