Head

GCC Code Coverage Report

Directory:	cvmfs/		Exec	Total	Coverage
File:	cvmfs/monitor.cc	Lines:	16	229	7.0 %
Date:	2019-02-03 02:48:13	Branches:	3	105	2.9 %


/**
 * This file is part of the CernVM File System.
 *
 * This module forks a watchdog process that listens on
 * a pipe and prints a stackstrace into syslog, when cvmfs
 * fails.
 *
 * Also, it handles getting and setting the maximum number of file descriptors.
 */

#include "cvmfs_config.h"
#include "monitor.h"

#include <errno.h>
#include <execinfo.h>
#include <pthread.h>
#include <signal.h>
#include <sys/resource.h>
#include <sys/types.h>
#ifdef __APPLE__
  #include <sys/ucontext.h>
#else
  #include <ucontext.h>
#endif
#include <sys/uio.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <string>
#include <vector>

#ifndef CVMFS_LIBCVMFS
#include "cvmfs.h"
#endif
#include "logging.h"
#include "platform.h"
#include "smalloc.h"
#include "util/posix.h"
#include "util/string.h"

// Used for address offset calculation
#ifndef CVMFS_LIBCVMFS
extern loader::CvmfsExports *g_cvmfs_exports;
#endif

using namespace std;  // NOLINT

Watchdog *Watchdog::instance_ = NULL;


Watchdog *Watchdog::Create(const string &crash_dump_path) {
  assert(instance_ == NULL);
  instance_ = new Watchdog(crash_dump_path);
  return instance_;
}


/**
 * Uses an external shell and gdb to create a full stack trace of the dying
 * process. The same shell is used to force-quit the client afterwards.
 */
string Watchdog::GenerateStackTrace(const string &exe_path, pid_t pid) {
  int retval;
  string result = "";

  // re-gain root permissions to allow for ptrace of died cvmfs2 process
  const bool retrievable = true;
  if (!SwitchCredentials(0, getgid(), retrievable)) {
    result += "failed to re-gain root permissions... still give it a try\n";
  }

  // run gdb and attach to the dying process
  int fd_stdin;
  int fd_stdout;
  int fd_stderr;
  vector<string> argv;
#ifndef __APPLE__
  argv.push_back("-q");
  argv.push_back("-n");
  argv.push_back(exe_path);
#else
  argv.push_back("-p");
#endif
  argv.push_back(StringifyInt(pid));
  pid_t gdb_pid = 0;
  const bool double_fork = false;
  retval = ExecuteBinary(&fd_stdin,
                         &fd_stdout,
                         &fd_stderr,
#ifdef __APPLE__
                          "lldb",
#else
                          "gdb",
#endif
                          argv,
                          double_fork,
                         &gdb_pid);
  assert(retval);


  // Skip the gdb startup output
  ReadUntilGdbPrompt(fd_stdout);

  // Send stacktrace command to gdb
#ifdef __APPLE__
  const string gdb_cmd = "bt all\n" "quit\n";
#else
  const string gdb_cmd = "thread apply all bt\n" "quit\n";
#endif
  // The execve can have failed, which can't be detected in ExecuteBinary.
  // Instead, writing to the pipe will fail.
  ssize_t nbytes = write(fd_stdin, gdb_cmd.data(), gdb_cmd.length());
  if ((nbytes < 0) || (static_cast<unsigned>(nbytes) != gdb_cmd.length())) {
    result += "failed to start gdb/lldb (" + StringifyInt(nbytes) + " bytes "
              "written, errno " + StringifyInt(errno) + ")\n";
    return result;
  }

  // Read the stack trace from the stdout of our gdb process
#ifdef __APPLE__
  // lldb has one more prompt
  result += ReadUntilGdbPrompt(fd_stdout);
#endif
  result += ReadUntilGdbPrompt(fd_stdout) + "\n\n";

  // Close the connection to the terminated gdb process
  close(fd_stderr);
  close(fd_stdout);
  close(fd_stdin);

  // Make sure gdb has terminated (wait for it for a short while)
  unsigned int timeout = 15;
  int statloc;
  while (timeout > 0 && waitpid(gdb_pid, &statloc, WNOHANG) != gdb_pid) {
    --timeout;
    SafeSleepMs(1000);
  }

  // when the timeout expired, gdb probably hangs... we need to kill it
  if (timeout == 0) {
    result += "gdb did not exit as expected. sending SIGKILL... ";
    result += (kill(gdb_pid, SIGKILL) != 0) ? "failed\n" : "okay\n";
  }

  return result;
}


pid_t Watchdog::GetPid() {
  if (instance_ != NULL) {
    if (!instance_->spawned_)
      return getpid();
    else
      return instance_->watchdog_pid_;
  }
  return getpid();
}


/**
 * Log a string to syslog and into the crash dump file.
 * We expect ideally nothing to be logged, so that file is created on demand.
 */
void Watchdog::LogEmergency(string msg) {
  char ctime_buffer[32];

  if (!crash_dump_path_.empty()) {
    FILE *fp = fopen(crash_dump_path_.c_str(), "a");
    if (fp) {
      time_t now = time(NULL);
      msg += "\nTimestamp: " + string(ctime_r(&now, ctime_buffer));
      if (fwrite(&msg[0], 1, msg.length(), fp) != msg.length())
        msg += " (failed to report into crash dump file "
               + crash_dump_path_ + ")";
      fclose(fp);
    } else {
      msg += " (failed to open crash dump file " + crash_dump_path_ + ")";
    }
  }
  LogCvmfs(kLogMonitor, kLogSyslogErr, "%s", msg.c_str());
}


/**
 * Reads from the file descriptor until the specific gdb prompt is reached or
 * the pipe gets broken.
 *
 * @param fd_pipe  the file descriptor of the pipe to be read
 * @return         the data read from the pipe
 */
string Watchdog::ReadUntilGdbPrompt(int fd_pipe) {
#ifdef __APPLE__
  static const string gdb_prompt = "(lldb)";
#else
  static const string gdb_prompt = "\n(gdb) ";
#endif

  string        result;
  char          mini_buffer;
  int           chars_io;
  unsigned int  ring_buffer_pos = 0;

  // read from stdout of gdb until gdb prompt occures --> (gdb)
  while (1) {
    chars_io = read(fd_pipe, &mini_buffer, 1);

    // in case something goes wrong...
    if (chars_io <= 0) break;

    result += mini_buffer;

    // find the gdb_promt in the stdout data
    if (mini_buffer == gdb_prompt[ring_buffer_pos]) {
      ++ring_buffer_pos;
      if (ring_buffer_pos == gdb_prompt.size()) {
        break;
      }
    } else {
      ring_buffer_pos = 0;
    }
  }

  return result;
}


void Watchdog::RegisterOnCrash(void (*CleanupOnCrash)(void)) {
  on_crash_ = CleanupOnCrash;
}


/**
 * Generates useful information from the backtrace log in the pipe.
 */
string Watchdog::ReportStacktrace() {
  // Re-activate ��Syslog, if necessary
  SetLogMicroSyslog(GetLogMicroSyslog());

  CrashData crash_data;
  if (!pipe_watchdog_->Read(&crash_data)) {
    return "failed to read crash data (" + StringifyInt(errno) + ")";
  }

  string debug = "--\n";
  debug += "Signal: "    + StringifyInt(crash_data.signal);
  debug += ", errno: "   + StringifyInt(crash_data.sys_errno);
  debug += ", version: " + string(VERSION);
  debug += ", PID: "     + StringifyInt(crash_data.pid) + "\n";
  debug += "Executable path: " + exe_path_ + "\n";

  debug += GenerateStackTrace(exe_path_, crash_data.pid);

  // Give the dying process the finishing stroke
  if (kill(crash_data.pid, SIGKILL) != 0) {
    debug += "Failed to kill cvmfs client! (";
    switch (errno) {
      case EINVAL:
        debug += "invalid signal";
        break;
      case EPERM:
        debug += "permission denied";
        break;
      case ESRCH:
        debug += "no such process";
        break;
      default:
        debug += "unknown error " + StringifyInt(errno);
    }
    debug += ")\n\n";
  }

  return debug;
}


void Watchdog::SendTrace(int sig, siginfo_t *siginfo, void *context) {
  int send_errno = errno;
  if (platform_spinlock_trylock(&Me()->lock_handler_) != 0) {
    // Concurrent call, wait for the first one to exit the process
    while (true) {}
  }

  // Set the original signal handler for the raised signal in
  // SIGQUIT (watchdog process will raise SIGQUIT)
  (void) sigaction(SIGQUIT, &(Me()->old_signal_handlers_[sig]), NULL);

  // Inform the watchdog that CernVM-FS crashed
  if (!Me()->pipe_watchdog_->Write(ControlFlow::kProduceStacktrace)) {
    _exit(1);
  }

  // Send crash information to the watchdog
  CrashData crash_data;
  crash_data.signal     = sig;
  crash_data.sys_errno  = send_errno;
  crash_data.pid        = getpid();
  if (!Me()->pipe_watchdog_->Write(crash_data)) {
    _exit(1);
  }

  // Do not die before the stack trace was generated
  // kill -SIGQUIT <pid> will finish this
  int counter = 0;
  while (true) {
    SafeSleepMs(100);
    // quit anyway after 30 seconds
    if (++counter == 300) {
      LogCvmfs(kLogCvmfs, kLogSyslogErr, "stack trace generation failed");
      // Last attempt to log something useful
#ifndef CVMFS_LIBCVMFS
      LogCvmfs(kLogCvmfs, kLogSyslogErr, "Signal %d, errno %d",
               sig, send_errno);
      void *addr[kMaxBacktrace];
      // Note: this doesn't work due to the signal stack on OS X (it works on
      // Linux).  Since anyway lldb is supposed to produce the backtrace, we
      // consider it more important to protect cvmfs against stack overflows.
      int num_addr = backtrace(addr, kMaxBacktrace);
      char **symbols = backtrace_symbols(addr, num_addr);
      string backtrace = "Backtrace (" + StringifyInt(num_addr) +
                         " symbols):\n";
      for (int i = 0; i < num_addr; ++i)
        backtrace += string(symbols[i]) + "\n";
      LogCvmfs(kLogCvmfs, kLogSyslogErr, "%s", backtrace.c_str());
      LogCvmfs(kLogCvmfs, kLogSyslogErr, "address of g_cvmfs_exports: %p",
               &g_cvmfs_exports);
#endif

      _exit(1);
    }
  }

  _exit(1);
}


/**
 * Sets the signal handlers of the current process according to the ones
 * defined in the given SigactionMap.
 *
 * @param signal_handlers  a map of SIGNAL -> struct sigaction
 * @return                 a SigactionMap containing the old handlers
 */
Watchdog::SigactionMap Watchdog::SetSignalHandlers(
  const SigactionMap &signal_handlers)
{
  SigactionMap old_signal_handlers;
  SigactionMap::const_iterator i     = signal_handlers.begin();
  SigactionMap::const_iterator iend  = signal_handlers.end();
  for (; i != iend; ++i) {
    struct sigaction old_signal_handler;
    if (sigaction(i->first, &i->second, &old_signal_handler) != 0) {
      abort();
    }
    old_signal_handlers[i->first] = old_signal_handler;
  }

  return old_signal_handlers;
}


/**
 * Fork the watchdog process.
 */
void Watchdog::Spawn() {
  Pipe pipe_pid;
  pipe_watchdog_ = new Pipe();

  pid_t pid;
  int statloc;
  int max_fd = sysconf(_SC_OPEN_MAX);
  assert(max_fd >= 0);
  switch (pid = fork()) {
    case -1: abort();
    case 0:
      // Double fork to avoid zombie
      switch (fork()) {
        case -1: exit(1);
        case 0: {
          close(pipe_watchdog_->write_end);
          Daemonize();
          // send the watchdog PID to cvmfs
          pid_t watchdog_pid = getpid();
          pipe_pid.Write(watchdog_pid);
          close(pipe_pid.write_end);
          // Close all unused file descriptors
          // close also usyslog, only get it back if necessary
          // string usyslog_save = GetLogMicroSyslog();
          string debuglog_save = GetLogDebugFile();
          // SetLogMicroSyslog("");
          SetLogDebugFile("");
          for (int fd = 0; fd < max_fd; fd++) {
            if (fd != pipe_watchdog_->read_end)
              close(fd);
          }
          // SetLogMicroSyslog(usyslog_save);  // no-op if usyslog not used
          SetLogDebugFile(debuglog_save);  // no-op if debug log not used
          Supervise();
          exit(0);
        }
        default:
          exit(0);
      }
    default:
      close(pipe_watchdog_->read_end);
      if (waitpid(pid, &statloc, 0) != pid) abort();
      if (!WIFEXITED(statloc) || WEXITSTATUS(statloc)) abort();
  }

  // retrieve the watchdog PID from the pipe
  close(pipe_pid.write_end);
  pipe_pid.Read(&watchdog_pid_);
  close(pipe_pid.read_end);

  // lower restrictions for ptrace
  if (!platform_allow_ptrace(watchdog_pid_)) {
    LogCvmfs(kLogMonitor, kLogSyslogWarn,
             "failed to allow ptrace() for watchdog (PID: %d). "
             "Post crash stacktrace might not work",
             watchdog_pid_);
  }

  // Extra stack for signal handlers
  int stack_size = kSignalHandlerStacksize;  // 2 MB
  sighandler_stack_.ss_sp = smalloc(stack_size);
  sighandler_stack_.ss_size = stack_size;
  sighandler_stack_.ss_flags = 0;
  if (sigaltstack(&sighandler_stack_, NULL) != 0)
    abort();

  // define our crash signal handler
  struct sigaction sa;
  memset(&sa, 0, sizeof(sa));
  sa.sa_sigaction = SendTrace;
  sa.sa_flags = SA_SIGINFO | SA_ONSTACK;
  sigfillset(&sa.sa_mask);

  SigactionMap signal_handlers;
  signal_handlers[SIGQUIT] = sa;
  signal_handlers[SIGILL]  = sa;
  signal_handlers[SIGABRT] = sa;
  signal_handlers[SIGFPE]  = sa;
  signal_handlers[SIGSEGV] = sa;
  signal_handlers[SIGBUS]  = sa;
  signal_handlers[SIGPIPE] = sa;
  signal_handlers[SIGXFSZ] = sa;
  old_signal_handlers_ = SetSignalHandlers(signal_handlers);

  spawned_ = true;
}


void Watchdog::Supervise() {
  signal(SIGPIPE, SIG_IGN);
  ControlFlow::Flags control_flow;

  if (!pipe_watchdog_->Read(&control_flow)) {
    // Re-activate ��Syslog, if necessary
    SetLogMicroSyslog(GetLogMicroSyslog());
    LogEmergency("unexpected termination (" + StringifyInt(control_flow) + ")");
    if (on_crash_) on_crash_();
  } else {
    switch (control_flow) {
      case ControlFlow::kProduceStacktrace:
        LogEmergency(ReportStacktrace());
        if (on_crash_) on_crash_();
        break;

      case ControlFlow::kQuit:
        break;

      default:
        // Re-activate ��Syslog, if necessary
        SetLogMicroSyslog(GetLogMicroSyslog());
        LogEmergency("unexpected error");
        break;
    }
  }

  close(pipe_watchdog_->read_end);
}


Watchdog::Watchdog(const string &crash_dump_path)
  : spawned_(false)
  , crash_dump_path_(crash_dump_path)
  , exe_path_(string(platform_getexepath()))
  , watchdog_pid_(0)
  , pipe_watchdog_(NULL)
  , on_crash_(NULL)
{
  int retval = platform_spinlock_init(&lock_handler_, 0);
  assert(retval == 0);
  memset(&sighandler_stack_, 0, sizeof(sighandler_stack_));
}


Watchdog::~Watchdog() {
  if (spawned_) {
    // Reset signal handlers
    signal(SIGQUIT, SIG_DFL);
    signal(SIGILL, SIG_DFL);
    signal(SIGABRT, SIG_DFL);
    signal(SIGFPE, SIG_DFL);
    signal(SIGSEGV, SIG_DFL);
    signal(SIGBUS, SIG_DFL);
    signal(SIGPIPE, SIG_DFL);
    signal(SIGXFSZ, SIG_DFL);
    free(sighandler_stack_.ss_sp);
    sighandler_stack_.ss_size = 0;

    pipe_watchdog_->Write(ControlFlow::kQuit);
    close(pipe_watchdog_->write_end);
  }

  platform_spinlock_destroy(&lock_handler_);
  LogCvmfs(kLogMonitor, kLogDebug, "monitor stopped");
  instance_ = NULL;
}



namespace monitor {

const unsigned kMinOpenFiles = 8192;

unsigned GetMaxOpenFiles() {
  static unsigned max_open_files;
  static bool     already_done = false;

  /* check number of open files (lazy evaluation) */
  if (!already_done) {
    unsigned int soft_limit = 0;
    int hard_limit = 0;

    struct rlimit rpl;
    memset(&rpl, 0, sizeof(rpl));
    getrlimit(RLIMIT_NOFILE, &rpl);
    soft_limit = rpl.rlim_cur;

#ifdef __APPLE__
    hard_limit = sysconf(_SC_OPEN_MAX);
    if (hard_limit < 0) {
      LogCvmfs(kLogMonitor, kLogStdout, "Warning: could not retrieve "
               "hard limit for the number of open files");
    }
#else
    hard_limit = rpl.rlim_max;
#endif

    if (soft_limit < kMinOpenFiles) {
      LogCvmfs(kLogMonitor, kLogSyslogWarn | kLogDebug,
               "Warning: current limits for number of open files are "
               "(%lu/%lu)\n"
               "CernVM-FS is likely to run out of file descriptors, "
               "set ulimit -n to at least %lu",
               soft_limit, hard_limit, kMinOpenFiles);
    }
    max_open_files = soft_limit;
    already_done   = true;
  }

  return max_open_files;
}

}  // namespace monitor


Generated by: GCOVR (Version 4.1)

Line	Branch	Exec	Source
1			/**
2			* This file is part of the CernVM File System.
3			*
4			* This module forks a watchdog process that listens on
5			* a pipe and prints a stackstrace into syslog, when cvmfs
6			* fails.
7			*
8			* Also, it handles getting and setting the maximum number of file descriptors.
9			*/
10
11			#include "cvmfs_config.h"
12			#include "monitor.h"
13
14			#include <errno.h>
15			#include <execinfo.h>
16			#include <pthread.h>
17			#include <signal.h>
18			#include <sys/resource.h>
19			#include <sys/types.h>
20			#ifdef __APPLE__
21			#include <sys/ucontext.h>
22			#else
23			#include <ucontext.h>
24			#endif
25			#include <sys/uio.h>
26			#include <sys/wait.h>
27			#include <time.h>
28			#include <unistd.h>
29
30			#include <cassert>
31			#include <cstdio>
32			#include <cstdlib>
33			#include <cstring>
34			#include <map>
35			#include <string>
36			#include <vector>
37
38			#ifndef CVMFS_LIBCVMFS
39			#include "cvmfs.h"
40			#endif
41			#include "logging.h"
42			#include "platform.h"
43			#include "smalloc.h"
44			#include "util/posix.h"
45			#include "util/string.h"
46
47			// Used for address offset calculation
48			#ifndef CVMFS_LIBCVMFS
49			extern loader::CvmfsExports *g_cvmfs_exports;
50			#endif
51
52			using namespace std; // NOLINT
53
54			Watchdog *Watchdog::instance_ = NULL;
55
56
57		15	Watchdog *Watchdog::Create(const string &crash_dump_path) {
58	✗✓	15	assert(instance_ == NULL);
59		15	instance_ = new Watchdog(crash_dump_path);
60		15	return instance_;
61			}
62
63
64			/**
65			* Uses an external shell and gdb to create a full stack trace of the dying
66			* process. The same shell is used to force-quit the client afterwards.
67			*/
68			string Watchdog::GenerateStackTrace(const string &exe_path, pid_t pid) {
69			int retval;
70			string result = "";
71
72			// re-gain root permissions to allow for ptrace of died cvmfs2 process
73			const bool retrievable = true;
74			if (!SwitchCredentials(0, getgid(), retrievable)) {
75			result += "failed to re-gain root permissions... still give it a try\n";
76			}
77
78			// run gdb and attach to the dying process
79			int fd_stdin;
80			int fd_stdout;
81			int fd_stderr;
82			vector<string> argv;
83			#ifndef __APPLE__
84			argv.push_back("-q");
85			argv.push_back("-n");
86			argv.push_back(exe_path);
87			#else
88			argv.push_back("-p");
89			#endif
90			argv.push_back(StringifyInt(pid));
91			pid_t gdb_pid = 0;
92			const bool double_fork = false;
93			retval = ExecuteBinary(&fd_stdin,
94			&fd_stdout,
95			&fd_stderr,
96			#ifdef __APPLE__
97			"lldb",
98			#else
99			"gdb",
100			#endif
101			argv,
102			double_fork,
103			&gdb_pid);
104			assert(retval);
105
106
107			// Skip the gdb startup output
108			ReadUntilGdbPrompt(fd_stdout);
109
110			// Send stacktrace command to gdb
111			#ifdef __APPLE__
112			const string gdb_cmd = "bt all\n" "quit\n";
113			#else
114			const string gdb_cmd = "thread apply all bt\n" "quit\n";
115			#endif
116			// The execve can have failed, which can't be detected in ExecuteBinary.
117			// Instead, writing to the pipe will fail.
118			ssize_t nbytes = write(fd_stdin, gdb_cmd.data(), gdb_cmd.length());
119			if ((nbytes < 0) \|\| (static_cast<unsigned>(nbytes) != gdb_cmd.length())) {
120			result += "failed to start gdb/lldb (" + StringifyInt(nbytes) + " bytes "
121			"written, errno " + StringifyInt(errno) + ")\n";
122			return result;
123			}
124
125			// Read the stack trace from the stdout of our gdb process
126			#ifdef __APPLE__
127			// lldb has one more prompt
128			result += ReadUntilGdbPrompt(fd_stdout);
129			#endif
130			result += ReadUntilGdbPrompt(fd_stdout) + "\n\n";
131
132			// Close the connection to the terminated gdb process
133			close(fd_stderr);
134			close(fd_stdout);
135			close(fd_stdin);
136
137			// Make sure gdb has terminated (wait for it for a short while)
138			unsigned int timeout = 15;
139			int statloc;
140			while (timeout > 0 && waitpid(gdb_pid, &statloc, WNOHANG) != gdb_pid) {
141			--timeout;
142			SafeSleepMs(1000);
143			}
144
145			// when the timeout expired, gdb probably hangs... we need to kill it
146			if (timeout == 0) {
147			result += "gdb did not exit as expected. sending SIGKILL... ";
148			result += (kill(gdb_pid, SIGKILL) != 0) ? "failed\n" : "okay\n";
149			}
150
151			return result;
152			}
153
154
155			pid_t Watchdog::GetPid() {
156			if (instance_ != NULL) {
157			if (!instance_->spawned_)
158			return getpid();
159			else
160			return instance_->watchdog_pid_;
161			}
162			return getpid();
163			}
164
165
166			/**
167			* Log a string to syslog and into the crash dump file.
168			* We expect ideally nothing to be logged, so that file is created on demand.
169			*/
170			void Watchdog::LogEmergency(string msg) {
171			char ctime_buffer[32];
172
173			if (!crash_dump_path_.empty()) {
174			FILE *fp = fopen(crash_dump_path_.c_str(), "a");
175			if (fp) {
176			time_t now = time(NULL);
177			msg += "\nTimestamp: " + string(ctime_r(&now, ctime_buffer));
178			if (fwrite(&msg[0], 1, msg.length(), fp) != msg.length())
179			msg += " (failed to report into crash dump file "
180			+ crash_dump_path_ + ")";
181			fclose(fp);
182			} else {
183			msg += " (failed to open crash dump file " + crash_dump_path_ + ")";
184			}
185			}
186			LogCvmfs(kLogMonitor, kLogSyslogErr, "%s", msg.c_str());
187			}
188
189
190			/**
191			* Reads from the file descriptor until the specific gdb prompt is reached or
192			* the pipe gets broken.
193			*
194			* @param fd_pipe the file descriptor of the pipe to be read
195			* @return the data read from the pipe
196			*/
197			string Watchdog::ReadUntilGdbPrompt(int fd_pipe) {
198			#ifdef __APPLE__
199			static const string gdb_prompt = "(lldb)";
200			#else
201			static const string gdb_prompt = "\n(gdb) ";
202			#endif
203
204			string result;
205			char mini_buffer;
206			int chars_io;
207			unsigned int ring_buffer_pos = 0;
208
209			// read from stdout of gdb until gdb prompt occures --> (gdb)
210			while (1) {
211			chars_io = read(fd_pipe, &mini_buffer, 1);
212
213			// in case something goes wrong...
214			if (chars_io <= 0) break;
215
216			result += mini_buffer;
217
218			// find the gdb_promt in the stdout data
219			if (mini_buffer == gdb_prompt[ring_buffer_pos]) {
220			++ring_buffer_pos;
221			if (ring_buffer_pos == gdb_prompt.size()) {
222			break;
223			}
224			} else {
225			ring_buffer_pos = 0;
226			}
227			}
228
229			return result;
230			}
231
232
233			void Watchdog::RegisterOnCrash(void (*CleanupOnCrash)(void)) {
234			on_crash_ = CleanupOnCrash;
235			}
236
237
238			/**
239			* Generates useful information from the backtrace log in the pipe.
240			*/
241			string Watchdog::ReportStacktrace() {
242			// Re-activate ��Syslog, if necessary
243			SetLogMicroSyslog(GetLogMicroSyslog());
244
245			CrashData crash_data;
246			if (!pipe_watchdog_->Read(&crash_data)) {
247			return "failed to read crash data (" + StringifyInt(errno) + ")";
248			}
249
250			string debug = "--\n";
251			debug += "Signal: " + StringifyInt(crash_data.signal);
252			debug += ", errno: " + StringifyInt(crash_data.sys_errno);
253			debug += ", version: " + string(VERSION);
254			debug += ", PID: " + StringifyInt(crash_data.pid) + "\n";
255			debug += "Executable path: " + exe_path_ + "\n";
256
257			debug += GenerateStackTrace(exe_path_, crash_data.pid);
258
259			// Give the dying process the finishing stroke
260			if (kill(crash_data.pid, SIGKILL) != 0) {
261			debug += "Failed to kill cvmfs client! (";
262			switch (errno) {
263			case EINVAL:
264			debug += "invalid signal";
265			break;
266			case EPERM:
267			debug += "permission denied";
268			break;
269			case ESRCH:
270			debug += "no such process";
271			break;
272			default:
273			debug += "unknown error " + StringifyInt(errno);
274			}
275			debug += ")\n\n";
276			}
277
278			return debug;
279			}
280
281
282			void Watchdog::SendTrace(int sig, siginfo_t siginfo, void context) {
283			int send_errno = errno;
284			if (platform_spinlock_trylock(&Me()->lock_handler_) != 0) {
285			// Concurrent call, wait for the first one to exit the process
286			while (true) {}
287			}
288
289			// Set the original signal handler for the raised signal in
290			// SIGQUIT (watchdog process will raise SIGQUIT)
291			(void) sigaction(SIGQUIT, &(Me()->old_signal_handlers_[sig]), NULL);
292
293			// Inform the watchdog that CernVM-FS crashed
294			if (!Me()->pipe_watchdog_->Write(ControlFlow::kProduceStacktrace)) {
295			_exit(1);
296			}
297
298			// Send crash information to the watchdog
299			CrashData crash_data;
300			crash_data.signal = sig;
301			crash_data.sys_errno = send_errno;
302			crash_data.pid = getpid();
303			if (!Me()->pipe_watchdog_->Write(crash_data)) {
304			_exit(1);
305			}
306
307			// Do not die before the stack trace was generated
308			// kill -SIGQUIT <pid> will finish this
309			int counter = 0;
310			while (true) {
311			SafeSleepMs(100);
312			// quit anyway after 30 seconds
313			if (++counter == 300) {
314			LogCvmfs(kLogCvmfs, kLogSyslogErr, "stack trace generation failed");
315			// Last attempt to log something useful
316			#ifndef CVMFS_LIBCVMFS
317			LogCvmfs(kLogCvmfs, kLogSyslogErr, "Signal %d, errno %d",
318			sig, send_errno);
319			void *addr[kMaxBacktrace];
320			// Note: this doesn't work due to the signal stack on OS X (it works on
321			// Linux). Since anyway lldb is supposed to produce the backtrace, we
322			// consider it more important to protect cvmfs against stack overflows.
323			int num_addr = backtrace(addr, kMaxBacktrace);
324			char **symbols = backtrace_symbols(addr, num_addr);
325			string backtrace = "Backtrace (" + StringifyInt(num_addr) +
326			" symbols):\n";
327			for (int i = 0; i < num_addr; ++i)
328			backtrace += string(symbols[i]) + "\n";
329			LogCvmfs(kLogCvmfs, kLogSyslogErr, "%s", backtrace.c_str());
330			LogCvmfs(kLogCvmfs, kLogSyslogErr, "address of g_cvmfs_exports: %p",
331			&g_cvmfs_exports);
332			#endif
333
334			_exit(1);
335			}
336			}
337
338			_exit(1);
339			}
340
341
342			/**
343			* Sets the signal handlers of the current process according to the ones
344			* defined in the given SigactionMap.
345			*
346			* @param signal_handlers a map of SIGNAL -> struct sigaction
347			* @return a SigactionMap containing the old handlers
348			*/
349			Watchdog::SigactionMap Watchdog::SetSignalHandlers(
350			const SigactionMap &signal_handlers)
351			{
352			SigactionMap old_signal_handlers;
353			SigactionMap::const_iterator i = signal_handlers.begin();
354			SigactionMap::const_iterator iend = signal_handlers.end();
355			for (; i != iend; ++i) {
356			struct sigaction old_signal_handler;
357			if (sigaction(i->first, &i->second, &old_signal_handler) != 0) {
358			abort();
359			}
360			old_signal_handlers[i->first] = old_signal_handler;
361			}
362
363			return old_signal_handlers;
364			}
365
366
367			/**
368			* Fork the watchdog process.
369			*/
370			void Watchdog::Spawn() {
371			Pipe pipe_pid;
372			pipe_watchdog_ = new Pipe();
373
374			pid_t pid;
375			int statloc;
376			int max_fd = sysconf(_SC_OPEN_MAX);
377			assert(max_fd >= 0);
378			switch (pid = fork()) {
379			case -1: abort();
380			case 0:
381			// Double fork to avoid zombie
382			switch (fork()) {
383			case -1: exit(1);
384			case 0: {
385			close(pipe_watchdog_->write_end);
386			Daemonize();
387			// send the watchdog PID to cvmfs
388			pid_t watchdog_pid = getpid();
389			pipe_pid.Write(watchdog_pid);
390			close(pipe_pid.write_end);
391			// Close all unused file descriptors
392			// close also usyslog, only get it back if necessary
393			// string usyslog_save = GetLogMicroSyslog();
394			string debuglog_save = GetLogDebugFile();
395			// SetLogMicroSyslog("");
396			SetLogDebugFile("");
397			for (int fd = 0; fd < max_fd; fd++) {
398			if (fd != pipe_watchdog_->read_end)
399			close(fd);
400			}
401			// SetLogMicroSyslog(usyslog_save); // no-op if usyslog not used
402			SetLogDebugFile(debuglog_save); // no-op if debug log not used
403			Supervise();
404			exit(0);
405			}
406			default:
407			exit(0);
408			}
409			default:
410			close(pipe_watchdog_->read_end);
411			if (waitpid(pid, &statloc, 0) != pid) abort();
412			if (!WIFEXITED(statloc) \|\| WEXITSTATUS(statloc)) abort();
413			}
414
415			// retrieve the watchdog PID from the pipe
416			close(pipe_pid.write_end);
417			pipe_pid.Read(&watchdog_pid_);
418			close(pipe_pid.read_end);
419
420			// lower restrictions for ptrace
421			if (!platform_allow_ptrace(watchdog_pid_)) {
422			LogCvmfs(kLogMonitor, kLogSyslogWarn,
423			"failed to allow ptrace() for watchdog (PID: %d). "
424			"Post crash stacktrace might not work",
425			watchdog_pid_);
426			}
427
428			// Extra stack for signal handlers
429			int stack_size = kSignalHandlerStacksize; // 2 MB
430			sighandler_stack_.ss_sp = smalloc(stack_size);
431			sighandler_stack_.ss_size = stack_size;
432			sighandler_stack_.ss_flags = 0;
433			if (sigaltstack(&sighandler_stack_, NULL) != 0)
434			abort();
435
436			// define our crash signal handler
437			struct sigaction sa;
438			memset(&sa, 0, sizeof(sa));
439			sa.sa_sigaction = SendTrace;
440			sa.sa_flags = SA_SIGINFO \| SA_ONSTACK;
441			sigfillset(&sa.sa_mask);
442
443			SigactionMap signal_handlers;
444			signal_handlers[SIGQUIT] = sa;
445			signal_handlers[SIGILL] = sa;
446			signal_handlers[SIGABRT] = sa;
447			signal_handlers[SIGFPE] = sa;
448			signal_handlers[SIGSEGV] = sa;
449			signal_handlers[SIGBUS] = sa;
450			signal_handlers[SIGPIPE] = sa;
451			signal_handlers[SIGXFSZ] = sa;
452			old_signal_handlers_ = SetSignalHandlers(signal_handlers);
453
454			spawned_ = true;
455			}
456
457
458			void Watchdog::Supervise() {
459			signal(SIGPIPE, SIG_IGN);
460			ControlFlow::Flags control_flow;
461
462			if (!pipe_watchdog_->Read(&control_flow)) {
463			// Re-activate ��Syslog, if necessary
464			SetLogMicroSyslog(GetLogMicroSyslog());
465			LogEmergency("unexpected termination (" + StringifyInt(control_flow) + ")");
466			if (on_crash_) on_crash_();
467			} else {
468			switch (control_flow) {
469			case ControlFlow::kProduceStacktrace:
470			LogEmergency(ReportStacktrace());
471			if (on_crash_) on_crash_();
472			break;
473
474			case ControlFlow::kQuit:
475			break;
476
477			default:
478			// Re-activate ��Syslog, if necessary
479			SetLogMicroSyslog(GetLogMicroSyslog());
480			LogEmergency("unexpected error");
481			break;
482			}
483			}
484
485			close(pipe_watchdog_->read_end);
486			}
487
488
489		15	Watchdog::Watchdog(const string &crash_dump_path)
490			: spawned_(false)
491			, crash_dump_path_(crash_dump_path)
492			, exe_path_(string(platform_getexepath()))
493			, watchdog_pid_(0)
494			, pipe_watchdog_(NULL)
495		15	, on_crash_(NULL)
496			{
497		15	int retval = platform_spinlock_init(&lock_handler_, 0);
498	✗✓	15	assert(retval == 0);
499		15	memset(&sighandler_stack_, 0, sizeof(sighandler_stack_));
500		15	}
501
502
503		1	Watchdog::~Watchdog() {
504	✗✓	1	if (spawned_) {
505			// Reset signal handlers
506			signal(SIGQUIT, SIG_DFL);
507			signal(SIGILL, SIG_DFL);
508			signal(SIGABRT, SIG_DFL);
509			signal(SIGFPE, SIG_DFL);
510			signal(SIGSEGV, SIG_DFL);
511			signal(SIGBUS, SIG_DFL);
512			signal(SIGPIPE, SIG_DFL);
513			signal(SIGXFSZ, SIG_DFL);
514			free(sighandler_stack_.ss_sp);
515			sighandler_stack_.ss_size = 0;
516
517			pipe_watchdog_->Write(ControlFlow::kQuit);
518			close(pipe_watchdog_->write_end);
519			}
520
521		1	platform_spinlock_destroy(&lock_handler_);
522		1	LogCvmfs(kLogMonitor, kLogDebug, "monitor stopped");
523		1	instance_ = NULL;
524		1	}
525
526
527
528			namespace monitor {
529
530			const unsigned kMinOpenFiles = 8192;
531
532			unsigned GetMaxOpenFiles() {
533			static unsigned max_open_files;
534			static bool already_done = false;
535
536			/* check number of open files (lazy evaluation) */
537			if (!already_done) {
538			unsigned int soft_limit = 0;
539			int hard_limit = 0;
540
541			struct rlimit rpl;
542			memset(&rpl, 0, sizeof(rpl));
543			getrlimit(RLIMIT_NOFILE, &rpl);
544			soft_limit = rpl.rlim_cur;
545
546			#ifdef __APPLE__
547			hard_limit = sysconf(_SC_OPEN_MAX);
548			if (hard_limit < 0) {
549			LogCvmfs(kLogMonitor, kLogStdout, "Warning: could not retrieve "
550			"hard limit for the number of open files");
551			}
552			#else
553			hard_limit = rpl.rlim_max;
554			#endif
555
556			if (soft_limit < kMinOpenFiles) {
557			LogCvmfs(kLogMonitor, kLogSyslogWarn \| kLogDebug,
558			"Warning: current limits for number of open files are "
559			"(%lu/%lu)\n"
560			"CernVM-FS is likely to run out of file descriptors, "
561			"set ulimit -n to at least %lu",
562			soft_limit, hard_limit, kMinOpenFiles);
563			}
564			max_open_files = soft_limit;
565			already_done = true;
566			}
567
568			return max_open_files;
569			}
570
571			} // namespace monitor