GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/talk.cc
Date: 2026-04-19 02:41:37
Exec Total Coverage
Lines: 0 898 0.0%
Branches: 0 2914 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * Implements a socket interface to cvmfs. This way commands can be send
5 * to cvmfs. When cvmfs is running, the socket
6 * /var/cache/cvmfs2/$INSTANCE/cvmfs_io
7 * is available for command input and reply messages, resp.
8 *
9 * Cvmfs comes with the cvmfs_talk script, that handles writing and reading the
10 * socket.
11 *
12 * The talk module runs in a separate thread.
13 */
14
15 #ifndef __STDC_FORMAT_MACROS
16 #define __STDC_FORMAT_MACROS
17 #endif
18
19
20 #include "talk.h"
21
22 #include <errno.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <stdint.h>
26 #include <sys/socket.h>
27 #include <sys/stat.h>
28 #include <sys/types.h>
29 #include <sys/uio.h>
30 #include <sys/un.h>
31 #include <unistd.h>
32
33 #include <cassert>
34 #include <cstdlib>
35 #include <cstring>
36 #include <string>
37 #include <vector>
38
39 #include "cache.h"
40 #include "cache_posix.h"
41 #include "catalog_mgr_client.h"
42 #include "cvmfs.h"
43 #include "duplex_sqlite3.h"
44 #include "fuse_remount.h"
45 #include "glue_buffer.h"
46 #include "loader.h"
47 #include "lru_md.h"
48 #include "monitor.h"
49 #include "mountpoint.h"
50 #include "network/download.h"
51 #include "nfs_maps.h"
52 #include "options.h"
53 #include "quota.h"
54 #include "shortstring.h"
55 #include "statistics.h"
56 #include "tracer.h"
57 #include "util/logging.h"
58 #include "util/pointer.h"
59 #include "wpad.h"
60
61 using namespace std; // NOLINT
62
63
64
65
66 void TalkManager::Answer(int con_fd, const string &msg) {
67 (void)send(con_fd, &msg[0], msg.length(), MSG_NOSIGNAL);
68 }
69
70
71 void TalkManager::AnswerStringList(int con_fd, const vector<string> &list) {
72 string list_str;
73 for (unsigned i = 0; i < list.size(); ++i) {
74 list_str += list[i] + "\n";
75 }
76 Answer(con_fd, list_str);
77 }
78
79
80 TalkManager *TalkManager::Create(const string &socket_path,
81 MountPoint *mount_point,
82 FuseRemounter *remounter) {
83 UniquePtr<TalkManager> talk_manager(
84 new TalkManager(socket_path, mount_point, remounter));
85
86 talk_manager->socket_fd_ = MakeSocket(socket_path, 0660);
87 if (talk_manager->socket_fd_ == -1)
88 return NULL;
89 if (listen(talk_manager->socket_fd_, 1) == -1)
90 return NULL;
91
92 LogCvmfs(kLogTalk, kLogDebug, "socket created at %s (fd %d)",
93 socket_path.c_str(), talk_manager->socket_fd_);
94
95 return talk_manager.Release();
96 }
97
98
99 string TalkManager::FormatMetalinkInfo(
100 download::DownloadManager *download_mgr) {
101 vector<string> metalink_chain;
102 unsigned active_metalink;
103
104 download_mgr->GetMetalinkInfo(&metalink_chain, &active_metalink);
105 if (metalink_chain.size() == 0)
106 return "No metalinks defined\n";
107
108 string metalink_str;
109 for (unsigned i = 0; i < metalink_chain.size(); ++i) {
110 metalink_str += " [" + StringifyInt(i) + "] " + metalink_chain[i] + "\n";
111 }
112 metalink_str += "Active metalink " + StringifyInt(active_metalink) + ": "
113 + metalink_chain[active_metalink] + "\n";
114 return metalink_str;
115 }
116
117 string TalkManager::FormatHostInfo(download::DownloadManager *download_mgr) {
118 vector<string> host_chain;
119 vector<int> rtt;
120 unsigned active_host;
121
122 download_mgr->GetHostInfo(&host_chain, &rtt, &active_host);
123 if (host_chain.size() == 0)
124 return "No hosts defined\n";
125
126 string host_str;
127 for (unsigned i = 0; i < host_chain.size(); ++i) {
128 host_str += " [" + StringifyInt(i) + "] " + host_chain[i] + " (";
129 if (rtt[i] == download::DownloadManager::kProbeUnprobed)
130 host_str += "unprobed";
131 else if (rtt[i] == download::DownloadManager::kProbeDown)
132 host_str += "host down";
133 else if (rtt[i] == download::DownloadManager::kProbeGeo)
134 host_str += "geographically ordered";
135 else
136 host_str += StringifyInt(rtt[i]) + " ms";
137 host_str += ")\n";
138 }
139 host_str += "Active host " + StringifyInt(active_host) + ": "
140 + host_chain[active_host] + "\n";
141 return host_str;
142 }
143
144 string TalkManager::FormatProxyInfo(download::DownloadManager *download_mgr) {
145 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
146 unsigned active_group;
147 unsigned fallback_group;
148
149 download_mgr->GetProxyInfo(&proxy_chain, &active_group, &fallback_group);
150 string proxy_str;
151 if (proxy_chain.size()) {
152 proxy_str += "Load-balance groups:\n";
153 for (unsigned i = 0; i < proxy_chain.size(); ++i) {
154 vector<string> urls;
155 for (unsigned j = 0; j < proxy_chain[i].size(); ++j) {
156 urls.push_back(proxy_chain[i][j].Print());
157 }
158 proxy_str += "[" + StringifyInt(i) + "] " + JoinStrings(urls, ", ")
159 + "\n";
160 }
161 proxy_str += "Active proxy: [" + StringifyInt(active_group) + "] "
162 + proxy_chain[active_group][0].url + "\n";
163 if (fallback_group < proxy_chain.size())
164 proxy_str += "First fallback group: [" + StringifyInt(fallback_group)
165 + "]\n";
166 } else {
167 proxy_str = "No proxies defined\n";
168 }
169 return proxy_str;
170 }
171
172
173 /**
174 * Listener thread on the socket.
175 * TODO(jblomer): create Format... helpers to shorten this method
176 */
177 void *TalkManager::MainResponder(void *data) {
178 TalkManager *talk_mgr = reinterpret_cast<TalkManager *>(data);
179 MountPoint *mount_point = talk_mgr->mount_point_;
180 FileSystem *file_system = mount_point->file_system();
181 FuseRemounter *remounter = talk_mgr->remounter_;
182 LogCvmfs(kLogTalk, kLogDebug, "talk thread started");
183
184 struct sockaddr_un remote;
185 socklen_t socket_size = sizeof(remote);
186 int con_fd = -1;
187 while (true) {
188 if (con_fd >= 0) {
189 shutdown(con_fd, SHUT_RDWR);
190 close(con_fd);
191 }
192 LogCvmfs(kLogTalk, kLogDebug, "accepting connections on socketfd %d",
193 talk_mgr->socket_fd_);
194 if ((con_fd = accept(talk_mgr->socket_fd_, (struct sockaddr *)&remote,
195 &socket_size))
196 < 0) {
197 LogCvmfs(kLogTalk, kLogDebug, "terminating talk thread (fd %d, errno %d)",
198 con_fd, errno);
199 break;
200 }
201
202 char buf[kMaxCommandSize];
203 int bytes_read;
204 if ((bytes_read = recv(con_fd, buf, sizeof(buf), 0)) <= 0)
205 continue;
206
207 if (buf[bytes_read - 1] == '\0')
208 bytes_read--;
209 const string line = string(buf, bytes_read);
210 LogCvmfs(kLogTalk, kLogDebug, "received %s (length %lu)", line.c_str(),
211 line.length());
212
213 if (line == "tracebuffer flush") {
214 mount_point->tracer()->Flush();
215 talk_mgr->Answer(con_fd, "OK\n");
216 } else if (line == "cache size") {
217 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
218 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
219 talk_mgr->Answer(con_fd, "Cache cannot report its size\n");
220 } else {
221 const uint64_t size_unpinned = quota_mgr->GetSize();
222 const uint64_t size_pinned = quota_mgr->GetSizePinned();
223 const string size_str = "Current cache size is "
224 + StringifyInt(size_unpinned / (1024 * 1024))
225 + "MB (" + StringifyInt(size_unpinned)
226 + " Bytes), pinned: "
227 + StringifyInt(size_pinned / (1024 * 1024))
228 + "MB (" + StringifyInt(size_pinned)
229 + " Bytes)\n";
230 talk_mgr->Answer(con_fd, size_str);
231 }
232 } else if (line == "cache instance") {
233 talk_mgr->Answer(con_fd, file_system->cache_mgr()->Describe());
234 } else if (line == "cache list") {
235 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
236 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
237 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
238 } else {
239 const vector<string> ls = quota_mgr->List();
240 talk_mgr->AnswerStringList(con_fd, ls);
241 }
242 } else if (line == "cache list pinned") {
243 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
244 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
245 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
246 } else {
247 const vector<string> ls_pinned = quota_mgr->ListPinned();
248 talk_mgr->AnswerStringList(con_fd, ls_pinned);
249 }
250 } else if (line == "cache list catalogs") {
251 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
252 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
253 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
254 } else {
255 const vector<string> ls_catalogs = quota_mgr->ListCatalogs();
256 talk_mgr->AnswerStringList(con_fd, ls_catalogs);
257 }
258 } else if (line.substr(0, 12) == "cleanup rate") {
259 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
260 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
261 talk_mgr->Answer(con_fd, "Unsupported by this cache\n");
262 } else {
263 if (line.length() < 14) {
264 talk_mgr->Answer(con_fd, "Usage: cleanup rate <period in mn>\n");
265 } else {
266 const uint64_t period_s = String2Uint64(line.substr(13)) * 60;
267 const uint64_t rate = quota_mgr->GetCleanupRate(period_s);
268 talk_mgr->Answer(con_fd, StringifyInt(rate) + "\n");
269 }
270 }
271 } else if (line.substr(0, 15) == "cache limit set") {
272 if (line.length() < 16) {
273 talk_mgr->Answer(con_fd, "Usage: cache limit set <MB>\n");
274 } else {
275 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
276 const uint64_t size = String2Uint64(line.substr(16));
277 if (size < 1000) {
278 talk_mgr->Answer(con_fd, "New limit too low (minimum 1000)\n");
279 } else {
280 if (quota_mgr->SetLimit(size * 1024 * 1024)) {
281 file_system->options_mgr()->SetValueFromTalk("CVMFS_QUOTA_LIMIT",
282 StringifyUint(size));
283 talk_mgr->Answer(con_fd, "OK\n");
284 } else {
285 talk_mgr->Answer(con_fd, "Limit not reset\n");
286 }
287 }
288 }
289 } else if (line == "cache limit get") {
290 std::string limit_from_options;
291 file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT",
292 &limit_from_options);
293 talk_mgr->Answer(con_fd, limit_from_options + "\n");
294 } else if (line.substr(0, 7) == "cleanup") {
295 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
296 if (!quota_mgr->HasCapability(QuotaManager::kCapShrink)) {
297 talk_mgr->Answer(con_fd, "Cache cannot trigger eviction\n");
298 } else {
299 if (line.length() < 9) {
300 talk_mgr->Answer(con_fd, "Usage: cleanup <MB>\n");
301 } else {
302 const uint64_t size = String2Uint64(line.substr(8)) * 1024 * 1024;
303 if (quota_mgr->Cleanup(size)) {
304 talk_mgr->Answer(con_fd, "OK\n");
305 } else {
306 talk_mgr->Answer(con_fd, "Not fully cleaned "
307 "(there might be pinned chunks)\n");
308 }
309 }
310 }
311 } else if (line.substr(0, 5) == "evict") {
312 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
313 if (line.length() < 7) {
314 talk_mgr->Answer(con_fd, "Usage: evict <path>\n");
315 } else {
316 const string path = line.substr(6);
317 const bool found_regular = cvmfs::Evict(path);
318 if (found_regular)
319 talk_mgr->Answer(con_fd, "OK\n");
320 else
321 talk_mgr->Answer(con_fd, "No such regular file\n");
322 }
323 } else if (line.substr(0, 3) == "pin") {
324 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
325 if (line.length() < 5) {
326 talk_mgr->Answer(con_fd, "Usage: pin <path>\n");
327 } else {
328 const string path = line.substr(4);
329 const bool found_regular = cvmfs::Pin(path);
330 if (found_regular)
331 talk_mgr->Answer(con_fd, "OK\n");
332 else
333 talk_mgr->Answer(con_fd, "No such regular file or pinning failed\n");
334 }
335 } else if (line == "mountpoint") {
336 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->mount_point + "\n");
337 } else if (line == "device id") {
338 if (cvmfs::loader_exports_->version >= 5)
339 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->device_id + "\n");
340 else
341 talk_mgr->Answer(con_fd, "0:0\n");
342 } else if (line.substr(0, 13) == "send mount fd") {
343 // Hidden command intended to be used only by the cvmfs mount helper
344 if (line.length() < 15) {
345 talk_mgr->Answer(con_fd, "EINVAL\n");
346 } else {
347 const std::string socket_path = line.substr(14);
348 const bool retval = cvmfs::SendFuseFd(socket_path);
349 talk_mgr->Answer(con_fd, retval ? "OK\n" : "Failed\n");
350 LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
351 "Attempt to send fuse connection info to new mount (via %s)%s",
352 socket_path.c_str(), retval ? "" : " -- failed!");
353 }
354 } else if (line.substr(0, 7) == "remount") {
355 FuseRemounter::Status status;
356 if (line == "remount sync")
357 status = remounter->CheckSynchronously();
358 else
359 status = remounter->Check();
360 switch (status) {
361 case FuseRemounter::kStatusFailGeneral:
362 talk_mgr->Answer(con_fd, "Failed\n");
363 break;
364 case FuseRemounter::kStatusFailNoSpace:
365 talk_mgr->Answer(con_fd, "Failed (no space)\n");
366 break;
367 case FuseRemounter::kStatusUp2Date:
368 talk_mgr->Answer(con_fd, "Catalog up to date\n");
369 break;
370 case FuseRemounter::kStatusDraining:
371 talk_mgr->Answer(con_fd, "New revision applied\n");
372 break;
373 case FuseRemounter::kStatusMaintenance:
374 talk_mgr->Answer(con_fd, "In maintenance mode\n");
375 break;
376 default:
377 talk_mgr->Answer(con_fd, "internal error\n");
378 }
379 } else if (line.substr(0, 6) == "chroot") {
380 if (line.length() < 8) {
381 talk_mgr->Answer(con_fd, "Usage: chroot <hash>\n");
382 } else {
383 const std::string root_hash = Trim(line.substr(7),
384 true /* trim_newline */);
385 const FuseRemounter::Status status = remounter->ChangeRoot(
386 MkFromHexPtr(shash::HexPtr(root_hash), shash::kSuffixCatalog));
387 switch (status) {
388 case FuseRemounter::kStatusUp2Date:
389 talk_mgr->Answer(con_fd, "OK\n");
390 break;
391 default:
392 talk_mgr->Answer(con_fd, "Failed\n");
393 break;
394 }
395 }
396 } else if (line == "detach nested catalogs") {
397 mount_point->catalog_mgr()->DetachNested();
398 talk_mgr->Answer(con_fd, "OK\n");
399 } else if (line == "revision") {
400 const string revision = StringifyInt(
401 mount_point->catalog_mgr()->GetRevision());
402 talk_mgr->Answer(con_fd, revision + "\n");
403 } else if (line == "max ttl info") {
404 const unsigned max_ttl = mount_point->GetMaxTtlMn();
405 if (max_ttl == 0) {
406 talk_mgr->Answer(con_fd, "unset\n");
407 } else {
408 const string max_ttl_str = StringifyInt(max_ttl) + " minutes\n";
409 talk_mgr->Answer(con_fd, max_ttl_str);
410 }
411 } else if (line.substr(0, 11) == "max ttl set") {
412 if (line.length() < 13) {
413 talk_mgr->Answer(con_fd, "Usage: max ttl set <minutes>\n");
414 } else {
415 const unsigned max_ttl = String2Uint64(line.substr(12));
416 mount_point->SetMaxTtlMn(max_ttl);
417 talk_mgr->Answer(con_fd, "OK\n");
418 }
419 } else if (line.substr(0, 14) == "nameserver get") {
420 const string dns_server = mount_point->download_mgr()->GetDnsServer();
421 const string reply = !dns_server.empty()
422 ? std::string("DNS server address: ")
423 + dns_server + "\n"
424 : std::string("DNS server not set.\n");
425 talk_mgr->Answer(con_fd, reply);
426 } else if (line.substr(0, 14) == "nameserver set") {
427 if (line.length() < 16) {
428 talk_mgr->Answer(con_fd, "Usage: nameserver set <host>\n");
429 } else {
430 const string host = line.substr(15);
431 mount_point->download_mgr()->SetDnsServer(host);
432 talk_mgr->Answer(con_fd, "OK\n");
433 }
434 } else if (line.substr(0, 22) == "__testing_freeze_cvmfs") {
435 const std::string fs_dir = line.substr(23) + "/dir";
436 mkdir(fs_dir.c_str(), 0700);
437 } else if (line == "external metalink info") {
438 const string external_metalink_info = talk_mgr->FormatMetalinkInfo(
439 mount_point->external_download_mgr());
440 talk_mgr->Answer(con_fd, external_metalink_info);
441 } else if (line == "metalink info") {
442 const string metalink_info = talk_mgr->FormatMetalinkInfo(
443 mount_point->download_mgr());
444 talk_mgr->Answer(con_fd, metalink_info);
445 } else if (line == "external host info") {
446 const string external_host_info = talk_mgr->FormatHostInfo(
447 mount_point->external_download_mgr());
448 talk_mgr->Answer(con_fd, external_host_info);
449 } else if (line == "host info") {
450 const string host_info = talk_mgr->FormatHostInfo(
451 mount_point->download_mgr());
452 talk_mgr->Answer(con_fd, host_info);
453 } else if (line == "host probe") {
454 mount_point->download_mgr()->ProbeHosts();
455 talk_mgr->Answer(con_fd, "OK\n");
456 } else if (line == "host probe geo") {
457 const bool retval = mount_point->download_mgr()->ProbeGeo();
458 if (retval)
459 talk_mgr->Answer(con_fd, "OK\n");
460 else
461 talk_mgr->Answer(con_fd, "Failed\n");
462 } else if (line == "external metalink switch") {
463 mount_point->external_download_mgr()->SwitchMetalink();
464 talk_mgr->Answer(con_fd, "OK\n");
465 } else if (line == "metalink switch") {
466 mount_point->download_mgr()->SwitchMetalink();
467 talk_mgr->Answer(con_fd, "OK\n");
468 } else if (line == "external host switch") {
469 mount_point->external_download_mgr()->SwitchHost();
470 talk_mgr->Answer(con_fd, "OK\n");
471 } else if (line == "host switch") {
472 mount_point->download_mgr()->SwitchHost();
473 talk_mgr->Answer(con_fd, "OK\n");
474 } else if (line.substr(0, 21) == "external metalink set") {
475 if (line.length() < 23) {
476 talk_mgr->Answer(con_fd, "Usage: external metalink set <URL>\n");
477 } else {
478 const std::string host = line.substr(22);
479 mount_point->external_download_mgr()->SetMetalinkChain(host);
480 talk_mgr->Answer(con_fd, "OK\n");
481 }
482 } else if (line.substr(0, 12) == "metalink set") {
483 if (line.length() < 14) {
484 talk_mgr->Answer(con_fd, "Usage: metalink set <URL>\n");
485 } else {
486 const std::string host = line.substr(13);
487 mount_point->download_mgr()->SetMetalinkChain(host);
488 talk_mgr->Answer(con_fd, "OK\n");
489 }
490 } else if (line.substr(0, 17) == "external host set") {
491 if (line.length() < 19) {
492 talk_mgr->Answer(con_fd, "Usage: external host set <URL>\n");
493 } else {
494 const std::string host = line.substr(18);
495 mount_point->external_download_mgr()->SetHostChain(host);
496 talk_mgr->Answer(con_fd, "OK\n");
497 }
498 } else if (line.substr(0, 8) == "host set") {
499 if (line.length() < 10) {
500 talk_mgr->Answer(con_fd, "Usage: host set <host list>\n");
501 } else {
502 const string hosts = line.substr(9);
503 mount_point->download_mgr()->SetHostChain(hosts);
504 talk_mgr->Answer(con_fd, "OK\n");
505 }
506 } else if (line == "external proxy info") {
507 const string external_proxy_info = talk_mgr->FormatProxyInfo(
508 mount_point->external_download_mgr());
509 talk_mgr->Answer(con_fd, external_proxy_info);
510 } else if (line == "proxy info") {
511 const string proxy_info = talk_mgr->FormatProxyInfo(
512 mount_point->download_mgr());
513 talk_mgr->Answer(con_fd, proxy_info);
514 } else if (line == "proxy rebalance") {
515 mount_point->download_mgr()->RebalanceProxies();
516 talk_mgr->Answer(con_fd, "OK\n");
517 } else if (line == "proxy group switch") {
518 mount_point->download_mgr()->SwitchProxyGroup();
519 talk_mgr->Answer(con_fd, "OK\n");
520 } else if (line.substr(0, 18) == "external proxy set") {
521 if (line.length() < 20) {
522 talk_mgr->Answer(con_fd, "Usage: external proxy set <proxy list>\n");
523 } else {
524 const string external_proxies = line.substr(19);
525 mount_point->external_download_mgr()->SetProxyChain(
526 external_proxies, "", download::DownloadManager::kSetProxyRegular);
527 talk_mgr->Answer(con_fd, "OK\n");
528 }
529 } else if (line.substr(0, 9) == "proxy set") {
530 if (line.length() < 11) {
531 talk_mgr->Answer(con_fd, "Usage: proxy set <proxy list>\n");
532 } else {
533 string proxies = line.substr(10);
534 proxies = download::ResolveProxyDescription(
535 proxies, "", mount_point->download_mgr());
536 if (proxies == "") {
537 talk_mgr->Answer(con_fd, "Failed, no valid proxies\n");
538 } else {
539 mount_point->download_mgr()->SetProxyChain(
540 proxies, "", download::DownloadManager::kSetProxyRegular);
541 talk_mgr->Answer(con_fd, "OK\n");
542 }
543 }
544 } else if (line.substr(0, 14) == "proxy fallback") {
545 if (line.length() < 15) {
546 talk_mgr->Answer(con_fd, "Usage: proxy fallback <proxy list>\n");
547 } else {
548 const string fallback_proxies = line.substr(15);
549 mount_point->download_mgr()->SetProxyChain(
550 "", fallback_proxies, download::DownloadManager::kSetProxyFallback);
551 talk_mgr->Answer(con_fd, "OK\n");
552 }
553 } else if (line == "timeout info") {
554 unsigned timeout;
555 unsigned timeout_direct;
556 mount_point->download_mgr()->GetTimeout(&timeout, &timeout_direct);
557 string timeout_str = "Timeout with proxy: ";
558 if (timeout)
559 timeout_str += StringifyInt(timeout) + "s\n";
560 else
561 timeout_str += "no timeout\n";
562 timeout_str += "Timeout without proxy: ";
563 if (timeout_direct)
564 timeout_str += StringifyInt(timeout_direct) + "s\n";
565 else
566 timeout_str += "no timeout\n";
567 talk_mgr->Answer(con_fd, timeout_str);
568 } else if (line.substr(0, 11) == "timeout set") {
569 if (line.length() < 13) {
570 talk_mgr->Answer(con_fd, "Usage: timeout set <proxy> <direct>\n");
571 } else {
572 uint64_t timeout;
573 uint64_t timeout_direct;
574 String2Uint64Pair(line.substr(12), &timeout, &timeout_direct);
575 mount_point->download_mgr()->SetTimeout(timeout, timeout_direct);
576 talk_mgr->Answer(con_fd, "OK\n");
577 }
578 } else if (line == "open catalogs") {
579 talk_mgr->Answer(con_fd, mount_point->catalog_mgr()->PrintHierarchy());
580 } else if (line == "drop metadata caches") {
581 // For testing
582 mount_point->inode_cache()->Pause();
583 mount_point->path_cache()->Pause();
584 mount_point->md5path_cache()->Pause();
585 mount_point->inode_cache()->Drop();
586 mount_point->path_cache()->Drop();
587 mount_point->md5path_cache()->Drop();
588 mount_point->inode_cache()->Resume();
589 mount_point->path_cache()->Resume();
590 mount_point->md5path_cache()->Resume();
591 talk_mgr->Answer(con_fd, "OK\n");
592 } else if (line == "internal affairs") {
593 int current;
594 int highwater;
595 string result;
596
597 result += "Inode Generation:\n " + cvmfs::PrintInodeGeneration();
598
599 // Manually setting the values of the ShortString counters
600 mount_point->statistics()
601 ->Lookup("pathstring.n_instances")
602 ->Set(PathString::num_instances());
603 mount_point->statistics()
604 ->Lookup("pathstring.n_overflows")
605 ->Set(PathString::num_overflows());
606 mount_point->statistics()
607 ->Lookup("namestring.n_instances")
608 ->Set(NameString::num_instances());
609 mount_point->statistics()
610 ->Lookup("namestring.n_overflows")
611 ->Set(NameString::num_overflows());
612 mount_point->statistics()
613 ->Lookup("linkstring.n_instances")
614 ->Set(LinkString::num_instances());
615 mount_point->statistics()
616 ->Lookup("linkstring.n_overflows")
617 ->Set(LinkString::num_overflows());
618
619 // Manually setting the inode tracker numbers
620 glue::InodeTracker::Statistics inode_stats = mount_point->inode_tracker()
621 ->GetStatistics();
622 const glue::DentryTracker::Statistics
623 dentry_stats = mount_point->dentry_tracker()->GetStatistics();
624 const glue::PageCacheTracker::Statistics
625 page_cache_stats = mount_point->page_cache_tracker()->GetStatistics();
626 mount_point->statistics()
627 ->Lookup("inode_tracker.n_insert")
628 ->Set(atomic_read64(&inode_stats.num_inserts));
629 mount_point->statistics()
630 ->Lookup("inode_tracker.n_remove")
631 ->Set(atomic_read64(&inode_stats.num_removes));
632 mount_point->statistics()
633 ->Lookup("inode_tracker.no_reference")
634 ->Set(atomic_read64(&inode_stats.num_references));
635 mount_point->statistics()
636 ->Lookup("inode_tracker.n_hit_inode")
637 ->Set(atomic_read64(&inode_stats.num_hits_inode));
638 mount_point->statistics()
639 ->Lookup("inode_tracker.n_hit_path")
640 ->Set(atomic_read64(&inode_stats.num_hits_path));
641 mount_point->statistics()
642 ->Lookup("inode_tracker.n_miss_path")
643 ->Set(atomic_read64(&inode_stats.num_misses_path));
644 mount_point->statistics()
645 ->Lookup("dentry_tracker.n_insert")
646 ->Set(dentry_stats.num_insert);
647 mount_point->statistics()
648 ->Lookup("dentry_tracker.n_remove")
649 ->Set(dentry_stats.num_remove);
650 mount_point->statistics()
651 ->Lookup("dentry_tracker.n_prune")
652 ->Set(dentry_stats.num_prune);
653 mount_point->statistics()
654 ->Lookup("page_cache_tracker.n_insert")
655 ->Set(page_cache_stats.n_insert);
656 mount_point->statistics()
657 ->Lookup("page_cache_tracker.n_remove")
658 ->Set(page_cache_stats.n_remove);
659 mount_point->statistics()
660 ->Lookup("page_cache_tracker.n_open_direct")
661 ->Set(page_cache_stats.n_open_direct);
662 mount_point->statistics()
663 ->Lookup("page_cache_tracker.n_open_flush")
664 ->Set(page_cache_stats.n_open_flush);
665 mount_point->statistics()
666 ->Lookup("page_cache_tracker.n_open_cached")
667 ->Set(page_cache_stats.n_open_cached);
668
669 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
670 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(
671 file_system->cache_mgr());
672 result += "\nCache Mode: ";
673 switch (cache_mgr->cache_mode()) {
674 case PosixCacheManager::kCacheReadWrite:
675 result += "read-write";
676 break;
677 case PosixCacheManager::kCacheReadOnly:
678 result += "read-only";
679 break;
680 default:
681 result += "unknown";
682 }
683 }
684 bool drainout_mode;
685 bool maintenance_mode;
686 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
687 result += "\nDrainout Mode: " + StringifyBool(drainout_mode) + "\n";
688 result += "Maintenance Mode: " + StringifyBool(maintenance_mode) + "\n";
689
690 if (file_system->IsNfsSource()) {
691 result += "\nNFS Map Statistics:\n";
692 result += file_system->nfs_maps()->GetStatistics();
693 }
694
695 result += "SQlite Statistics:\n";
696 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
697 result += " Number of allocations " + StringifyInt(current) + "\n";
698
699 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
700 result += " General purpose allocator " + StringifyInt(current / 1024)
701 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
702
703 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
704 result += " Largest malloc " + StringifyInt(highwater) + " Bytes\n";
705
706 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
707 result += " Page cache allocations " + StringifyInt(current) + " / "
708 + StringifyInt(highwater) + "\n";
709
710 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
711 result += " Page cache overflows " + StringifyInt(current / 1024)
712 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
713
714 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
715 result += " Largest page cache allocation " + StringifyInt(highwater)
716 + " Bytes\n";
717
718 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
719 result += " Scratch allocations " + StringifyInt(current) + " / "
720 + StringifyInt(highwater) + "\n";
721
722 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
723 result += " Scratch overflows " + StringifyInt(current) + " / "
724 + StringifyInt(highwater) + "\n";
725
726 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
727 result += " Largest scratch allocation " + StringifyInt(highwater / 1024)
728 + " KB\n";
729
730 result += "\nPer-Connection Memory Statistics:\n"
731 + mount_point->catalog_mgr()->PrintAllMemStatistics();
732
733 result += "\nLatency distribution of system calls:\n";
734
735 result += "Lookup\n" + file_system->hist_fs_lookup()->ToString();
736 result += "Forget\n" + file_system->hist_fs_forget()->ToString();
737 result += "Multi-Forget\n"
738 + file_system->hist_fs_forget_multi()->ToString();
739 result += "Getattr\n" + file_system->hist_fs_getattr()->ToString();
740 result += "Readlink\n" + file_system->hist_fs_readlink()->ToString();
741 result += "Opendir\n" + file_system->hist_fs_opendir()->ToString();
742 result += "Releasedir\n" + file_system->hist_fs_releasedir()->ToString();
743 result += "Readdir\n" + file_system->hist_fs_readdir()->ToString();
744 result += "Open\n" + file_system->hist_fs_open()->ToString();
745 result += "Read\n" + file_system->hist_fs_read()->ToString();
746 result += "Release\n" + file_system->hist_fs_release()->ToString();
747
748 result += "\nRaw Counters:\n"
749 + mount_point->statistics()->PrintList(
750 perf::Statistics::kPrintHeader);
751
752 talk_mgr->Answer(con_fd, result);
753 } else if (line == "reset error counters") {
754 file_system->ResetErrorCounters();
755 talk_mgr->Answer(con_fd, "OK\n");
756 } else if (line == "pid") {
757 const string pid_str = StringifyInt(cvmfs::pid_) + "\n";
758 talk_mgr->Answer(con_fd, pid_str);
759 } else if (line == "pid cachemgr") {
760 const string pid_str = StringifyInt(file_system->cache_mgr()
761 ->quota_mgr()
762 ->GetPid())
763 + "\n";
764 talk_mgr->Answer(con_fd, pid_str);
765 } else if (line == "pid watchdog") {
766 const string pid_str = StringifyInt(Watchdog::GetPid()) + "\n";
767 talk_mgr->Answer(con_fd, pid_str);
768 } else if (line == "parameters") {
769 talk_mgr->Answer(con_fd, file_system->options_mgr()->Dump());
770 } else if (line == "hotpatch history") {
771 string history_str = StringifyTime(cvmfs::loader_exports_->boot_time,
772 true)
773 + " (start of CernVM-FS loader "
774 + cvmfs::loader_exports_->loader_version + ")\n";
775 for (loader::EventList::const_iterator
776 i = cvmfs::loader_exports_->history.begin(),
777 iEnd = cvmfs::loader_exports_->history.end();
778 i != iEnd;
779 ++i) {
780 history_str += StringifyTime((*i)->timestamp, true)
781 + " (loaded CernVM-FS Fuse Module " + (*i)->so_version
782 + ")\n";
783 }
784 talk_mgr->Answer(con_fd, history_str);
785 } else if (line == "vfs inodes") {
786 string result;
787 glue::InodeTracker::Cursor cursor(
788 mount_point->inode_tracker()->BeginEnumerate());
789 uint64_t inode;
790 while (mount_point->inode_tracker()->NextInode(&cursor, &inode)) {
791 result += StringifyInt(inode) + "\n";
792 }
793 mount_point->inode_tracker()->EndEnumerate(&cursor);
794 talk_mgr->Answer(con_fd, result);
795 } else if (line == "vfs entries") {
796 string result;
797 glue::InodeTracker::Cursor cursor(
798 mount_point->inode_tracker()->BeginEnumerate());
799 uint64_t inode_parent;
800 NameString name;
801 while (mount_point->inode_tracker()->NextEntry(&cursor, &inode_parent,
802 &name)) {
803 result += "<" + StringifyInt(inode_parent) + ">/" + name.ToString()
804 + "\n";
805 }
806 mount_point->inode_tracker()->EndEnumerate(&cursor);
807 talk_mgr->Answer(con_fd, result);
808 } else if (line == "version") {
809 const string version_str = string(CVMFS_VERSION)
810 + " (CernVM-FS Fuse Module)\n"
811 + cvmfs::loader_exports_->loader_version
812 + " (Loader)\n";
813 talk_mgr->Answer(con_fd, version_str);
814 } else if (line == "version patchlevel") {
815 talk_mgr->Answer(con_fd, string(CVMFS_PATCH_LEVEL) + "\n");
816 } else if (line == "tear down to read-only") {
817 if (file_system->cache_mgr()->id() != kPosixCacheManager) {
818 talk_mgr->Answer(con_fd, "not supported\n");
819 } else {
820 // hack
821 cvmfs::UnregisterQuotaListener();
822 file_system->TearDown2ReadOnly();
823 talk_mgr->Answer(con_fd, "In read-only mode\n");
824 }
825 } else if (line == "latency") {
826 const string result = talk_mgr->FormatLatencies(*mount_point,
827 file_system);
828 talk_mgr->Answer(con_fd, result);
829 } else if (line == "metrics prometheus") {
830 const string result = talk_mgr->FormatPrometheusMetrics(*mount_point,
831 file_system);
832 talk_mgr->Answer(con_fd, result);
833 } else {
834 talk_mgr->Answer(con_fd, "unknown command\n");
835 }
836 }
837
838 return NULL;
839 } // NOLINT(readability/fn_size)
840
841 string TalkManager::FormatLatencies(const MountPoint &mount_point,
842 FileSystem *file_system) {
843 string result;
844 const unsigned int bufSize = 300;
845 char buffer[bufSize];
846
847 vector<float> qs;
848 qs.push_back(.1);
849 qs.push_back(.2);
850 qs.push_back(.25);
851 qs.push_back(.3);
852 qs.push_back(.4);
853 qs.push_back(.5);
854 qs.push_back(.6);
855 qs.push_back(.7);
856 qs.push_back(.75);
857 qs.push_back(.8);
858 qs.push_back(.9);
859 qs.push_back(.95);
860 qs.push_back(.99);
861 qs.push_back(.999);
862 qs.push_back(.9999);
863
864 const string repo(mount_point.fqrn());
865
866 unsigned int format_index = snprintf(
867 buffer, bufSize, "\"%s\",\"%s\",\"%s\",\"%s\"", "repository", "action",
868 "total_count", "time_unit");
869 for (unsigned int i = 0; i < qs.size(); i++) {
870 format_index += snprintf(buffer + format_index, bufSize - format_index,
871 ",%0.5f", qs[i]);
872 }
873 format_index += snprintf(buffer + format_index, bufSize - format_index, "\n");
874 assert(format_index < bufSize);
875
876 result += buffer;
877 memset(buffer, 0, sizeof(buffer));
878 format_index = 0;
879
880 vector<Log2Histogram *> hist;
881 vector<string> names;
882 hist.push_back(file_system->hist_fs_lookup());
883 names.push_back("lookup");
884 hist.push_back(file_system->hist_fs_forget());
885 names.push_back("forget");
886 hist.push_back(file_system->hist_fs_forget_multi());
887 names.push_back("forget_multi");
888 hist.push_back(file_system->hist_fs_getattr());
889 names.push_back("getattr");
890 hist.push_back(file_system->hist_fs_readlink());
891 names.push_back("readlink");
892 hist.push_back(file_system->hist_fs_opendir());
893 names.push_back("opendir");
894 hist.push_back(file_system->hist_fs_releasedir());
895 names.push_back("releasedir");
896 hist.push_back(file_system->hist_fs_readdir());
897 names.push_back("readdir");
898 hist.push_back(file_system->hist_fs_open());
899 names.push_back("open");
900 hist.push_back(file_system->hist_fs_read());
901 names.push_back("read");
902 hist.push_back(file_system->hist_fs_release());
903 names.push_back("release");
904
905 for (unsigned int j = 0; j < hist.size(); j++) {
906 Log2Histogram *h = hist[j];
907 unsigned int format_index = snprintf(
908 buffer, bufSize, "\"%s\",\"%s\",%" PRIu64 ",\"nanoseconds\"",
909 repo.c_str(), names[j].c_str(), h->N());
910 for (unsigned int i = 0; i < qs.size(); i++) {
911 format_index += snprintf(buffer + format_index, bufSize - format_index,
912 ",%u", h->GetQuantile(qs[i]));
913 }
914 format_index += snprintf(buffer + format_index, bufSize - format_index,
915 "\n");
916 assert(format_index < bufSize);
917
918 result += buffer;
919 memset(buffer, 0, sizeof(buffer));
920 format_index = 0;
921 }
922 return result;
923 }
924
925 string TalkManager::FormatPrometheusMetrics(MountPoint &mount_point,
926 FileSystem *file_system) {
927 string result;
928 const string fqrn = mount_point.fqrn();
929 const string mountpoint = cvmfs::loader_exports_->mount_point;
930
931 // Helper function to format a prometheus metric
932 class MetricFormatter {
933 public:
934 explicit MetricFormatter(string &result_ref) : result_(result_ref) {}
935 void operator()(const string &name, const string &type,
936 const string &help, const string &labels,
937 const string &value) {
938 result_ += "# HELP " + name + " " + help + "\n";
939 result_ += "# TYPE " + name + " " + type + "\n";
940 result_ += name + "{" + labels + "} " + value + "\n";
941 }
942 private:
943 string &result_;
944 };
945 MetricFormatter format_metric(result);
946
947 // Get cache information
948 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
949 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
950 const uint64_t size_unpinned = quota_mgr->GetSize();
951 const uint64_t size_pinned = quota_mgr->GetSizePinned();
952
953 format_metric("cvmfs_cached_bytes", "gauge",
954 "CVMFS currently cached bytes.",
955 "repo=\"" + fqrn + "\"", StringifyUint(size_unpinned));
956 format_metric("cvmfs_pinned_bytes", "gauge",
957 "CVMFS currently pinned bytes.",
958 "repo=\"" + fqrn + "\"", StringifyUint(size_pinned));
959 }
960
961 // Get cache limit from parameters
962 string cache_limit_str;
963 if (file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT", &cache_limit_str)) {
964 const uint64_t cache_limit_mb = String2Uint64(cache_limit_str);
965 const uint64_t cache_limit_bytes = cache_limit_mb * 1024 * 1024;
966 format_metric("cvmfs_total_cache_size_bytes", "gauge",
967 "CVMFS configured cache size via CVMFS_QUOTA_LIMIT.",
968 "repo=\"" + fqrn + "\"", StringifyUint(cache_limit_bytes));
969 }
970
971 // Get cache base directory for df information
972 string cache_base;
973 if (file_system->options_mgr()->GetValue("CVMFS_CACHE_BASE", &cache_base)) {
974 struct statvfs stat_info;
975 if (statvfs(cache_base.c_str(), &stat_info) == 0) {
976 const uint64_t total_size = static_cast<uint64_t>(stat_info.f_blocks) * stat_info.f_frsize;
977 const uint64_t avail_size = static_cast<uint64_t>(stat_info.f_bavail) * stat_info.f_frsize;
978
979 format_metric("cvmfs_physical_cache_size_bytes", "gauge",
980 "CVMFS cache volume physical size.",
981 "repo=\"" + fqrn + "\"", StringifyUint(total_size));
982 format_metric("cvmfs_physical_cache_avail_bytes", "gauge",
983 "CVMFS cache volume physical free space available.",
984 "repo=\"" + fqrn + "\"", StringifyUint(avail_size));
985 }
986 }
987
988 // Version and revision information
989 const string version = string(CVMFS_VERSION) + "." + string(CVMFS_PATCH_LEVEL);
990 const uint64_t revision = mount_point.catalog_mgr()->GetRevision();
991 format_metric("cvmfs_repo", "gauge",
992 "Shows the version of CVMFS used by this repository.",
993 "repo=\"" + fqrn + "\",mountpoint=\"" + mountpoint +
994 "\",version=\"" + version + "\",revision=\"" + StringifyUint(revision) + "\"",
995 "1");
996
997 // Statistics-based metrics
998 perf::Statistics *statistics = mount_point.statistics();
999
1000 // Download statistics
1001 const int64_t rx_bytes = statistics->Lookup("download.sz_transferred_bytes")->Get();
1002 format_metric("cvmfs_rx_total", "counter",
1003 "Shows the overall amount of downloaded bytes since mounting.",
1004 "repo=\"" + fqrn + "\"", StringifyInt(rx_bytes));
1005
1006 const int64_t n_downloads = statistics->Lookup("fetch.n_downloads")->Get();
1007 format_metric("cvmfs_ndownload_total", "counter",
1008 "Shows the overall number of downloaded files since mounting.",
1009 "repo=\"" + fqrn + "\"", StringifyInt(n_downloads));
1010
1011 // Hit rate calculation
1012 const int64_t n_invocations = statistics->Lookup("fetch.n_invocations")->Get();
1013 if (n_invocations > 0) {
1014 const float hitrate = 100.0 * (1.0 - (static_cast<float>(n_downloads) /
1015 static_cast<float>(n_invocations)));
1016 format_metric("cvmfs_hitrate", "gauge",
1017 "CVMFS cache hit rate (%)",
1018 "repo=\"" + fqrn + "\"", StringifyDouble(hitrate));
1019 } else {
1020 format_metric("cvmfs_hitrate", "gauge",
1021 "CVMFS cache hit rate (%)",
1022 "repo=\"" + fqrn + "\"", "0");
1023 }
1024
1025 // Speed calculation
1026 const int64_t transfer_time = statistics->Lookup("download.sz_transfer_time")->Get();
1027 if (transfer_time > 0) {
1028 const int64_t speed = (1000 * (rx_bytes / 1024)) / transfer_time;
1029 format_metric("cvmfs_speed", "gauge",
1030 "Shows the average download speed.",
1031 "repo=\"" + fqrn + "\"", StringifyInt(speed));
1032 } else {
1033 format_metric("cvmfs_speed", "gauge",
1034 "Shows the average download speed.",
1035 "repo=\"" + fqrn + "\"", "0");
1036 }
1037
1038 // Uptime calculation
1039 const time_t now = time(NULL);
1040 const uint64_t uptime_seconds = now - cvmfs::loader_exports_->boot_time;
1041 const uint64_t mount_epoch_time = now - uptime_seconds;
1042 format_metric("cvmfs_uptime_seconds", "counter",
1043 "Shows the time since the repo was mounted.",
1044 "repo=\"" + fqrn + "\"", StringifyUint(uptime_seconds));
1045 format_metric("cvmfs_mount_epoch_timestamp", "counter",
1046 "Shows the epoch time the repo was mounted.",
1047 "repo=\"" + fqrn + "\"", StringifyUint(mount_epoch_time));
1048
1049 // Catalog expiry - access through the TalkManager's remounter member
1050 const time_t catalogs_valid_until = remounter_->catalogs_valid_until();
1051 if (catalogs_valid_until != MountPoint::kIndefiniteDeadline) {
1052 const int64_t expires_seconds = (catalogs_valid_until - now);
1053 format_metric("cvmfs_repo_expires_seconds", "gauge",
1054 "Shows the remaining life time of the mounted root file catalog in seconds.",
1055 "repo=\"" + fqrn + "\"", StringifyInt(expires_seconds));
1056 }
1057
1058 // I/O error count
1059 const uint64_t nioerr = file_system->io_error_info()->count();
1060 format_metric("cvmfs_nioerr_total", "counter",
1061 "Shows the total number of I/O errors encountered since mounting.",
1062 "repo=\"" + fqrn + "\"", StringifyUint(nioerr));
1063
1064 // Timeout information
1065 unsigned timeout_proxy, timeout_direct;
1066 mount_point.download_mgr()->GetTimeout(&timeout_proxy, &timeout_direct);
1067 format_metric("cvmfs_timeout", "gauge",
1068 "Shows the timeout for proxied connections in seconds.",
1069 "repo=\"" + fqrn + "\"", StringifyUint(timeout_proxy));
1070 format_metric("cvmfs_timeout_direct", "gauge",
1071 "Shows the timeout for direct connections in seconds.",
1072 "repo=\"" + fqrn + "\"", StringifyUint(timeout_direct));
1073
1074 // Last I/O error timestamp
1075 const int64_t timestamp_last_ioerr = file_system->io_error_info()->timestamp_last();
1076 format_metric("cvmfs_timestamp_last_ioerr", "counter",
1077 "Shows the timestamp of the last ioerror.",
1078 "repo=\"" + fqrn + "\"", StringifyInt(timestamp_last_ioerr));
1079
1080 // CPU usage from /proc/pid/stat
1081 const pid_t pid = cvmfs::pid_;
1082 const string proc_stat_path = "/proc/" + StringifyInt(pid) + "/stat";
1083 FILE *stat_file = fopen(proc_stat_path.c_str(), "r");
1084 if (stat_file) {
1085 char stat_line[1024];
1086 if (fgets(stat_line, sizeof(stat_line), stat_file)) {
1087 vector<string> stat_fields = SplitString(string(stat_line), ' ');
1088 if (stat_fields.size() > 15) {
1089 const uint64_t utime = String2Uint64(stat_fields[13]);
1090 const uint64_t stime = String2Uint64(stat_fields[14]);
1091 const long clock_tick = sysconf(_SC_CLK_TCK);
1092 if (clock_tick > 0) {
1093 const double user_seconds = static_cast<double>(utime) / clock_tick;
1094 const double system_seconds = static_cast<double>(stime) / clock_tick;
1095 format_metric("cvmfs_cpu_user_total", "counter",
1096 "CPU time used in userspace by CVMFS mount in seconds.",
1097 "repo=\"" + fqrn + "\"", StringifyDouble(user_seconds));
1098 format_metric("cvmfs_cpu_system_total", "counter",
1099 "CPU time used in the kernel system calls by CVMFS mount in seconds.",
1100 "repo=\"" + fqrn + "\"", StringifyDouble(system_seconds));
1101 }
1102 }
1103 }
1104 fclose(stat_file);
1105 }
1106
1107 // File descriptor and directory counts
1108 format_metric("cvmfs_usedfd", "gauge",
1109 "Shows the number of open directories currently used by file system clients.",
1110 "repo=\"" + fqrn + "\"",
1111 file_system->no_open_files()->ToString());
1112 format_metric("cvmfs_useddirp", "gauge",
1113 "Shows the number of file descriptors currently issued to file system clients.",
1114 "repo=\"" + fqrn + "\"",
1115 file_system->no_open_dirs()->ToString());
1116 format_metric("cvmfs_ndiropen", "gauge",
1117 "Shows the overall number of opened directories.",
1118 "repo=\"" + fqrn + "\"",
1119 file_system->n_fs_dir_open()->ToString());
1120
1121 // Inode max
1122 format_metric("cvmfs_inode_max", "gauge",
1123 "Shows the highest possible inode with the current set of loaded catalogs.",
1124 "repo=\"" + fqrn + "\"",
1125 StringifyInt(mount_point.inode_annotation()->GetGeneration() +
1126 mount_point.catalog_mgr()->inode_gauge()));
1127
1128 // Process ID
1129 format_metric("cvmfs_pid", "gauge",
1130 "Shows the process id of the CernVM-FS Fuse process.",
1131 "repo=\"" + fqrn + "\"", StringifyInt(pid));
1132
1133 // Catalog count
1134 const int n_catalogs = mount_point.catalog_mgr()->GetNumCatalogs();
1135 format_metric("cvmfs_nclg", "gauge",
1136 "Shows the number of currently loaded nested catalogs.",
1137 "repo=\"" + fqrn + "\"", StringifyInt(n_catalogs));
1138
1139 // Cleanup rate (24 hours)
1140 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
1141 const uint64_t period_s = 24 * 60 * 60;
1142 const uint64_t cleanup_rate = quota_mgr->GetCleanupRate(period_s);
1143 format_metric("cvmfs_ncleanup24", "gauge",
1144 "Shows the number of cache cleanups in the last 24 hours.",
1145 "repo=\"" + fqrn + "\"", StringifyUint(cleanup_rate));
1146 } else {
1147 format_metric("cvmfs_ncleanup24", "gauge",
1148 "Shows the number of cache cleanups in the last 24 hours.",
1149 "repo=\"" + fqrn + "\"", "-1");
1150 }
1151
1152 // Active proxy
1153 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
1154 unsigned current_group;
1155 mount_point.download_mgr()->GetProxyInfo(&proxy_chain, &current_group, NULL);
1156 string active_proxy = "DIRECT";
1157 if (proxy_chain.size() > 0 && current_group < proxy_chain.size() &&
1158 proxy_chain[current_group].size() > 0) {
1159 active_proxy = proxy_chain[current_group][0].url;
1160 }
1161 format_metric("cvmfs_active_proxy", "gauge",
1162 "Shows the active proxy in use for this mount.",
1163 "repo=\"" + fqrn + "\",proxy=\"" + active_proxy + "\"", "1");
1164
1165 // Proxy list metrics
1166 for (unsigned int i = 0; i < proxy_chain.size(); i++) {
1167 for (unsigned int j = 0; j < proxy_chain[i].size(); j++) {
1168 format_metric("cvmfs_proxy", "gauge",
1169 "Shows all registered proxies for this repository.",
1170 "repo=\"" + fqrn + "\",group=\"" + StringifyInt(i) +
1171 "\",url=\"" + proxy_chain[i][j].url + "\"", "1");
1172 }
1173 }
1174
1175 // Internal affairs metrics (excluding histograms)
1176
1177 // Update string counters manually (same as internal affairs does)
1178 mount_point.statistics()->Lookup("pathstring.n_instances")->Set(PathString::num_instances());
1179 mount_point.statistics()->Lookup("pathstring.n_overflows")->Set(PathString::num_overflows());
1180 mount_point.statistics()->Lookup("namestring.n_instances")->Set(NameString::num_instances());
1181 mount_point.statistics()->Lookup("namestring.n_overflows")->Set(NameString::num_overflows());
1182 mount_point.statistics()->Lookup("linkstring.n_instances")->Set(LinkString::num_instances());
1183 mount_point.statistics()->Lookup("linkstring.n_overflows")->Set(LinkString::num_overflows());
1184
1185 // String statistics
1186 const int64_t pathstring_instances = mount_point.statistics()->Lookup("pathstring.n_instances")->Get();
1187 const int64_t pathstring_overflows = mount_point.statistics()->Lookup("pathstring.n_overflows")->Get();
1188 const int64_t namestring_instances = mount_point.statistics()->Lookup("namestring.n_instances")->Get();
1189 const int64_t namestring_overflows = mount_point.statistics()->Lookup("namestring.n_overflows")->Get();
1190 const int64_t linkstring_instances = mount_point.statistics()->Lookup("linkstring.n_instances")->Get();
1191 const int64_t linkstring_overflows = mount_point.statistics()->Lookup("linkstring.n_overflows")->Get();
1192
1193 format_metric("cvmfs_pathstring_instances", "gauge",
1194 "Number of PathString instances.",
1195 "repo=\"" + fqrn + "\"", StringifyInt(pathstring_instances));
1196 format_metric("cvmfs_pathstring_overflows", "counter",
1197 "Number of PathString overflows.",
1198 "repo=\"" + fqrn + "\"", StringifyInt(pathstring_overflows));
1199 format_metric("cvmfs_namestring_instances", "gauge",
1200 "Number of NameString instances.",
1201 "repo=\"" + fqrn + "\"", StringifyInt(namestring_instances));
1202 format_metric("cvmfs_namestring_overflows", "counter",
1203 "Number of NameString overflows.",
1204 "repo=\"" + fqrn + "\"", StringifyInt(namestring_overflows));
1205 format_metric("cvmfs_linkstring_instances", "gauge",
1206 "Number of LinkString instances.",
1207 "repo=\"" + fqrn + "\"", StringifyInt(linkstring_instances));
1208 format_metric("cvmfs_linkstring_overflows", "counter",
1209 "Number of LinkString overflows.",
1210 "repo=\"" + fqrn + "\"", StringifyInt(linkstring_overflows));
1211
1212 // Tracker statistics (same as internal affairs does)
1213 glue::InodeTracker::Statistics inode_stats = mount_point.inode_tracker()->GetStatistics();
1214 const glue::DentryTracker::Statistics dentry_stats = mount_point.dentry_tracker()->GetStatistics();
1215 const glue::PageCacheTracker::Statistics page_cache_stats = mount_point.page_cache_tracker()->GetStatistics();
1216
1217 // Update statistics manually
1218 mount_point.statistics()->Lookup("inode_tracker.n_insert")->Set(atomic_read64(&inode_stats.num_inserts));
1219 mount_point.statistics()->Lookup("inode_tracker.n_remove")->Set(atomic_read64(&inode_stats.num_removes));
1220 mount_point.statistics()->Lookup("inode_tracker.no_reference")->Set(atomic_read64(&inode_stats.num_references));
1221 mount_point.statistics()->Lookup("inode_tracker.n_hit_inode")->Set(atomic_read64(&inode_stats.num_hits_inode));
1222 mount_point.statistics()->Lookup("inode_tracker.n_hit_path")->Set(atomic_read64(&inode_stats.num_hits_path));
1223 mount_point.statistics()->Lookup("inode_tracker.n_miss_path")->Set(atomic_read64(&inode_stats.num_misses_path));
1224 mount_point.statistics()->Lookup("dentry_tracker.n_insert")->Set(dentry_stats.num_insert);
1225 mount_point.statistics()->Lookup("dentry_tracker.n_remove")->Set(dentry_stats.num_remove);
1226 mount_point.statistics()->Lookup("dentry_tracker.n_prune")->Set(dentry_stats.num_prune);
1227 mount_point.statistics()->Lookup("page_cache_tracker.n_insert")->Set(page_cache_stats.n_insert);
1228 mount_point.statistics()->Lookup("page_cache_tracker.n_remove")->Set(page_cache_stats.n_remove);
1229 mount_point.statistics()->Lookup("page_cache_tracker.n_open_direct")->Set(page_cache_stats.n_open_direct);
1230 mount_point.statistics()->Lookup("page_cache_tracker.n_open_flush")->Set(page_cache_stats.n_open_flush);
1231 mount_point.statistics()->Lookup("page_cache_tracker.n_open_cached")->Set(page_cache_stats.n_open_cached);
1232
1233 // Inode tracker metrics
1234 format_metric("cvmfs_inode_tracker_inserts_total", "counter",
1235 "Number of inode tracker insertions.",
1236 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_inserts)));
1237 format_metric("cvmfs_inode_tracker_removes_total", "counter",
1238 "Number of inode tracker removals.",
1239 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_removes)));
1240 format_metric("cvmfs_inode_tracker_references", "gauge",
1241 "Number of inode tracker references.",
1242 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_references)));
1243 format_metric("cvmfs_inode_tracker_hits_inode_total", "counter",
1244 "Number of inode tracker inode hits.",
1245 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_hits_inode)));
1246 format_metric("cvmfs_inode_tracker_hits_path_total", "counter",
1247 "Number of inode tracker path hits.",
1248 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_hits_path)));
1249 format_metric("cvmfs_inode_tracker_misses_path_total", "counter",
1250 "Number of inode tracker path misses.",
1251 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_misses_path)));
1252
1253 // Dentry tracker metrics
1254 format_metric("cvmfs_dentry_tracker_inserts_total", "counter",
1255 "Number of dentry tracker insertions.",
1256 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_insert));
1257 format_metric("cvmfs_dentry_tracker_removes_total", "counter",
1258 "Number of dentry tracker removals.",
1259 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_remove));
1260 format_metric("cvmfs_dentry_tracker_prunes_total", "counter",
1261 "Number of dentry tracker prunes.",
1262 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_prune));
1263
1264 // Page cache tracker metrics
1265 format_metric("cvmfs_page_cache_tracker_inserts_total", "counter",
1266 "Number of page cache tracker insertions.",
1267 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_insert));
1268 format_metric("cvmfs_page_cache_tracker_removes_total", "counter",
1269 "Number of page cache tracker removals.",
1270 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_remove));
1271 format_metric("cvmfs_page_cache_tracker_opens_direct_total", "counter",
1272 "Number of page cache tracker direct opens.",
1273 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_direct));
1274 format_metric("cvmfs_page_cache_tracker_opens_flush_total", "counter",
1275 "Number of page cache tracker flush opens.",
1276 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_flush));
1277 format_metric("cvmfs_page_cache_tracker_opens_cached_total", "counter",
1278 "Number of page cache tracker cached opens.",
1279 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_cached));
1280
1281 // Cache mode information
1282 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
1283 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(file_system->cache_mgr());
1284 int cache_mode_value = 0;
1285 switch (cache_mgr->cache_mode()) {
1286 case PosixCacheManager::kCacheReadWrite:
1287 cache_mode_value = 1;
1288 break;
1289 case PosixCacheManager::kCacheReadOnly:
1290 cache_mode_value = 2;
1291 break;
1292 default:
1293 cache_mode_value = 0;
1294 }
1295 format_metric("cvmfs_cache_mode", "gauge",
1296 "Cache mode (0=unknown, 1=read-write, 2=read-only).",
1297 "repo=\"" + fqrn + "\"", StringifyInt(cache_mode_value));
1298 }
1299
1300 // Drainout and maintenance mode
1301 bool drainout_mode;
1302 bool maintenance_mode;
1303 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
1304 format_metric("cvmfs_drainout_mode", "gauge",
1305 "Drainout mode status (0=false, 1=true).",
1306 "repo=\"" + fqrn + "\"", StringifyInt(drainout_mode ? 1 : 0));
1307 format_metric("cvmfs_maintenance_mode", "gauge",
1308 "Maintenance mode status (0=false, 1=true).",
1309 "repo=\"" + fqrn + "\"", StringifyInt(maintenance_mode ? 1 : 0));
1310
1311 // SQLite statistics
1312 int current, highwater;
1313
1314 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
1315 format_metric("cvmfs_sqlite_malloc_count", "gauge",
1316 "Number of SQLite allocations.",
1317 "repo=\"" + fqrn + "\"", StringifyInt(current));
1318
1319 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
1320 format_metric("cvmfs_sqlite_memory_used_bytes", "gauge",
1321 "SQLite general purpose allocator memory used.",
1322 "repo=\"" + fqrn + "\"", StringifyInt(current));
1323 format_metric("cvmfs_sqlite_memory_used_highwater_bytes", "gauge",
1324 "SQLite general purpose allocator memory used high water mark.",
1325 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1326
1327 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
1328 format_metric("cvmfs_sqlite_largest_malloc_bytes", "gauge",
1329 "SQLite largest malloc size.",
1330 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1331
1332 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
1333 format_metric("cvmfs_sqlite_pagecache_used", "gauge",
1334 "SQLite page cache allocations used.",
1335 "repo=\"" + fqrn + "\"", StringifyInt(current));
1336 format_metric("cvmfs_sqlite_pagecache_used_highwater", "gauge",
1337 "SQLite page cache allocations used high water mark.",
1338 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1339
1340 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
1341 format_metric("cvmfs_sqlite_pagecache_overflow_bytes", "gauge",
1342 "SQLite page cache overflow bytes.",
1343 "repo=\"" + fqrn + "\"", StringifyInt(current));
1344 format_metric("cvmfs_sqlite_pagecache_overflow_highwater_bytes", "gauge",
1345 "SQLite page cache overflow bytes high water mark.",
1346 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1347
1348 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
1349 format_metric("cvmfs_sqlite_largest_pagecache_bytes", "gauge",
1350 "SQLite largest page cache allocation size.",
1351 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1352
1353 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
1354 format_metric("cvmfs_sqlite_scratch_used", "gauge",
1355 "SQLite scratch allocations used.",
1356 "repo=\"" + fqrn + "\"", StringifyInt(current));
1357 format_metric("cvmfs_sqlite_scratch_used_highwater", "gauge",
1358 "SQLite scratch allocations used high water mark.",
1359 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1360
1361 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
1362 format_metric("cvmfs_sqlite_scratch_overflow", "gauge",
1363 "SQLite scratch overflows.",
1364 "repo=\"" + fqrn + "\"", StringifyInt(current));
1365 format_metric("cvmfs_sqlite_scratch_overflow_highwater", "gauge",
1366 "SQLite scratch overflows high water mark.",
1367 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1368
1369 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
1370 format_metric("cvmfs_sqlite_largest_scratch_bytes", "gauge",
1371 "SQLite largest scratch allocation size.",
1372 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1373
1374 // NFS statistics (if applicable)
1375 if (file_system->IsNfsSource()) {
1376 format_metric("cvmfs_nfs_mode", "gauge",
1377 "NFS mode enabled (1=true, 0=false).",
1378 "repo=\"" + fqrn + "\"", "1");
1379 // Note: NFS map statistics are complex strings, skipping detailed parsing for now
1380 } else {
1381 format_metric("cvmfs_nfs_mode", "gauge",
1382 "NFS mode enabled (1=true, 0=false).",
1383 "repo=\"" + fqrn + "\"", "0");
1384 }
1385
1386 return result;
1387 }
1388
1389 TalkManager::TalkManager(const string &socket_path,
1390 MountPoint *mount_point,
1391 FuseRemounter *remounter)
1392 : socket_path_(socket_path)
1393 , socket_fd_(-1)
1394 , mount_point_(mount_point)
1395 , remounter_(remounter)
1396 , spawned_(false) {
1397 memset(&thread_talk_, 0, sizeof(thread_talk_));
1398 }
1399
1400
1401 TalkManager::~TalkManager() {
1402 if (!socket_path_.empty()) {
1403 const int retval = unlink(socket_path_.c_str());
1404 if ((retval != 0) && (errno != ENOENT)) {
1405 LogCvmfs(kLogTalk, kLogSyslogWarn,
1406 "Could not remove cvmfs_io socket from cache directory (%d)",
1407 errno);
1408 }
1409 }
1410
1411 if (socket_fd_ >= 0) {
1412 shutdown(socket_fd_, SHUT_RDWR);
1413 close(socket_fd_);
1414 }
1415
1416 if (spawned_) {
1417 pthread_join(thread_talk_, NULL);
1418 LogCvmfs(kLogTalk, kLogDebug, "talk thread stopped");
1419 }
1420 }
1421
1422
1423 void TalkManager::Spawn() {
1424 const int retval = pthread_create(&thread_talk_, NULL, MainResponder, this);
1425 assert(retval == 0);
1426 spawned_ = true;
1427 }
1428