GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/talk.cc
Date: 2025-08-31 02:39:21
Exec Total Coverage
Lines: 0 898 0.0%
Branches: 0 2914 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * Implements a socket interface to cvmfs. This way commands can be send
5 * to cvmfs. When cvmfs is running, the socket
6 * /var/cache/cvmfs2/$INSTANCE/cvmfs_io
7 * is available for command input and reply messages, resp.
8 *
9 * Cvmfs comes with the cvmfs_talk script, that handles writing and reading the
10 * socket.
11 *
12 * The talk module runs in a separate thread.
13 */
14
15 #ifndef __STDC_FORMAT_MACROS
16 #define __STDC_FORMAT_MACROS
17 #endif
18
19
20 #include "talk.h"
21
22 #include <errno.h>
23 #include <inttypes.h>
24 #include <pthread.h>
25 #include <stdint.h>
26 #include <sys/socket.h>
27 #include <sys/stat.h>
28 #include <sys/types.h>
29 #include <sys/uio.h>
30 #include <sys/un.h>
31 #include <unistd.h>
32
33 #include <cassert>
34 #include <cstdlib>
35 #include <cstring>
36 #include <string>
37 #include <vector>
38
39 #include "cache.h"
40 #include "cache_posix.h"
41 #include "catalog_mgr_client.h"
42 #include "cvmfs.h"
43 #include "duplex_sqlite3.h"
44 #include "fuse_remount.h"
45 #include "glue_buffer.h"
46 #include "loader.h"
47 #include "lru_md.h"
48 #include "monitor.h"
49 #include "mountpoint.h"
50 #include "network/download.h"
51 #include "nfs_maps.h"
52 #include "options.h"
53 #include "quota.h"
54 #include "shortstring.h"
55 #include "statistics.h"
56 #include "tracer.h"
57 #include "util/logging.h"
58 #include "util/platform.h"
59 #include "util/pointer.h"
60 #include "wpad.h"
61
62 using namespace std; // NOLINT
63
64
65
66
67 void TalkManager::Answer(int con_fd, const string &msg) {
68 (void)send(con_fd, &msg[0], msg.length(), MSG_NOSIGNAL);
69 }
70
71
72 void TalkManager::AnswerStringList(int con_fd, const vector<string> &list) {
73 string list_str;
74 for (unsigned i = 0; i < list.size(); ++i) {
75 list_str += list[i] + "\n";
76 }
77 Answer(con_fd, list_str);
78 }
79
80
81 TalkManager *TalkManager::Create(const string &socket_path,
82 MountPoint *mount_point,
83 FuseRemounter *remounter) {
84 UniquePtr<TalkManager> talk_manager(
85 new TalkManager(socket_path, mount_point, remounter));
86
87 talk_manager->socket_fd_ = MakeSocket(socket_path, 0660);
88 if (talk_manager->socket_fd_ == -1)
89 return NULL;
90 if (listen(talk_manager->socket_fd_, 1) == -1)
91 return NULL;
92
93 LogCvmfs(kLogTalk, kLogDebug, "socket created at %s (fd %d)",
94 socket_path.c_str(), talk_manager->socket_fd_);
95
96 return talk_manager.Release();
97 }
98
99
100 string TalkManager::FormatMetalinkInfo(
101 download::DownloadManager *download_mgr) {
102 vector<string> metalink_chain;
103 unsigned active_metalink;
104
105 download_mgr->GetMetalinkInfo(&metalink_chain, &active_metalink);
106 if (metalink_chain.size() == 0)
107 return "No metalinks defined\n";
108
109 string metalink_str;
110 for (unsigned i = 0; i < metalink_chain.size(); ++i) {
111 metalink_str += " [" + StringifyInt(i) + "] " + metalink_chain[i] + "\n";
112 }
113 metalink_str += "Active metalink " + StringifyInt(active_metalink) + ": "
114 + metalink_chain[active_metalink] + "\n";
115 return metalink_str;
116 }
117
118 string TalkManager::FormatHostInfo(download::DownloadManager *download_mgr) {
119 vector<string> host_chain;
120 vector<int> rtt;
121 unsigned active_host;
122
123 download_mgr->GetHostInfo(&host_chain, &rtt, &active_host);
124 if (host_chain.size() == 0)
125 return "No hosts defined\n";
126
127 string host_str;
128 for (unsigned i = 0; i < host_chain.size(); ++i) {
129 host_str += " [" + StringifyInt(i) + "] " + host_chain[i] + " (";
130 if (rtt[i] == download::DownloadManager::kProbeUnprobed)
131 host_str += "unprobed";
132 else if (rtt[i] == download::DownloadManager::kProbeDown)
133 host_str += "host down";
134 else if (rtt[i] == download::DownloadManager::kProbeGeo)
135 host_str += "geographically ordered";
136 else
137 host_str += StringifyInt(rtt[i]) + " ms";
138 host_str += ")\n";
139 }
140 host_str += "Active host " + StringifyInt(active_host) + ": "
141 + host_chain[active_host] + "\n";
142 return host_str;
143 }
144
145 string TalkManager::FormatProxyInfo(download::DownloadManager *download_mgr) {
146 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
147 unsigned active_group;
148 unsigned fallback_group;
149
150 download_mgr->GetProxyInfo(&proxy_chain, &active_group, &fallback_group);
151 string proxy_str;
152 if (proxy_chain.size()) {
153 proxy_str += "Load-balance groups:\n";
154 for (unsigned i = 0; i < proxy_chain.size(); ++i) {
155 vector<string> urls;
156 for (unsigned j = 0; j < proxy_chain[i].size(); ++j) {
157 urls.push_back(proxy_chain[i][j].Print());
158 }
159 proxy_str += "[" + StringifyInt(i) + "] " + JoinStrings(urls, ", ")
160 + "\n";
161 }
162 proxy_str += "Active proxy: [" + StringifyInt(active_group) + "] "
163 + proxy_chain[active_group][0].url + "\n";
164 if (fallback_group < proxy_chain.size())
165 proxy_str += "First fallback group: [" + StringifyInt(fallback_group)
166 + "]\n";
167 } else {
168 proxy_str = "No proxies defined\n";
169 }
170 return proxy_str;
171 }
172
173
174 /**
175 * Listener thread on the socket.
176 * TODO(jblomer): create Format... helpers to shorten this method
177 */
178 void *TalkManager::MainResponder(void *data) {
179 TalkManager *talk_mgr = reinterpret_cast<TalkManager *>(data);
180 MountPoint *mount_point = talk_mgr->mount_point_;
181 FileSystem *file_system = mount_point->file_system();
182 FuseRemounter *remounter = talk_mgr->remounter_;
183 LogCvmfs(kLogTalk, kLogDebug, "talk thread started");
184
185 struct sockaddr_un remote;
186 socklen_t socket_size = sizeof(remote);
187 int con_fd = -1;
188 while (true) {
189 if (con_fd >= 0) {
190 shutdown(con_fd, SHUT_RDWR);
191 close(con_fd);
192 }
193 LogCvmfs(kLogTalk, kLogDebug, "accepting connections on socketfd %d",
194 talk_mgr->socket_fd_);
195 if ((con_fd = accept(talk_mgr->socket_fd_, (struct sockaddr *)&remote,
196 &socket_size))
197 < 0) {
198 LogCvmfs(kLogTalk, kLogDebug, "terminating talk thread (fd %d, errno %d)",
199 con_fd, errno);
200 break;
201 }
202
203 char buf[kMaxCommandSize];
204 int bytes_read;
205 if ((bytes_read = recv(con_fd, buf, sizeof(buf), 0)) <= 0)
206 continue;
207
208 if (buf[bytes_read - 1] == '\0')
209 bytes_read--;
210 const string line = string(buf, bytes_read);
211 LogCvmfs(kLogTalk, kLogDebug, "received %s (length %lu)", line.c_str(),
212 line.length());
213
214 if (line == "tracebuffer flush") {
215 mount_point->tracer()->Flush();
216 talk_mgr->Answer(con_fd, "OK\n");
217 } else if (line == "cache size") {
218 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
219 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
220 talk_mgr->Answer(con_fd, "Cache cannot report its size\n");
221 } else {
222 const uint64_t size_unpinned = quota_mgr->GetSize();
223 const uint64_t size_pinned = quota_mgr->GetSizePinned();
224 const string size_str = "Current cache size is "
225 + StringifyInt(size_unpinned / (1024 * 1024))
226 + "MB (" + StringifyInt(size_unpinned)
227 + " Bytes), pinned: "
228 + StringifyInt(size_pinned / (1024 * 1024))
229 + "MB (" + StringifyInt(size_pinned)
230 + " Bytes)\n";
231 talk_mgr->Answer(con_fd, size_str);
232 }
233 } else if (line == "cache instance") {
234 talk_mgr->Answer(con_fd, file_system->cache_mgr()->Describe());
235 } else if (line == "cache list") {
236 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
237 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
238 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
239 } else {
240 const vector<string> ls = quota_mgr->List();
241 talk_mgr->AnswerStringList(con_fd, ls);
242 }
243 } else if (line == "cache list pinned") {
244 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
245 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
246 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
247 } else {
248 const vector<string> ls_pinned = quota_mgr->ListPinned();
249 talk_mgr->AnswerStringList(con_fd, ls_pinned);
250 }
251 } else if (line == "cache list catalogs") {
252 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
253 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
254 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
255 } else {
256 const vector<string> ls_catalogs = quota_mgr->ListCatalogs();
257 talk_mgr->AnswerStringList(con_fd, ls_catalogs);
258 }
259 } else if (line.substr(0, 12) == "cleanup rate") {
260 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
261 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
262 talk_mgr->Answer(con_fd, "Unsupported by this cache\n");
263 } else {
264 if (line.length() < 14) {
265 talk_mgr->Answer(con_fd, "Usage: cleanup rate <period in mn>\n");
266 } else {
267 const uint64_t period_s = String2Uint64(line.substr(13)) * 60;
268 const uint64_t rate = quota_mgr->GetCleanupRate(period_s);
269 talk_mgr->Answer(con_fd, StringifyInt(rate) + "\n");
270 }
271 }
272 } else if (line.substr(0, 15) == "cache limit set") {
273 if (line.length() < 16) {
274 talk_mgr->Answer(con_fd, "Usage: cache limit set <MB>\n");
275 } else {
276 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
277 const uint64_t size = String2Uint64(line.substr(16));
278 if (size < 1000) {
279 talk_mgr->Answer(con_fd, "New limit too low (minimum 1000)\n");
280 } else {
281 if (quota_mgr->SetLimit(size * 1024 * 1024)) {
282 file_system->options_mgr()->SetValueFromTalk("CVMFS_QUOTA_LIMIT",
283 StringifyUint(size));
284 talk_mgr->Answer(con_fd, "OK\n");
285 } else {
286 talk_mgr->Answer(con_fd, "Limit not reset\n");
287 }
288 }
289 }
290 } else if (line == "cache limit get") {
291 std::string limit_from_options;
292 file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT",
293 &limit_from_options);
294 talk_mgr->Answer(con_fd, limit_from_options + "\n");
295 } else if (line.substr(0, 7) == "cleanup") {
296 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
297 if (!quota_mgr->HasCapability(QuotaManager::kCapShrink)) {
298 talk_mgr->Answer(con_fd, "Cache cannot trigger eviction\n");
299 } else {
300 if (line.length() < 9) {
301 talk_mgr->Answer(con_fd, "Usage: cleanup <MB>\n");
302 } else {
303 const uint64_t size = String2Uint64(line.substr(8)) * 1024 * 1024;
304 if (quota_mgr->Cleanup(size)) {
305 talk_mgr->Answer(con_fd, "OK\n");
306 } else {
307 talk_mgr->Answer(con_fd, "Not fully cleaned "
308 "(there might be pinned chunks)\n");
309 }
310 }
311 }
312 } else if (line.substr(0, 5) == "evict") {
313 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
314 if (line.length() < 7) {
315 talk_mgr->Answer(con_fd, "Usage: evict <path>\n");
316 } else {
317 const string path = line.substr(6);
318 const bool found_regular = cvmfs::Evict(path);
319 if (found_regular)
320 talk_mgr->Answer(con_fd, "OK\n");
321 else
322 talk_mgr->Answer(con_fd, "No such regular file\n");
323 }
324 } else if (line.substr(0, 3) == "pin") {
325 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
326 if (line.length() < 5) {
327 talk_mgr->Answer(con_fd, "Usage: pin <path>\n");
328 } else {
329 const string path = line.substr(4);
330 const bool found_regular = cvmfs::Pin(path);
331 if (found_regular)
332 talk_mgr->Answer(con_fd, "OK\n");
333 else
334 talk_mgr->Answer(con_fd, "No such regular file or pinning failed\n");
335 }
336 } else if (line == "mountpoint") {
337 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->mount_point + "\n");
338 } else if (line == "device id") {
339 if (cvmfs::loader_exports_->version >= 5)
340 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->device_id + "\n");
341 else
342 talk_mgr->Answer(con_fd, "0:0\n");
343 } else if (line.substr(0, 13) == "send mount fd") {
344 // Hidden command intended to be used only by the cvmfs mount helper
345 if (line.length() < 15) {
346 talk_mgr->Answer(con_fd, "EINVAL\n");
347 } else {
348 const std::string socket_path = line.substr(14);
349 const bool retval = cvmfs::SendFuseFd(socket_path);
350 talk_mgr->Answer(con_fd, retval ? "OK\n" : "Failed\n");
351 LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
352 "Attempt to send fuse connection info to new mount (via %s)%s",
353 socket_path.c_str(), retval ? "" : " -- failed!");
354 }
355 } else if (line.substr(0, 7) == "remount") {
356 FuseRemounter::Status status;
357 if (line == "remount sync")
358 status = remounter->CheckSynchronously();
359 else
360 status = remounter->Check();
361 switch (status) {
362 case FuseRemounter::kStatusFailGeneral:
363 talk_mgr->Answer(con_fd, "Failed\n");
364 break;
365 case FuseRemounter::kStatusFailNoSpace:
366 talk_mgr->Answer(con_fd, "Failed (no space)\n");
367 break;
368 case FuseRemounter::kStatusUp2Date:
369 talk_mgr->Answer(con_fd, "Catalog up to date\n");
370 break;
371 case FuseRemounter::kStatusDraining:
372 talk_mgr->Answer(con_fd, "New revision applied\n");
373 break;
374 case FuseRemounter::kStatusMaintenance:
375 talk_mgr->Answer(con_fd, "In maintenance mode\n");
376 break;
377 default:
378 talk_mgr->Answer(con_fd, "internal error\n");
379 }
380 } else if (line.substr(0, 6) == "chroot") {
381 if (line.length() < 8) {
382 talk_mgr->Answer(con_fd, "Usage: chroot <hash>\n");
383 } else {
384 const std::string root_hash = Trim(line.substr(7),
385 true /* trim_newline */);
386 const FuseRemounter::Status status = remounter->ChangeRoot(
387 MkFromHexPtr(shash::HexPtr(root_hash), shash::kSuffixCatalog));
388 switch (status) {
389 case FuseRemounter::kStatusUp2Date:
390 talk_mgr->Answer(con_fd, "OK\n");
391 break;
392 default:
393 talk_mgr->Answer(con_fd, "Failed\n");
394 break;
395 }
396 }
397 } else if (line == "detach nested catalogs") {
398 mount_point->catalog_mgr()->DetachNested();
399 talk_mgr->Answer(con_fd, "OK\n");
400 } else if (line == "revision") {
401 const string revision = StringifyInt(
402 mount_point->catalog_mgr()->GetRevision());
403 talk_mgr->Answer(con_fd, revision + "\n");
404 } else if (line == "max ttl info") {
405 const unsigned max_ttl = mount_point->GetMaxTtlMn();
406 if (max_ttl == 0) {
407 talk_mgr->Answer(con_fd, "unset\n");
408 } else {
409 const string max_ttl_str = StringifyInt(max_ttl) + " minutes\n";
410 talk_mgr->Answer(con_fd, max_ttl_str);
411 }
412 } else if (line.substr(0, 11) == "max ttl set") {
413 if (line.length() < 13) {
414 talk_mgr->Answer(con_fd, "Usage: max ttl set <minutes>\n");
415 } else {
416 const unsigned max_ttl = String2Uint64(line.substr(12));
417 mount_point->SetMaxTtlMn(max_ttl);
418 talk_mgr->Answer(con_fd, "OK\n");
419 }
420 } else if (line.substr(0, 14) == "nameserver get") {
421 const string dns_server = mount_point->download_mgr()->GetDnsServer();
422 const string reply = !dns_server.empty()
423 ? std::string("DNS server address: ")
424 + dns_server + "\n"
425 : std::string("DNS server not set.\n");
426 talk_mgr->Answer(con_fd, reply);
427 } else if (line.substr(0, 14) == "nameserver set") {
428 if (line.length() < 16) {
429 talk_mgr->Answer(con_fd, "Usage: nameserver set <host>\n");
430 } else {
431 const string host = line.substr(15);
432 mount_point->download_mgr()->SetDnsServer(host);
433 talk_mgr->Answer(con_fd, "OK\n");
434 }
435 } else if (line.substr(0, 22) == "__testing_freeze_cvmfs") {
436 const std::string fs_dir = line.substr(23) + "/dir";
437 mkdir(fs_dir.c_str(), 0700);
438 } else if (line == "external metalink info") {
439 const string external_metalink_info = talk_mgr->FormatMetalinkInfo(
440 mount_point->external_download_mgr());
441 talk_mgr->Answer(con_fd, external_metalink_info);
442 } else if (line == "metalink info") {
443 const string metalink_info = talk_mgr->FormatMetalinkInfo(
444 mount_point->download_mgr());
445 talk_mgr->Answer(con_fd, metalink_info);
446 } else if (line == "external host info") {
447 const string external_host_info = talk_mgr->FormatHostInfo(
448 mount_point->external_download_mgr());
449 talk_mgr->Answer(con_fd, external_host_info);
450 } else if (line == "host info") {
451 const string host_info = talk_mgr->FormatHostInfo(
452 mount_point->download_mgr());
453 talk_mgr->Answer(con_fd, host_info);
454 } else if (line == "host probe") {
455 mount_point->download_mgr()->ProbeHosts();
456 talk_mgr->Answer(con_fd, "OK\n");
457 } else if (line == "host probe geo") {
458 const bool retval = mount_point->download_mgr()->ProbeGeo();
459 if (retval)
460 talk_mgr->Answer(con_fd, "OK\n");
461 else
462 talk_mgr->Answer(con_fd, "Failed\n");
463 } else if (line == "external metalink switch") {
464 mount_point->external_download_mgr()->SwitchMetalink();
465 talk_mgr->Answer(con_fd, "OK\n");
466 } else if (line == "metalink switch") {
467 mount_point->download_mgr()->SwitchMetalink();
468 talk_mgr->Answer(con_fd, "OK\n");
469 } else if (line == "external host switch") {
470 mount_point->external_download_mgr()->SwitchHost();
471 talk_mgr->Answer(con_fd, "OK\n");
472 } else if (line == "host switch") {
473 mount_point->download_mgr()->SwitchHost();
474 talk_mgr->Answer(con_fd, "OK\n");
475 } else if (line.substr(0, 21) == "external metalink set") {
476 if (line.length() < 23) {
477 talk_mgr->Answer(con_fd, "Usage: external metalink set <URL>\n");
478 } else {
479 const std::string host = line.substr(22);
480 mount_point->external_download_mgr()->SetMetalinkChain(host);
481 talk_mgr->Answer(con_fd, "OK\n");
482 }
483 } else if (line.substr(0, 12) == "metalink set") {
484 if (line.length() < 14) {
485 talk_mgr->Answer(con_fd, "Usage: metalink set <URL>\n");
486 } else {
487 const std::string host = line.substr(13);
488 mount_point->download_mgr()->SetMetalinkChain(host);
489 talk_mgr->Answer(con_fd, "OK\n");
490 }
491 } else if (line.substr(0, 17) == "external host set") {
492 if (line.length() < 19) {
493 talk_mgr->Answer(con_fd, "Usage: external host set <URL>\n");
494 } else {
495 const std::string host = line.substr(18);
496 mount_point->external_download_mgr()->SetHostChain(host);
497 talk_mgr->Answer(con_fd, "OK\n");
498 }
499 } else if (line.substr(0, 8) == "host set") {
500 if (line.length() < 10) {
501 talk_mgr->Answer(con_fd, "Usage: host set <host list>\n");
502 } else {
503 const string hosts = line.substr(9);
504 mount_point->download_mgr()->SetHostChain(hosts);
505 talk_mgr->Answer(con_fd, "OK\n");
506 }
507 } else if (line == "external proxy info") {
508 const string external_proxy_info = talk_mgr->FormatProxyInfo(
509 mount_point->external_download_mgr());
510 talk_mgr->Answer(con_fd, external_proxy_info);
511 } else if (line == "proxy info") {
512 const string proxy_info = talk_mgr->FormatProxyInfo(
513 mount_point->download_mgr());
514 talk_mgr->Answer(con_fd, proxy_info);
515 } else if (line == "proxy rebalance") {
516 mount_point->download_mgr()->RebalanceProxies();
517 talk_mgr->Answer(con_fd, "OK\n");
518 } else if (line == "proxy group switch") {
519 mount_point->download_mgr()->SwitchProxyGroup();
520 talk_mgr->Answer(con_fd, "OK\n");
521 } else if (line.substr(0, 18) == "external proxy set") {
522 if (line.length() < 20) {
523 talk_mgr->Answer(con_fd, "Usage: external proxy set <proxy list>\n");
524 } else {
525 const string external_proxies = line.substr(19);
526 mount_point->external_download_mgr()->SetProxyChain(
527 external_proxies, "", download::DownloadManager::kSetProxyRegular);
528 talk_mgr->Answer(con_fd, "OK\n");
529 }
530 } else if (line.substr(0, 9) == "proxy set") {
531 if (line.length() < 11) {
532 talk_mgr->Answer(con_fd, "Usage: proxy set <proxy list>\n");
533 } else {
534 string proxies = line.substr(10);
535 proxies = download::ResolveProxyDescription(
536 proxies, "", mount_point->download_mgr());
537 if (proxies == "") {
538 talk_mgr->Answer(con_fd, "Failed, no valid proxies\n");
539 } else {
540 mount_point->download_mgr()->SetProxyChain(
541 proxies, "", download::DownloadManager::kSetProxyRegular);
542 talk_mgr->Answer(con_fd, "OK\n");
543 }
544 }
545 } else if (line.substr(0, 14) == "proxy fallback") {
546 if (line.length() < 15) {
547 talk_mgr->Answer(con_fd, "Usage: proxy fallback <proxy list>\n");
548 } else {
549 const string fallback_proxies = line.substr(15);
550 mount_point->download_mgr()->SetProxyChain(
551 "", fallback_proxies, download::DownloadManager::kSetProxyFallback);
552 talk_mgr->Answer(con_fd, "OK\n");
553 }
554 } else if (line == "timeout info") {
555 unsigned timeout;
556 unsigned timeout_direct;
557 mount_point->download_mgr()->GetTimeout(&timeout, &timeout_direct);
558 string timeout_str = "Timeout with proxy: ";
559 if (timeout)
560 timeout_str += StringifyInt(timeout) + "s\n";
561 else
562 timeout_str += "no timeout\n";
563 timeout_str += "Timeout without proxy: ";
564 if (timeout_direct)
565 timeout_str += StringifyInt(timeout_direct) + "s\n";
566 else
567 timeout_str += "no timeout\n";
568 talk_mgr->Answer(con_fd, timeout_str);
569 } else if (line.substr(0, 11) == "timeout set") {
570 if (line.length() < 13) {
571 talk_mgr->Answer(con_fd, "Usage: timeout set <proxy> <direct>\n");
572 } else {
573 uint64_t timeout;
574 uint64_t timeout_direct;
575 String2Uint64Pair(line.substr(12), &timeout, &timeout_direct);
576 mount_point->download_mgr()->SetTimeout(timeout, timeout_direct);
577 talk_mgr->Answer(con_fd, "OK\n");
578 }
579 } else if (line == "open catalogs") {
580 talk_mgr->Answer(con_fd, mount_point->catalog_mgr()->PrintHierarchy());
581 } else if (line == "drop metadata caches") {
582 // For testing
583 mount_point->inode_cache()->Pause();
584 mount_point->path_cache()->Pause();
585 mount_point->md5path_cache()->Pause();
586 mount_point->inode_cache()->Drop();
587 mount_point->path_cache()->Drop();
588 mount_point->md5path_cache()->Drop();
589 mount_point->inode_cache()->Resume();
590 mount_point->path_cache()->Resume();
591 mount_point->md5path_cache()->Resume();
592 talk_mgr->Answer(con_fd, "OK\n");
593 } else if (line == "internal affairs") {
594 int current;
595 int highwater;
596 string result;
597
598 result += "Inode Generation:\n " + cvmfs::PrintInodeGeneration();
599
600 // Manually setting the values of the ShortString counters
601 mount_point->statistics()
602 ->Lookup("pathstring.n_instances")
603 ->Set(PathString::num_instances());
604 mount_point->statistics()
605 ->Lookup("pathstring.n_overflows")
606 ->Set(PathString::num_overflows());
607 mount_point->statistics()
608 ->Lookup("namestring.n_instances")
609 ->Set(NameString::num_instances());
610 mount_point->statistics()
611 ->Lookup("namestring.n_overflows")
612 ->Set(NameString::num_overflows());
613 mount_point->statistics()
614 ->Lookup("linkstring.n_instances")
615 ->Set(LinkString::num_instances());
616 mount_point->statistics()
617 ->Lookup("linkstring.n_overflows")
618 ->Set(LinkString::num_overflows());
619
620 // Manually setting the inode tracker numbers
621 glue::InodeTracker::Statistics inode_stats = mount_point->inode_tracker()
622 ->GetStatistics();
623 const glue::DentryTracker::Statistics
624 dentry_stats = mount_point->dentry_tracker()->GetStatistics();
625 const glue::PageCacheTracker::Statistics
626 page_cache_stats = mount_point->page_cache_tracker()->GetStatistics();
627 mount_point->statistics()
628 ->Lookup("inode_tracker.n_insert")
629 ->Set(atomic_read64(&inode_stats.num_inserts));
630 mount_point->statistics()
631 ->Lookup("inode_tracker.n_remove")
632 ->Set(atomic_read64(&inode_stats.num_removes));
633 mount_point->statistics()
634 ->Lookup("inode_tracker.no_reference")
635 ->Set(atomic_read64(&inode_stats.num_references));
636 mount_point->statistics()
637 ->Lookup("inode_tracker.n_hit_inode")
638 ->Set(atomic_read64(&inode_stats.num_hits_inode));
639 mount_point->statistics()
640 ->Lookup("inode_tracker.n_hit_path")
641 ->Set(atomic_read64(&inode_stats.num_hits_path));
642 mount_point->statistics()
643 ->Lookup("inode_tracker.n_miss_path")
644 ->Set(atomic_read64(&inode_stats.num_misses_path));
645 mount_point->statistics()
646 ->Lookup("dentry_tracker.n_insert")
647 ->Set(dentry_stats.num_insert);
648 mount_point->statistics()
649 ->Lookup("dentry_tracker.n_remove")
650 ->Set(dentry_stats.num_remove);
651 mount_point->statistics()
652 ->Lookup("dentry_tracker.n_prune")
653 ->Set(dentry_stats.num_prune);
654 mount_point->statistics()
655 ->Lookup("page_cache_tracker.n_insert")
656 ->Set(page_cache_stats.n_insert);
657 mount_point->statistics()
658 ->Lookup("page_cache_tracker.n_remove")
659 ->Set(page_cache_stats.n_remove);
660 mount_point->statistics()
661 ->Lookup("page_cache_tracker.n_open_direct")
662 ->Set(page_cache_stats.n_open_direct);
663 mount_point->statistics()
664 ->Lookup("page_cache_tracker.n_open_flush")
665 ->Set(page_cache_stats.n_open_flush);
666 mount_point->statistics()
667 ->Lookup("page_cache_tracker.n_open_cached")
668 ->Set(page_cache_stats.n_open_cached);
669
670 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
671 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(
672 file_system->cache_mgr());
673 result += "\nCache Mode: ";
674 switch (cache_mgr->cache_mode()) {
675 case PosixCacheManager::kCacheReadWrite:
676 result += "read-write";
677 break;
678 case PosixCacheManager::kCacheReadOnly:
679 result += "read-only";
680 break;
681 default:
682 result += "unknown";
683 }
684 }
685 bool drainout_mode;
686 bool maintenance_mode;
687 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
688 result += "\nDrainout Mode: " + StringifyBool(drainout_mode) + "\n";
689 result += "Maintenance Mode: " + StringifyBool(maintenance_mode) + "\n";
690
691 if (file_system->IsNfsSource()) {
692 result += "\nNFS Map Statistics:\n";
693 result += file_system->nfs_maps()->GetStatistics();
694 }
695
696 result += "SQlite Statistics:\n";
697 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
698 result += " Number of allocations " + StringifyInt(current) + "\n";
699
700 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
701 result += " General purpose allocator " + StringifyInt(current / 1024)
702 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
703
704 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
705 result += " Largest malloc " + StringifyInt(highwater) + " Bytes\n";
706
707 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
708 result += " Page cache allocations " + StringifyInt(current) + " / "
709 + StringifyInt(highwater) + "\n";
710
711 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
712 result += " Page cache overflows " + StringifyInt(current / 1024)
713 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
714
715 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
716 result += " Largest page cache allocation " + StringifyInt(highwater)
717 + " Bytes\n";
718
719 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
720 result += " Scratch allocations " + StringifyInt(current) + " / "
721 + StringifyInt(highwater) + "\n";
722
723 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
724 result += " Scratch overflows " + StringifyInt(current) + " / "
725 + StringifyInt(highwater) + "\n";
726
727 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
728 result += " Largest scratch allocation " + StringifyInt(highwater / 1024)
729 + " KB\n";
730
731 result += "\nPer-Connection Memory Statistics:\n"
732 + mount_point->catalog_mgr()->PrintAllMemStatistics();
733
734 result += "\nLatency distribution of system calls:\n";
735
736 result += "Lookup\n" + file_system->hist_fs_lookup()->ToString();
737 result += "Forget\n" + file_system->hist_fs_forget()->ToString();
738 result += "Multi-Forget\n"
739 + file_system->hist_fs_forget_multi()->ToString();
740 result += "Getattr\n" + file_system->hist_fs_getattr()->ToString();
741 result += "Readlink\n" + file_system->hist_fs_readlink()->ToString();
742 result += "Opendir\n" + file_system->hist_fs_opendir()->ToString();
743 result += "Releasedir\n" + file_system->hist_fs_releasedir()->ToString();
744 result += "Readdir\n" + file_system->hist_fs_readdir()->ToString();
745 result += "Open\n" + file_system->hist_fs_open()->ToString();
746 result += "Read\n" + file_system->hist_fs_read()->ToString();
747 result += "Release\n" + file_system->hist_fs_release()->ToString();
748
749 result += "\nRaw Counters:\n"
750 + mount_point->statistics()->PrintList(
751 perf::Statistics::kPrintHeader);
752
753 talk_mgr->Answer(con_fd, result);
754 } else if (line == "reset error counters") {
755 file_system->ResetErrorCounters();
756 talk_mgr->Answer(con_fd, "OK\n");
757 } else if (line == "pid") {
758 const string pid_str = StringifyInt(cvmfs::pid_) + "\n";
759 talk_mgr->Answer(con_fd, pid_str);
760 } else if (line == "pid cachemgr") {
761 const string pid_str = StringifyInt(file_system->cache_mgr()
762 ->quota_mgr()
763 ->GetPid())
764 + "\n";
765 talk_mgr->Answer(con_fd, pid_str);
766 } else if (line == "pid watchdog") {
767 const string pid_str = StringifyInt(Watchdog::GetPid()) + "\n";
768 talk_mgr->Answer(con_fd, pid_str);
769 } else if (line == "parameters") {
770 talk_mgr->Answer(con_fd, file_system->options_mgr()->Dump());
771 } else if (line == "hotpatch history") {
772 string history_str = StringifyTime(cvmfs::loader_exports_->boot_time,
773 true)
774 + " (start of CernVM-FS loader "
775 + cvmfs::loader_exports_->loader_version + ")\n";
776 for (loader::EventList::const_iterator
777 i = cvmfs::loader_exports_->history.begin(),
778 iEnd = cvmfs::loader_exports_->history.end();
779 i != iEnd;
780 ++i) {
781 history_str += StringifyTime((*i)->timestamp, true)
782 + " (loaded CernVM-FS Fuse Module " + (*i)->so_version
783 + ")\n";
784 }
785 talk_mgr->Answer(con_fd, history_str);
786 } else if (line == "vfs inodes") {
787 string result;
788 glue::InodeTracker::Cursor cursor(
789 mount_point->inode_tracker()->BeginEnumerate());
790 uint64_t inode;
791 while (mount_point->inode_tracker()->NextInode(&cursor, &inode)) {
792 result += StringifyInt(inode) + "\n";
793 }
794 mount_point->inode_tracker()->EndEnumerate(&cursor);
795 talk_mgr->Answer(con_fd, result);
796 } else if (line == "vfs entries") {
797 string result;
798 glue::InodeTracker::Cursor cursor(
799 mount_point->inode_tracker()->BeginEnumerate());
800 uint64_t inode_parent;
801 NameString name;
802 while (mount_point->inode_tracker()->NextEntry(&cursor, &inode_parent,
803 &name)) {
804 result += "<" + StringifyInt(inode_parent) + ">/" + name.ToString()
805 + "\n";
806 }
807 mount_point->inode_tracker()->EndEnumerate(&cursor);
808 talk_mgr->Answer(con_fd, result);
809 } else if (line == "version") {
810 const string version_str = string(CVMFS_VERSION)
811 + " (CernVM-FS Fuse Module)\n"
812 + cvmfs::loader_exports_->loader_version
813 + " (Loader)\n";
814 talk_mgr->Answer(con_fd, version_str);
815 } else if (line == "version patchlevel") {
816 talk_mgr->Answer(con_fd, string(CVMFS_PATCH_LEVEL) + "\n");
817 } else if (line == "tear down to read-only") {
818 if (file_system->cache_mgr()->id() != kPosixCacheManager) {
819 talk_mgr->Answer(con_fd, "not supported\n");
820 } else {
821 // hack
822 cvmfs::UnregisterQuotaListener();
823 file_system->TearDown2ReadOnly();
824 talk_mgr->Answer(con_fd, "In read-only mode\n");
825 }
826 } else if (line == "latency") {
827 const string result = talk_mgr->FormatLatencies(*mount_point,
828 file_system);
829 talk_mgr->Answer(con_fd, result);
830 } else if (line == "metrics prometheus") {
831 const string result = talk_mgr->FormatPrometheusMetrics(*mount_point,
832 file_system);
833 talk_mgr->Answer(con_fd, result);
834 } else {
835 talk_mgr->Answer(con_fd, "unknown command\n");
836 }
837 }
838
839 return NULL;
840 } // NOLINT(readability/fn_size)
841
842 string TalkManager::FormatLatencies(const MountPoint &mount_point,
843 FileSystem *file_system) {
844 string result;
845 const unsigned int bufSize = 300;
846 char buffer[bufSize];
847
848 vector<float> qs;
849 qs.push_back(.1);
850 qs.push_back(.2);
851 qs.push_back(.25);
852 qs.push_back(.3);
853 qs.push_back(.4);
854 qs.push_back(.5);
855 qs.push_back(.6);
856 qs.push_back(.7);
857 qs.push_back(.75);
858 qs.push_back(.8);
859 qs.push_back(.9);
860 qs.push_back(.95);
861 qs.push_back(.99);
862 qs.push_back(.999);
863 qs.push_back(.9999);
864
865 const string repo(mount_point.fqrn());
866
867 unsigned int format_index = snprintf(
868 buffer, bufSize, "\"%s\",\"%s\",\"%s\",\"%s\"", "repository", "action",
869 "total_count", "time_unit");
870 for (unsigned int i = 0; i < qs.size(); i++) {
871 format_index += snprintf(buffer + format_index, bufSize - format_index,
872 ",%0.5f", qs[i]);
873 }
874 format_index += snprintf(buffer + format_index, bufSize - format_index, "\n");
875 assert(format_index < bufSize);
876
877 result += buffer;
878 memset(buffer, 0, sizeof(buffer));
879 format_index = 0;
880
881 vector<Log2Histogram *> hist;
882 vector<string> names;
883 hist.push_back(file_system->hist_fs_lookup());
884 names.push_back("lookup");
885 hist.push_back(file_system->hist_fs_forget());
886 names.push_back("forget");
887 hist.push_back(file_system->hist_fs_forget_multi());
888 names.push_back("forget_multi");
889 hist.push_back(file_system->hist_fs_getattr());
890 names.push_back("getattr");
891 hist.push_back(file_system->hist_fs_readlink());
892 names.push_back("readlink");
893 hist.push_back(file_system->hist_fs_opendir());
894 names.push_back("opendir");
895 hist.push_back(file_system->hist_fs_releasedir());
896 names.push_back("releasedir");
897 hist.push_back(file_system->hist_fs_readdir());
898 names.push_back("readdir");
899 hist.push_back(file_system->hist_fs_open());
900 names.push_back("open");
901 hist.push_back(file_system->hist_fs_read());
902 names.push_back("read");
903 hist.push_back(file_system->hist_fs_release());
904 names.push_back("release");
905
906 for (unsigned int j = 0; j < hist.size(); j++) {
907 Log2Histogram *h = hist[j];
908 unsigned int format_index = snprintf(
909 buffer, bufSize, "\"%s\",\"%s\",%" PRIu64 ",\"nanoseconds\"",
910 repo.c_str(), names[j].c_str(), h->N());
911 for (unsigned int i = 0; i < qs.size(); i++) {
912 format_index += snprintf(buffer + format_index, bufSize - format_index,
913 ",%u", h->GetQuantile(qs[i]));
914 }
915 format_index += snprintf(buffer + format_index, bufSize - format_index,
916 "\n");
917 assert(format_index < bufSize);
918
919 result += buffer;
920 memset(buffer, 0, sizeof(buffer));
921 format_index = 0;
922 }
923 return result;
924 }
925
926 string TalkManager::FormatPrometheusMetrics(MountPoint &mount_point,
927 FileSystem *file_system) {
928 string result;
929 const string fqrn = mount_point.fqrn();
930 const string mountpoint = cvmfs::loader_exports_->mount_point;
931
932 // Helper function to format a prometheus metric
933 class MetricFormatter {
934 public:
935 explicit MetricFormatter(string &result_ref) : result_(result_ref) {}
936 void operator()(const string &name, const string &type,
937 const string &help, const string &labels,
938 const string &value) {
939 result_ += "# HELP " + name + " " + help + "\n";
940 result_ += "# TYPE " + name + " " + type + "\n";
941 result_ += name + "{" + labels + "} " + value + "\n";
942 }
943 private:
944 string &result_;
945 };
946 MetricFormatter format_metric(result);
947
948 // Get cache information
949 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
950 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
951 const uint64_t size_unpinned = quota_mgr->GetSize();
952 const uint64_t size_pinned = quota_mgr->GetSizePinned();
953
954 format_metric("cvmfs_cached_bytes", "gauge",
955 "CVMFS currently cached bytes.",
956 "repo=\"" + fqrn + "\"", StringifyUint(size_unpinned));
957 format_metric("cvmfs_pinned_bytes", "gauge",
958 "CVMFS currently pinned bytes.",
959 "repo=\"" + fqrn + "\"", StringifyUint(size_pinned));
960 }
961
962 // Get cache limit from parameters
963 string cache_limit_str;
964 if (file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT", &cache_limit_str)) {
965 const uint64_t cache_limit_mb = String2Uint64(cache_limit_str);
966 const uint64_t cache_limit_bytes = cache_limit_mb * 1024 * 1024;
967 format_metric("cvmfs_total_cache_size_bytes", "gauge",
968 "CVMFS configured cache size via CVMFS_QUOTA_LIMIT.",
969 "repo=\"" + fqrn + "\"", StringifyUint(cache_limit_bytes));
970 }
971
972 // Get cache base directory for df information
973 string cache_base;
974 if (file_system->options_mgr()->GetValue("CVMFS_CACHE_BASE", &cache_base)) {
975 struct statvfs stat_info;
976 if (statvfs(cache_base.c_str(), &stat_info) == 0) {
977 const uint64_t total_size = static_cast<uint64_t>(stat_info.f_blocks) * stat_info.f_frsize;
978 const uint64_t avail_size = static_cast<uint64_t>(stat_info.f_bavail) * stat_info.f_frsize;
979
980 format_metric("cvmfs_physical_cache_size_bytes", "gauge",
981 "CVMFS cache volume physical size.",
982 "repo=\"" + fqrn + "\"", StringifyUint(total_size));
983 format_metric("cvmfs_physical_cache_avail_bytes", "gauge",
984 "CVMFS cache volume physical free space available.",
985 "repo=\"" + fqrn + "\"", StringifyUint(avail_size));
986 }
987 }
988
989 // Version and revision information
990 const string version = string(CVMFS_VERSION) + "." + string(CVMFS_PATCH_LEVEL);
991 const uint64_t revision = mount_point.catalog_mgr()->GetRevision();
992 format_metric("cvmfs_repo", "gauge",
993 "Shows the version of CVMFS used by this repository.",
994 "repo=\"" + fqrn + "\",mountpoint=\"" + mountpoint +
995 "\",version=\"" + version + "\",revision=\"" + StringifyUint(revision) + "\"",
996 "1");
997
998 // Statistics-based metrics
999 perf::Statistics *statistics = mount_point.statistics();
1000
1001 // Download statistics
1002 const int64_t rx_bytes = statistics->Lookup("download.sz_transferred_bytes")->Get();
1003 format_metric("cvmfs_rx_total", "counter",
1004 "Shows the overall amount of downloaded bytes since mounting.",
1005 "repo=\"" + fqrn + "\"", StringifyInt(rx_bytes));
1006
1007 const int64_t n_downloads = statistics->Lookup("fetch.n_downloads")->Get();
1008 format_metric("cvmfs_ndownload_total", "counter",
1009 "Shows the overall number of downloaded files since mounting.",
1010 "repo=\"" + fqrn + "\"", StringifyInt(n_downloads));
1011
1012 // Hit rate calculation
1013 const int64_t n_invocations = statistics->Lookup("fetch.n_invocations")->Get();
1014 if (n_invocations > 0) {
1015 const float hitrate = 100.0 * (1.0 - (static_cast<float>(n_downloads) /
1016 static_cast<float>(n_invocations)));
1017 format_metric("cvmfs_hitrate", "gauge",
1018 "CVMFS cache hit rate (%)",
1019 "repo=\"" + fqrn + "\"", StringifyDouble(hitrate));
1020 } else {
1021 format_metric("cvmfs_hitrate", "gauge",
1022 "CVMFS cache hit rate (%)",
1023 "repo=\"" + fqrn + "\"", "0");
1024 }
1025
1026 // Speed calculation
1027 const int64_t transfer_time = statistics->Lookup("download.sz_transfer_time")->Get();
1028 if (transfer_time > 0) {
1029 const int64_t speed = (1000 * (rx_bytes / 1024)) / transfer_time;
1030 format_metric("cvmfs_speed", "gauge",
1031 "Shows the average download speed.",
1032 "repo=\"" + fqrn + "\"", StringifyInt(speed));
1033 } else {
1034 format_metric("cvmfs_speed", "gauge",
1035 "Shows the average download speed.",
1036 "repo=\"" + fqrn + "\"", "0");
1037 }
1038
1039 // Uptime calculation
1040 const time_t now = time(NULL);
1041 const uint64_t uptime_seconds = now - cvmfs::loader_exports_->boot_time;
1042 const uint64_t mount_epoch_time = now - uptime_seconds;
1043 format_metric("cvmfs_uptime_seconds", "counter",
1044 "Shows the time since the repo was mounted.",
1045 "repo=\"" + fqrn + "\"", StringifyUint(uptime_seconds));
1046 format_metric("cvmfs_mount_epoch_timestamp", "counter",
1047 "Shows the epoch time the repo was mounted.",
1048 "repo=\"" + fqrn + "\"", StringifyUint(mount_epoch_time));
1049
1050 // Catalog expiry - access through the TalkManager's remounter member
1051 const time_t catalogs_valid_until = remounter_->catalogs_valid_until();
1052 if (catalogs_valid_until != MountPoint::kIndefiniteDeadline) {
1053 const int64_t expires_seconds = (catalogs_valid_until - now);
1054 format_metric("cvmfs_repo_expires_seconds", "gauge",
1055 "Shows the remaining life time of the mounted root file catalog in seconds.",
1056 "repo=\"" + fqrn + "\"", StringifyInt(expires_seconds));
1057 }
1058
1059 // I/O error count
1060 const uint64_t nioerr = file_system->io_error_info()->count();
1061 format_metric("cvmfs_nioerr_total", "counter",
1062 "Shows the total number of I/O errors encountered since mounting.",
1063 "repo=\"" + fqrn + "\"", StringifyUint(nioerr));
1064
1065 // Timeout information
1066 unsigned timeout_proxy, timeout_direct;
1067 mount_point.download_mgr()->GetTimeout(&timeout_proxy, &timeout_direct);
1068 format_metric("cvmfs_timeout", "gauge",
1069 "Shows the timeout for proxied connections in seconds.",
1070 "repo=\"" + fqrn + "\"", StringifyUint(timeout_proxy));
1071 format_metric("cvmfs_timeout_direct", "gauge",
1072 "Shows the timeout for direct connections in seconds.",
1073 "repo=\"" + fqrn + "\"", StringifyUint(timeout_direct));
1074
1075 // Last I/O error timestamp
1076 const int64_t timestamp_last_ioerr = file_system->io_error_info()->timestamp_last();
1077 format_metric("cvmfs_timestamp_last_ioerr", "counter",
1078 "Shows the timestamp of the last ioerror.",
1079 "repo=\"" + fqrn + "\"", StringifyInt(timestamp_last_ioerr));
1080
1081 // CPU usage from /proc/pid/stat
1082 const pid_t pid = cvmfs::pid_;
1083 const string proc_stat_path = "/proc/" + StringifyInt(pid) + "/stat";
1084 FILE *stat_file = fopen(proc_stat_path.c_str(), "r");
1085 if (stat_file) {
1086 char stat_line[1024];
1087 if (fgets(stat_line, sizeof(stat_line), stat_file)) {
1088 vector<string> stat_fields = SplitString(string(stat_line), ' ');
1089 if (stat_fields.size() > 15) {
1090 const uint64_t utime = String2Uint64(stat_fields[13]);
1091 const uint64_t stime = String2Uint64(stat_fields[14]);
1092 const long clock_tick = sysconf(_SC_CLK_TCK);
1093 if (clock_tick > 0) {
1094 const double user_seconds = static_cast<double>(utime) / clock_tick;
1095 const double system_seconds = static_cast<double>(stime) / clock_tick;
1096 format_metric("cvmfs_cpu_user_total", "counter",
1097 "CPU time used in userspace by CVMFS mount in seconds.",
1098 "repo=\"" + fqrn + "\"", StringifyDouble(user_seconds));
1099 format_metric("cvmfs_cpu_system_total", "counter",
1100 "CPU time used in the kernel system calls by CVMFS mount in seconds.",
1101 "repo=\"" + fqrn + "\"", StringifyDouble(system_seconds));
1102 }
1103 }
1104 }
1105 fclose(stat_file);
1106 }
1107
1108 // File descriptor and directory counts
1109 format_metric("cvmfs_usedfd", "gauge",
1110 "Shows the number of open directories currently used by file system clients.",
1111 "repo=\"" + fqrn + "\"",
1112 file_system->no_open_files()->ToString());
1113 format_metric("cvmfs_useddirp", "gauge",
1114 "Shows the number of file descriptors currently issued to file system clients.",
1115 "repo=\"" + fqrn + "\"",
1116 file_system->no_open_dirs()->ToString());
1117 format_metric("cvmfs_ndiropen", "gauge",
1118 "Shows the overall number of opened directories.",
1119 "repo=\"" + fqrn + "\"",
1120 file_system->n_fs_dir_open()->ToString());
1121
1122 // Inode max
1123 format_metric("cvmfs_inode_max", "gauge",
1124 "Shows the highest possible inode with the current set of loaded catalogs.",
1125 "repo=\"" + fqrn + "\"",
1126 StringifyInt(mount_point.inode_annotation()->GetGeneration() +
1127 mount_point.catalog_mgr()->inode_gauge()));
1128
1129 // Process ID
1130 format_metric("cvmfs_pid", "gauge",
1131 "Shows the process id of the CernVM-FS Fuse process.",
1132 "repo=\"" + fqrn + "\"", StringifyInt(pid));
1133
1134 // Catalog count
1135 const int n_catalogs = mount_point.catalog_mgr()->GetNumCatalogs();
1136 format_metric("cvmfs_nclg", "gauge",
1137 "Shows the number of currently loaded nested catalogs.",
1138 "repo=\"" + fqrn + "\"", StringifyInt(n_catalogs));
1139
1140 // Cleanup rate (24 hours)
1141 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
1142 const uint64_t period_s = 24 * 60 * 60;
1143 const uint64_t cleanup_rate = quota_mgr->GetCleanupRate(period_s);
1144 format_metric("cvmfs_ncleanup24", "gauge",
1145 "Shows the number of cache cleanups in the last 24 hours.",
1146 "repo=\"" + fqrn + "\"", StringifyUint(cleanup_rate));
1147 } else {
1148 format_metric("cvmfs_ncleanup24", "gauge",
1149 "Shows the number of cache cleanups in the last 24 hours.",
1150 "repo=\"" + fqrn + "\"", "-1");
1151 }
1152
1153 // Active proxy
1154 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
1155 unsigned current_group;
1156 mount_point.download_mgr()->GetProxyInfo(&proxy_chain, &current_group, NULL);
1157 string active_proxy = "DIRECT";
1158 if (proxy_chain.size() > 0 && current_group < proxy_chain.size() &&
1159 proxy_chain[current_group].size() > 0) {
1160 active_proxy = proxy_chain[current_group][0].url;
1161 }
1162 format_metric("cvmfs_active_proxy", "gauge",
1163 "Shows the active proxy in use for this mount.",
1164 "repo=\"" + fqrn + "\",proxy=\"" + active_proxy + "\"", "1");
1165
1166 // Proxy list metrics
1167 for (unsigned int i = 0; i < proxy_chain.size(); i++) {
1168 for (unsigned int j = 0; j < proxy_chain[i].size(); j++) {
1169 format_metric("cvmfs_proxy", "gauge",
1170 "Shows all registered proxies for this repository.",
1171 "repo=\"" + fqrn + "\",group=\"" + StringifyInt(i) +
1172 "\",url=\"" + proxy_chain[i][j].url + "\"", "1");
1173 }
1174 }
1175
1176 // Internal affairs metrics (excluding histograms)
1177
1178 // Update string counters manually (same as internal affairs does)
1179 mount_point.statistics()->Lookup("pathstring.n_instances")->Set(PathString::num_instances());
1180 mount_point.statistics()->Lookup("pathstring.n_overflows")->Set(PathString::num_overflows());
1181 mount_point.statistics()->Lookup("namestring.n_instances")->Set(NameString::num_instances());
1182 mount_point.statistics()->Lookup("namestring.n_overflows")->Set(NameString::num_overflows());
1183 mount_point.statistics()->Lookup("linkstring.n_instances")->Set(LinkString::num_instances());
1184 mount_point.statistics()->Lookup("linkstring.n_overflows")->Set(LinkString::num_overflows());
1185
1186 // String statistics
1187 const int64_t pathstring_instances = mount_point.statistics()->Lookup("pathstring.n_instances")->Get();
1188 const int64_t pathstring_overflows = mount_point.statistics()->Lookup("pathstring.n_overflows")->Get();
1189 const int64_t namestring_instances = mount_point.statistics()->Lookup("namestring.n_instances")->Get();
1190 const int64_t namestring_overflows = mount_point.statistics()->Lookup("namestring.n_overflows")->Get();
1191 const int64_t linkstring_instances = mount_point.statistics()->Lookup("linkstring.n_instances")->Get();
1192 const int64_t linkstring_overflows = mount_point.statistics()->Lookup("linkstring.n_overflows")->Get();
1193
1194 format_metric("cvmfs_pathstring_instances", "gauge",
1195 "Number of PathString instances.",
1196 "repo=\"" + fqrn + "\"", StringifyInt(pathstring_instances));
1197 format_metric("cvmfs_pathstring_overflows", "counter",
1198 "Number of PathString overflows.",
1199 "repo=\"" + fqrn + "\"", StringifyInt(pathstring_overflows));
1200 format_metric("cvmfs_namestring_instances", "gauge",
1201 "Number of NameString instances.",
1202 "repo=\"" + fqrn + "\"", StringifyInt(namestring_instances));
1203 format_metric("cvmfs_namestring_overflows", "counter",
1204 "Number of NameString overflows.",
1205 "repo=\"" + fqrn + "\"", StringifyInt(namestring_overflows));
1206 format_metric("cvmfs_linkstring_instances", "gauge",
1207 "Number of LinkString instances.",
1208 "repo=\"" + fqrn + "\"", StringifyInt(linkstring_instances));
1209 format_metric("cvmfs_linkstring_overflows", "counter",
1210 "Number of LinkString overflows.",
1211 "repo=\"" + fqrn + "\"", StringifyInt(linkstring_overflows));
1212
1213 // Tracker statistics (same as internal affairs does)
1214 glue::InodeTracker::Statistics inode_stats = mount_point.inode_tracker()->GetStatistics();
1215 const glue::DentryTracker::Statistics dentry_stats = mount_point.dentry_tracker()->GetStatistics();
1216 const glue::PageCacheTracker::Statistics page_cache_stats = mount_point.page_cache_tracker()->GetStatistics();
1217
1218 // Update statistics manually
1219 mount_point.statistics()->Lookup("inode_tracker.n_insert")->Set(atomic_read64(&inode_stats.num_inserts));
1220 mount_point.statistics()->Lookup("inode_tracker.n_remove")->Set(atomic_read64(&inode_stats.num_removes));
1221 mount_point.statistics()->Lookup("inode_tracker.no_reference")->Set(atomic_read64(&inode_stats.num_references));
1222 mount_point.statistics()->Lookup("inode_tracker.n_hit_inode")->Set(atomic_read64(&inode_stats.num_hits_inode));
1223 mount_point.statistics()->Lookup("inode_tracker.n_hit_path")->Set(atomic_read64(&inode_stats.num_hits_path));
1224 mount_point.statistics()->Lookup("inode_tracker.n_miss_path")->Set(atomic_read64(&inode_stats.num_misses_path));
1225 mount_point.statistics()->Lookup("dentry_tracker.n_insert")->Set(dentry_stats.num_insert);
1226 mount_point.statistics()->Lookup("dentry_tracker.n_remove")->Set(dentry_stats.num_remove);
1227 mount_point.statistics()->Lookup("dentry_tracker.n_prune")->Set(dentry_stats.num_prune);
1228 mount_point.statistics()->Lookup("page_cache_tracker.n_insert")->Set(page_cache_stats.n_insert);
1229 mount_point.statistics()->Lookup("page_cache_tracker.n_remove")->Set(page_cache_stats.n_remove);
1230 mount_point.statistics()->Lookup("page_cache_tracker.n_open_direct")->Set(page_cache_stats.n_open_direct);
1231 mount_point.statistics()->Lookup("page_cache_tracker.n_open_flush")->Set(page_cache_stats.n_open_flush);
1232 mount_point.statistics()->Lookup("page_cache_tracker.n_open_cached")->Set(page_cache_stats.n_open_cached);
1233
1234 // Inode tracker metrics
1235 format_metric("cvmfs_inode_tracker_inserts_total", "counter",
1236 "Number of inode tracker insertions.",
1237 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_inserts)));
1238 format_metric("cvmfs_inode_tracker_removes_total", "counter",
1239 "Number of inode tracker removals.",
1240 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_removes)));
1241 format_metric("cvmfs_inode_tracker_references", "gauge",
1242 "Number of inode tracker references.",
1243 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_references)));
1244 format_metric("cvmfs_inode_tracker_hits_inode_total", "counter",
1245 "Number of inode tracker inode hits.",
1246 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_hits_inode)));
1247 format_metric("cvmfs_inode_tracker_hits_path_total", "counter",
1248 "Number of inode tracker path hits.",
1249 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_hits_path)));
1250 format_metric("cvmfs_inode_tracker_misses_path_total", "counter",
1251 "Number of inode tracker path misses.",
1252 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_misses_path)));
1253
1254 // Dentry tracker metrics
1255 format_metric("cvmfs_dentry_tracker_inserts_total", "counter",
1256 "Number of dentry tracker insertions.",
1257 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_insert));
1258 format_metric("cvmfs_dentry_tracker_removes_total", "counter",
1259 "Number of dentry tracker removals.",
1260 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_remove));
1261 format_metric("cvmfs_dentry_tracker_prunes_total", "counter",
1262 "Number of dentry tracker prunes.",
1263 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_prune));
1264
1265 // Page cache tracker metrics
1266 format_metric("cvmfs_page_cache_tracker_inserts_total", "counter",
1267 "Number of page cache tracker insertions.",
1268 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_insert));
1269 format_metric("cvmfs_page_cache_tracker_removes_total", "counter",
1270 "Number of page cache tracker removals.",
1271 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_remove));
1272 format_metric("cvmfs_page_cache_tracker_opens_direct_total", "counter",
1273 "Number of page cache tracker direct opens.",
1274 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_direct));
1275 format_metric("cvmfs_page_cache_tracker_opens_flush_total", "counter",
1276 "Number of page cache tracker flush opens.",
1277 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_flush));
1278 format_metric("cvmfs_page_cache_tracker_opens_cached_total", "counter",
1279 "Number of page cache tracker cached opens.",
1280 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_cached));
1281
1282 // Cache mode information
1283 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
1284 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(file_system->cache_mgr());
1285 int cache_mode_value = 0;
1286 switch (cache_mgr->cache_mode()) {
1287 case PosixCacheManager::kCacheReadWrite:
1288 cache_mode_value = 1;
1289 break;
1290 case PosixCacheManager::kCacheReadOnly:
1291 cache_mode_value = 2;
1292 break;
1293 default:
1294 cache_mode_value = 0;
1295 }
1296 format_metric("cvmfs_cache_mode", "gauge",
1297 "Cache mode (0=unknown, 1=read-write, 2=read-only).",
1298 "repo=\"" + fqrn + "\"", StringifyInt(cache_mode_value));
1299 }
1300
1301 // Drainout and maintenance mode
1302 bool drainout_mode;
1303 bool maintenance_mode;
1304 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
1305 format_metric("cvmfs_drainout_mode", "gauge",
1306 "Drainout mode status (0=false, 1=true).",
1307 "repo=\"" + fqrn + "\"", StringifyInt(drainout_mode ? 1 : 0));
1308 format_metric("cvmfs_maintenance_mode", "gauge",
1309 "Maintenance mode status (0=false, 1=true).",
1310 "repo=\"" + fqrn + "\"", StringifyInt(maintenance_mode ? 1 : 0));
1311
1312 // SQLite statistics
1313 int current, highwater;
1314
1315 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
1316 format_metric("cvmfs_sqlite_malloc_count", "gauge",
1317 "Number of SQLite allocations.",
1318 "repo=\"" + fqrn + "\"", StringifyInt(current));
1319
1320 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
1321 format_metric("cvmfs_sqlite_memory_used_bytes", "gauge",
1322 "SQLite general purpose allocator memory used.",
1323 "repo=\"" + fqrn + "\"", StringifyInt(current));
1324 format_metric("cvmfs_sqlite_memory_used_highwater_bytes", "gauge",
1325 "SQLite general purpose allocator memory used high water mark.",
1326 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1327
1328 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
1329 format_metric("cvmfs_sqlite_largest_malloc_bytes", "gauge",
1330 "SQLite largest malloc size.",
1331 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1332
1333 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
1334 format_metric("cvmfs_sqlite_pagecache_used", "gauge",
1335 "SQLite page cache allocations used.",
1336 "repo=\"" + fqrn + "\"", StringifyInt(current));
1337 format_metric("cvmfs_sqlite_pagecache_used_highwater", "gauge",
1338 "SQLite page cache allocations used high water mark.",
1339 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1340
1341 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
1342 format_metric("cvmfs_sqlite_pagecache_overflow_bytes", "gauge",
1343 "SQLite page cache overflow bytes.",
1344 "repo=\"" + fqrn + "\"", StringifyInt(current));
1345 format_metric("cvmfs_sqlite_pagecache_overflow_highwater_bytes", "gauge",
1346 "SQLite page cache overflow bytes high water mark.",
1347 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1348
1349 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
1350 format_metric("cvmfs_sqlite_largest_pagecache_bytes", "gauge",
1351 "SQLite largest page cache allocation size.",
1352 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1353
1354 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
1355 format_metric("cvmfs_sqlite_scratch_used", "gauge",
1356 "SQLite scratch allocations used.",
1357 "repo=\"" + fqrn + "\"", StringifyInt(current));
1358 format_metric("cvmfs_sqlite_scratch_used_highwater", "gauge",
1359 "SQLite scratch allocations used high water mark.",
1360 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1361
1362 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
1363 format_metric("cvmfs_sqlite_scratch_overflow", "gauge",
1364 "SQLite scratch overflows.",
1365 "repo=\"" + fqrn + "\"", StringifyInt(current));
1366 format_metric("cvmfs_sqlite_scratch_overflow_highwater", "gauge",
1367 "SQLite scratch overflows high water mark.",
1368 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1369
1370 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
1371 format_metric("cvmfs_sqlite_largest_scratch_bytes", "gauge",
1372 "SQLite largest scratch allocation size.",
1373 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1374
1375 // NFS statistics (if applicable)
1376 if (file_system->IsNfsSource()) {
1377 format_metric("cvmfs_nfs_mode", "gauge",
1378 "NFS mode enabled (1=true, 0=false).",
1379 "repo=\"" + fqrn + "\"", "1");
1380 // Note: NFS map statistics are complex strings, skipping detailed parsing for now
1381 } else {
1382 format_metric("cvmfs_nfs_mode", "gauge",
1383 "NFS mode enabled (1=true, 0=false).",
1384 "repo=\"" + fqrn + "\"", "0");
1385 }
1386
1387 return result;
1388 }
1389
1390 TalkManager::TalkManager(const string &socket_path,
1391 MountPoint *mount_point,
1392 FuseRemounter *remounter)
1393 : socket_path_(socket_path)
1394 , socket_fd_(-1)
1395 , mount_point_(mount_point)
1396 , remounter_(remounter)
1397 , spawned_(false) {
1398 memset(&thread_talk_, 0, sizeof(thread_talk_));
1399 }
1400
1401
1402 TalkManager::~TalkManager() {
1403 if (!socket_path_.empty()) {
1404 const int retval = unlink(socket_path_.c_str());
1405 if ((retval != 0) && (errno != ENOENT)) {
1406 LogCvmfs(kLogTalk, kLogSyslogWarn,
1407 "Could not remove cvmfs_io socket from cache directory (%d)",
1408 errno);
1409 }
1410 }
1411
1412 if (socket_fd_ >= 0) {
1413 shutdown(socket_fd_, SHUT_RDWR);
1414 close(socket_fd_);
1415 }
1416
1417 if (spawned_) {
1418 pthread_join(thread_talk_, NULL);
1419 LogCvmfs(kLogTalk, kLogDebug, "talk thread stopped");
1420 }
1421 }
1422
1423
1424 void TalkManager::Spawn() {
1425 const int retval = pthread_create(&thread_talk_, NULL, MainResponder, this);
1426 assert(retval == 0);
1427 spawned_ = true;
1428 }
1429