GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/talk.cc
Date: 2026-05-10 02:36:07
Exec Total Coverage
Lines: 0 898 0.0%
Branches: 0 2914 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * Implements a socket interface to cvmfs. This way commands can be send
5 * to cvmfs. When cvmfs is running, the socket
6 * /var/cache/cvmfs2/$INSTANCE/cvmfs_io
7 * is available for command input and reply messages, resp.
8 *
9 * Cvmfs comes with the cvmfs_talk script, that handles writing and reading the
10 * socket.
11 *
12 * The talk module runs in a separate thread.
13 */
14
15 #include "talk.h"
16
17 #include <errno.h>
18 #include <inttypes.h>
19 #include <pthread.h>
20 #include <stdint.h>
21 #include <sys/socket.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <sys/uio.h>
25 #include <sys/un.h>
26 #include <unistd.h>
27
28 #include <cassert>
29 #include <cstdlib>
30 #include <cstring>
31 #include <string>
32 #include <vector>
33
34 #include "cache.h"
35 #include "cache_posix.h"
36 #include "catalog_mgr_client.h"
37 #include "cvmfs.h"
38 #include "duplex_sqlite3.h"
39 #include "fuse_remount.h"
40 #include "glue_buffer.h"
41 #include "loader.h"
42 #include "lru_md.h"
43 #include "monitor.h"
44 #include "mountpoint.h"
45 #include "network/download.h"
46 #include "nfs_maps.h"
47 #include "options.h"
48 #include "quota.h"
49 #include "shortstring.h"
50 #include "statistics.h"
51 #include "tracer.h"
52 #include "util/logging.h"
53 #include "util/pointer.h"
54 #include "wpad.h"
55
56 using namespace std; // NOLINT
57
58
59
60
61 void TalkManager::Answer(int con_fd, const string &msg) {
62 (void)send(con_fd, &msg[0], msg.length(), MSG_NOSIGNAL);
63 }
64
65
66 void TalkManager::AnswerStringList(int con_fd, const vector<string> &list) {
67 string list_str;
68 for (unsigned i = 0; i < list.size(); ++i) {
69 list_str += list[i] + "\n";
70 }
71 Answer(con_fd, list_str);
72 }
73
74
75 TalkManager *TalkManager::Create(const string &socket_path,
76 MountPoint *mount_point,
77 FuseRemounter *remounter) {
78 UniquePtr<TalkManager> talk_manager(
79 new TalkManager(socket_path, mount_point, remounter));
80
81 talk_manager->socket_fd_ = MakeSocket(socket_path, 0660);
82 if (talk_manager->socket_fd_ == -1)
83 return NULL;
84 if (listen(talk_manager->socket_fd_, 1) == -1)
85 return NULL;
86
87 LogCvmfs(kLogTalk, kLogDebug, "socket created at %s (fd %d)",
88 socket_path.c_str(), talk_manager->socket_fd_);
89
90 return talk_manager.Release();
91 }
92
93
94 string TalkManager::FormatMetalinkInfo(
95 download::DownloadManager *download_mgr) {
96 vector<string> metalink_chain;
97 unsigned active_metalink;
98
99 download_mgr->GetMetalinkInfo(&metalink_chain, &active_metalink);
100 if (metalink_chain.size() == 0)
101 return "No metalinks defined\n";
102
103 string metalink_str;
104 for (unsigned i = 0; i < metalink_chain.size(); ++i) {
105 metalink_str += " [" + StringifyInt(i) + "] " + metalink_chain[i] + "\n";
106 }
107 metalink_str += "Active metalink " + StringifyInt(active_metalink) + ": "
108 + metalink_chain[active_metalink] + "\n";
109 return metalink_str;
110 }
111
112 string TalkManager::FormatHostInfo(download::DownloadManager *download_mgr) {
113 vector<string> host_chain;
114 vector<int> rtt;
115 unsigned active_host;
116
117 download_mgr->GetHostInfo(&host_chain, &rtt, &active_host);
118 if (host_chain.size() == 0)
119 return "No hosts defined\n";
120
121 string host_str;
122 for (unsigned i = 0; i < host_chain.size(); ++i) {
123 host_str += " [" + StringifyInt(i) + "] " + host_chain[i] + " (";
124 if (rtt[i] == download::DownloadManager::kProbeUnprobed)
125 host_str += "unprobed";
126 else if (rtt[i] == download::DownloadManager::kProbeDown)
127 host_str += "host down";
128 else if (rtt[i] == download::DownloadManager::kProbeGeo)
129 host_str += "geographically ordered";
130 else
131 host_str += StringifyInt(rtt[i]) + " ms";
132 host_str += ")\n";
133 }
134 host_str += "Active host " + StringifyInt(active_host) + ": "
135 + host_chain[active_host] + "\n";
136 return host_str;
137 }
138
139 string TalkManager::FormatProxyInfo(download::DownloadManager *download_mgr) {
140 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
141 unsigned active_group;
142 unsigned fallback_group;
143
144 download_mgr->GetProxyInfo(&proxy_chain, &active_group, &fallback_group);
145 string proxy_str;
146 if (proxy_chain.size()) {
147 proxy_str += "Load-balance groups:\n";
148 for (unsigned i = 0; i < proxy_chain.size(); ++i) {
149 vector<string> urls;
150 for (unsigned j = 0; j < proxy_chain[i].size(); ++j) {
151 urls.push_back(proxy_chain[i][j].Print());
152 }
153 proxy_str += "[" + StringifyInt(i) + "] " + JoinStrings(urls, ", ")
154 + "\n";
155 }
156 proxy_str += "Active proxy: [" + StringifyInt(active_group) + "] "
157 + proxy_chain[active_group][0].url + "\n";
158 if (fallback_group < proxy_chain.size())
159 proxy_str += "First fallback group: [" + StringifyInt(fallback_group)
160 + "]\n";
161 } else {
162 proxy_str = "No proxies defined\n";
163 }
164 return proxy_str;
165 }
166
167
168 /**
169 * Listener thread on the socket.
170 * TODO(jblomer): create Format... helpers to shorten this method
171 */
172 void *TalkManager::MainResponder(void *data) {
173 TalkManager *talk_mgr = reinterpret_cast<TalkManager *>(data);
174 MountPoint *mount_point = talk_mgr->mount_point_;
175 FileSystem *file_system = mount_point->file_system();
176 FuseRemounter *remounter = talk_mgr->remounter_;
177 LogCvmfs(kLogTalk, kLogDebug, "talk thread started");
178
179 struct sockaddr_un remote;
180 socklen_t socket_size = sizeof(remote);
181 int con_fd = -1;
182 while (true) {
183 if (con_fd >= 0) {
184 shutdown(con_fd, SHUT_RDWR);
185 close(con_fd);
186 }
187 LogCvmfs(kLogTalk, kLogDebug, "accepting connections on socketfd %d",
188 talk_mgr->socket_fd_);
189 if ((con_fd = accept(talk_mgr->socket_fd_, (struct sockaddr *)&remote,
190 &socket_size))
191 < 0) {
192 LogCvmfs(kLogTalk, kLogDebug, "terminating talk thread (fd %d, errno %d)",
193 con_fd, errno);
194 break;
195 }
196
197 char buf[kMaxCommandSize];
198 int bytes_read;
199 if ((bytes_read = recv(con_fd, buf, sizeof(buf), 0)) <= 0)
200 continue;
201
202 if (buf[bytes_read - 1] == '\0')
203 bytes_read--;
204 const string line = string(buf, bytes_read);
205 LogCvmfs(kLogTalk, kLogDebug, "received %s (length %lu)", line.c_str(),
206 line.length());
207
208 if (line == "tracebuffer flush") {
209 mount_point->tracer()->Flush();
210 talk_mgr->Answer(con_fd, "OK\n");
211 } else if (line == "cache size") {
212 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
213 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
214 talk_mgr->Answer(con_fd, "Cache cannot report its size\n");
215 } else {
216 const uint64_t size_unpinned = quota_mgr->GetSize();
217 const uint64_t size_pinned = quota_mgr->GetSizePinned();
218 const string size_str = "Current cache size is "
219 + StringifyInt(size_unpinned / (1024 * 1024))
220 + "MB (" + StringifyInt(size_unpinned)
221 + " Bytes), pinned: "
222 + StringifyInt(size_pinned / (1024 * 1024))
223 + "MB (" + StringifyInt(size_pinned)
224 + " Bytes)\n";
225 talk_mgr->Answer(con_fd, size_str);
226 }
227 } else if (line == "cache instance") {
228 talk_mgr->Answer(con_fd, file_system->cache_mgr()->Describe());
229 } else if (line == "cache list") {
230 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
231 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
232 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
233 } else {
234 const vector<string> ls = quota_mgr->List();
235 talk_mgr->AnswerStringList(con_fd, ls);
236 }
237 } else if (line == "cache list pinned") {
238 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
239 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
240 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
241 } else {
242 const vector<string> ls_pinned = quota_mgr->ListPinned();
243 talk_mgr->AnswerStringList(con_fd, ls_pinned);
244 }
245 } else if (line == "cache list catalogs") {
246 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
247 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
248 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
249 } else {
250 const vector<string> ls_catalogs = quota_mgr->ListCatalogs();
251 talk_mgr->AnswerStringList(con_fd, ls_catalogs);
252 }
253 } else if (line.substr(0, 12) == "cleanup rate") {
254 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
255 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
256 talk_mgr->Answer(con_fd, "Unsupported by this cache\n");
257 } else {
258 if (line.length() < 14) {
259 talk_mgr->Answer(con_fd, "Usage: cleanup rate <period in mn>\n");
260 } else {
261 const uint64_t period_s = String2Uint64(line.substr(13)) * 60;
262 const uint64_t rate = quota_mgr->GetCleanupRate(period_s);
263 talk_mgr->Answer(con_fd, StringifyInt(rate) + "\n");
264 }
265 }
266 } else if (line.substr(0, 15) == "cache limit set") {
267 if (line.length() < 16) {
268 talk_mgr->Answer(con_fd, "Usage: cache limit set <MB>\n");
269 } else {
270 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
271 const uint64_t size = String2Uint64(line.substr(16));
272 if (size < 1000) {
273 talk_mgr->Answer(con_fd, "New limit too low (minimum 1000)\n");
274 } else {
275 if (quota_mgr->SetLimit(size * 1024 * 1024)) {
276 file_system->options_mgr()->SetValueFromTalk("CVMFS_QUOTA_LIMIT",
277 StringifyUint(size));
278 talk_mgr->Answer(con_fd, "OK\n");
279 } else {
280 talk_mgr->Answer(con_fd, "Limit not reset\n");
281 }
282 }
283 }
284 } else if (line == "cache limit get") {
285 std::string limit_from_options;
286 file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT",
287 &limit_from_options);
288 talk_mgr->Answer(con_fd, limit_from_options + "\n");
289 } else if (line.substr(0, 7) == "cleanup") {
290 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
291 if (!quota_mgr->HasCapability(QuotaManager::kCapShrink)) {
292 talk_mgr->Answer(con_fd, "Cache cannot trigger eviction\n");
293 } else {
294 if (line.length() < 9) {
295 talk_mgr->Answer(con_fd, "Usage: cleanup <MB>\n");
296 } else {
297 const uint64_t size = String2Uint64(line.substr(8)) * 1024 * 1024;
298 if (quota_mgr->Cleanup(size)) {
299 talk_mgr->Answer(con_fd, "OK\n");
300 } else {
301 talk_mgr->Answer(con_fd, "Not fully cleaned "
302 "(there might be pinned chunks)\n");
303 }
304 }
305 }
306 } else if (line.substr(0, 5) == "evict") {
307 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
308 if (line.length() < 7) {
309 talk_mgr->Answer(con_fd, "Usage: evict <path>\n");
310 } else {
311 const string path = line.substr(6);
312 const bool found_regular = cvmfs::Evict(path);
313 if (found_regular)
314 talk_mgr->Answer(con_fd, "OK\n");
315 else
316 talk_mgr->Answer(con_fd, "No such regular file\n");
317 }
318 } else if (line.substr(0, 3) == "pin") {
319 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
320 if (line.length() < 5) {
321 talk_mgr->Answer(con_fd, "Usage: pin <path>\n");
322 } else {
323 const string path = line.substr(4);
324 const bool found_regular = cvmfs::Pin(path);
325 if (found_regular)
326 talk_mgr->Answer(con_fd, "OK\n");
327 else
328 talk_mgr->Answer(con_fd, "No such regular file or pinning failed\n");
329 }
330 } else if (line == "mountpoint") {
331 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->mount_point + "\n");
332 } else if (line == "device id") {
333 if (cvmfs::loader_exports_->version >= 5)
334 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->device_id + "\n");
335 else
336 talk_mgr->Answer(con_fd, "0:0\n");
337 } else if (line.substr(0, 13) == "send mount fd") {
338 // Hidden command intended to be used only by the cvmfs mount helper
339 if (line.length() < 15) {
340 talk_mgr->Answer(con_fd, "EINVAL\n");
341 } else {
342 const std::string socket_path = line.substr(14);
343 const bool retval = cvmfs::SendFuseFd(socket_path);
344 talk_mgr->Answer(con_fd, retval ? "OK\n" : "Failed\n");
345 LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
346 "Attempt to send fuse connection info to new mount (via %s)%s",
347 socket_path.c_str(), retval ? "" : " -- failed!");
348 }
349 } else if (line.substr(0, 7) == "remount") {
350 FuseRemounter::Status status;
351 if (line == "remount sync")
352 status = remounter->CheckSynchronously();
353 else
354 status = remounter->Check();
355 switch (status) {
356 case FuseRemounter::kStatusFailGeneral:
357 talk_mgr->Answer(con_fd, "Failed\n");
358 break;
359 case FuseRemounter::kStatusFailNoSpace:
360 talk_mgr->Answer(con_fd, "Failed (no space)\n");
361 break;
362 case FuseRemounter::kStatusUp2Date:
363 talk_mgr->Answer(con_fd, "Catalog up to date\n");
364 break;
365 case FuseRemounter::kStatusDraining:
366 talk_mgr->Answer(con_fd, "New revision applied\n");
367 break;
368 case FuseRemounter::kStatusMaintenance:
369 talk_mgr->Answer(con_fd, "In maintenance mode\n");
370 break;
371 default:
372 talk_mgr->Answer(con_fd, "internal error\n");
373 }
374 } else if (line.substr(0, 6) == "chroot") {
375 if (line.length() < 8) {
376 talk_mgr->Answer(con_fd, "Usage: chroot <hash>\n");
377 } else {
378 const std::string root_hash = Trim(line.substr(7),
379 true /* trim_newline */);
380 const FuseRemounter::Status status = remounter->ChangeRoot(
381 MkFromHexPtr(shash::HexPtr(root_hash), shash::kSuffixCatalog));
382 switch (status) {
383 case FuseRemounter::kStatusUp2Date:
384 talk_mgr->Answer(con_fd, "OK\n");
385 break;
386 default:
387 talk_mgr->Answer(con_fd, "Failed\n");
388 break;
389 }
390 }
391 } else if (line == "detach nested catalogs") {
392 mount_point->catalog_mgr()->DetachNested();
393 talk_mgr->Answer(con_fd, "OK\n");
394 } else if (line == "revision") {
395 const string revision = StringifyInt(
396 mount_point->catalog_mgr()->GetRevision());
397 talk_mgr->Answer(con_fd, revision + "\n");
398 } else if (line == "max ttl info") {
399 const unsigned max_ttl = mount_point->GetMaxTtlMn();
400 if (max_ttl == 0) {
401 talk_mgr->Answer(con_fd, "unset\n");
402 } else {
403 const string max_ttl_str = StringifyInt(max_ttl) + " minutes\n";
404 talk_mgr->Answer(con_fd, max_ttl_str);
405 }
406 } else if (line.substr(0, 11) == "max ttl set") {
407 if (line.length() < 13) {
408 talk_mgr->Answer(con_fd, "Usage: max ttl set <minutes>\n");
409 } else {
410 const unsigned max_ttl = String2Uint64(line.substr(12));
411 mount_point->SetMaxTtlMn(max_ttl);
412 talk_mgr->Answer(con_fd, "OK\n");
413 }
414 } else if (line.substr(0, 14) == "nameserver get") {
415 const string dns_server = mount_point->download_mgr()->GetDnsServer();
416 const string reply = !dns_server.empty()
417 ? std::string("DNS server address: ")
418 + dns_server + "\n"
419 : std::string("DNS server not set.\n");
420 talk_mgr->Answer(con_fd, reply);
421 } else if (line.substr(0, 14) == "nameserver set") {
422 if (line.length() < 16) {
423 talk_mgr->Answer(con_fd, "Usage: nameserver set <host>\n");
424 } else {
425 const string host = line.substr(15);
426 mount_point->download_mgr()->SetDnsServer(host);
427 talk_mgr->Answer(con_fd, "OK\n");
428 }
429 } else if (line.substr(0, 22) == "__testing_freeze_cvmfs") {
430 const std::string fs_dir = line.substr(23) + "/dir";
431 mkdir(fs_dir.c_str(), 0700);
432 } else if (line == "external metalink info") {
433 const string external_metalink_info = talk_mgr->FormatMetalinkInfo(
434 mount_point->external_download_mgr());
435 talk_mgr->Answer(con_fd, external_metalink_info);
436 } else if (line == "metalink info") {
437 const string metalink_info = talk_mgr->FormatMetalinkInfo(
438 mount_point->download_mgr());
439 talk_mgr->Answer(con_fd, metalink_info);
440 } else if (line == "external host info") {
441 const string external_host_info = talk_mgr->FormatHostInfo(
442 mount_point->external_download_mgr());
443 talk_mgr->Answer(con_fd, external_host_info);
444 } else if (line == "host info") {
445 const string host_info = talk_mgr->FormatHostInfo(
446 mount_point->download_mgr());
447 talk_mgr->Answer(con_fd, host_info);
448 } else if (line == "host probe") {
449 mount_point->download_mgr()->ProbeHosts();
450 talk_mgr->Answer(con_fd, "OK\n");
451 } else if (line == "host probe geo") {
452 const bool retval = mount_point->download_mgr()->ProbeGeo();
453 if (retval)
454 talk_mgr->Answer(con_fd, "OK\n");
455 else
456 talk_mgr->Answer(con_fd, "Failed\n");
457 } else if (line == "external metalink switch") {
458 mount_point->external_download_mgr()->SwitchMetalink();
459 talk_mgr->Answer(con_fd, "OK\n");
460 } else if (line == "metalink switch") {
461 mount_point->download_mgr()->SwitchMetalink();
462 talk_mgr->Answer(con_fd, "OK\n");
463 } else if (line == "external host switch") {
464 mount_point->external_download_mgr()->SwitchHost();
465 talk_mgr->Answer(con_fd, "OK\n");
466 } else if (line == "host switch") {
467 mount_point->download_mgr()->SwitchHost();
468 talk_mgr->Answer(con_fd, "OK\n");
469 } else if (line.substr(0, 21) == "external metalink set") {
470 if (line.length() < 23) {
471 talk_mgr->Answer(con_fd, "Usage: external metalink set <URL>\n");
472 } else {
473 const std::string host = line.substr(22);
474 mount_point->external_download_mgr()->SetMetalinkChain(host);
475 talk_mgr->Answer(con_fd, "OK\n");
476 }
477 } else if (line.substr(0, 12) == "metalink set") {
478 if (line.length() < 14) {
479 talk_mgr->Answer(con_fd, "Usage: metalink set <URL>\n");
480 } else {
481 const std::string host = line.substr(13);
482 mount_point->download_mgr()->SetMetalinkChain(host);
483 talk_mgr->Answer(con_fd, "OK\n");
484 }
485 } else if (line.substr(0, 17) == "external host set") {
486 if (line.length() < 19) {
487 talk_mgr->Answer(con_fd, "Usage: external host set <URL>\n");
488 } else {
489 const std::string host = line.substr(18);
490 mount_point->external_download_mgr()->SetHostChain(host);
491 talk_mgr->Answer(con_fd, "OK\n");
492 }
493 } else if (line.substr(0, 8) == "host set") {
494 if (line.length() < 10) {
495 talk_mgr->Answer(con_fd, "Usage: host set <host list>\n");
496 } else {
497 const string hosts = line.substr(9);
498 mount_point->download_mgr()->SetHostChain(hosts);
499 talk_mgr->Answer(con_fd, "OK\n");
500 }
501 } else if (line == "external proxy info") {
502 const string external_proxy_info = talk_mgr->FormatProxyInfo(
503 mount_point->external_download_mgr());
504 talk_mgr->Answer(con_fd, external_proxy_info);
505 } else if (line == "proxy info") {
506 const string proxy_info = talk_mgr->FormatProxyInfo(
507 mount_point->download_mgr());
508 talk_mgr->Answer(con_fd, proxy_info);
509 } else if (line == "proxy rebalance") {
510 mount_point->download_mgr()->RebalanceProxies();
511 talk_mgr->Answer(con_fd, "OK\n");
512 } else if (line == "proxy group switch") {
513 mount_point->download_mgr()->SwitchProxyGroup();
514 talk_mgr->Answer(con_fd, "OK\n");
515 } else if (line.substr(0, 18) == "external proxy set") {
516 if (line.length() < 20) {
517 talk_mgr->Answer(con_fd, "Usage: external proxy set <proxy list>\n");
518 } else {
519 const string external_proxies = line.substr(19);
520 mount_point->external_download_mgr()->SetProxyChain(
521 external_proxies, "", download::DownloadManager::kSetProxyRegular);
522 talk_mgr->Answer(con_fd, "OK\n");
523 }
524 } else if (line.substr(0, 9) == "proxy set") {
525 if (line.length() < 11) {
526 talk_mgr->Answer(con_fd, "Usage: proxy set <proxy list>\n");
527 } else {
528 string proxies = line.substr(10);
529 proxies = download::ResolveProxyDescription(
530 proxies, "", mount_point->download_mgr());
531 if (proxies == "") {
532 talk_mgr->Answer(con_fd, "Failed, no valid proxies\n");
533 } else {
534 mount_point->download_mgr()->SetProxyChain(
535 proxies, "", download::DownloadManager::kSetProxyRegular);
536 talk_mgr->Answer(con_fd, "OK\n");
537 }
538 }
539 } else if (line.substr(0, 14) == "proxy fallback") {
540 if (line.length() < 15) {
541 talk_mgr->Answer(con_fd, "Usage: proxy fallback <proxy list>\n");
542 } else {
543 const string fallback_proxies = line.substr(15);
544 mount_point->download_mgr()->SetProxyChain(
545 "", fallback_proxies, download::DownloadManager::kSetProxyFallback);
546 talk_mgr->Answer(con_fd, "OK\n");
547 }
548 } else if (line == "timeout info") {
549 unsigned timeout;
550 unsigned timeout_direct;
551 mount_point->download_mgr()->GetTimeout(&timeout, &timeout_direct);
552 string timeout_str = "Timeout with proxy: ";
553 if (timeout)
554 timeout_str += StringifyInt(timeout) + "s\n";
555 else
556 timeout_str += "no timeout\n";
557 timeout_str += "Timeout without proxy: ";
558 if (timeout_direct)
559 timeout_str += StringifyInt(timeout_direct) + "s\n";
560 else
561 timeout_str += "no timeout\n";
562 talk_mgr->Answer(con_fd, timeout_str);
563 } else if (line.substr(0, 11) == "timeout set") {
564 if (line.length() < 13) {
565 talk_mgr->Answer(con_fd, "Usage: timeout set <proxy> <direct>\n");
566 } else {
567 uint64_t timeout;
568 uint64_t timeout_direct;
569 String2Uint64Pair(line.substr(12), &timeout, &timeout_direct);
570 mount_point->download_mgr()->SetTimeout(timeout, timeout_direct);
571 talk_mgr->Answer(con_fd, "OK\n");
572 }
573 } else if (line == "open catalogs") {
574 talk_mgr->Answer(con_fd, mount_point->catalog_mgr()->PrintHierarchy());
575 } else if (line == "drop metadata caches") {
576 // For testing
577 mount_point->inode_cache()->Pause();
578 mount_point->path_cache()->Pause();
579 mount_point->md5path_cache()->Pause();
580 mount_point->inode_cache()->Drop();
581 mount_point->path_cache()->Drop();
582 mount_point->md5path_cache()->Drop();
583 mount_point->inode_cache()->Resume();
584 mount_point->path_cache()->Resume();
585 mount_point->md5path_cache()->Resume();
586 talk_mgr->Answer(con_fd, "OK\n");
587 } else if (line == "internal affairs") {
588 int current;
589 int highwater;
590 string result;
591
592 result += "Inode Generation:\n " + cvmfs::PrintInodeGeneration();
593
594 // Manually setting the values of the ShortString counters
595 mount_point->statistics()
596 ->Lookup("pathstring.n_instances")
597 ->Set(PathString::num_instances());
598 mount_point->statistics()
599 ->Lookup("pathstring.n_overflows")
600 ->Set(PathString::num_overflows());
601 mount_point->statistics()
602 ->Lookup("namestring.n_instances")
603 ->Set(NameString::num_instances());
604 mount_point->statistics()
605 ->Lookup("namestring.n_overflows")
606 ->Set(NameString::num_overflows());
607 mount_point->statistics()
608 ->Lookup("linkstring.n_instances")
609 ->Set(LinkString::num_instances());
610 mount_point->statistics()
611 ->Lookup("linkstring.n_overflows")
612 ->Set(LinkString::num_overflows());
613
614 // Manually setting the inode tracker numbers
615 glue::InodeTracker::Statistics inode_stats = mount_point->inode_tracker()
616 ->GetStatistics();
617 const glue::DentryTracker::Statistics
618 dentry_stats = mount_point->dentry_tracker()->GetStatistics();
619 const glue::PageCacheTracker::Statistics
620 page_cache_stats = mount_point->page_cache_tracker()->GetStatistics();
621 mount_point->statistics()
622 ->Lookup("inode_tracker.n_insert")
623 ->Set(atomic_read64(&inode_stats.num_inserts));
624 mount_point->statistics()
625 ->Lookup("inode_tracker.n_remove")
626 ->Set(atomic_read64(&inode_stats.num_removes));
627 mount_point->statistics()
628 ->Lookup("inode_tracker.no_reference")
629 ->Set(atomic_read64(&inode_stats.num_references));
630 mount_point->statistics()
631 ->Lookup("inode_tracker.n_hit_inode")
632 ->Set(atomic_read64(&inode_stats.num_hits_inode));
633 mount_point->statistics()
634 ->Lookup("inode_tracker.n_hit_path")
635 ->Set(atomic_read64(&inode_stats.num_hits_path));
636 mount_point->statistics()
637 ->Lookup("inode_tracker.n_miss_path")
638 ->Set(atomic_read64(&inode_stats.num_misses_path));
639 mount_point->statistics()
640 ->Lookup("dentry_tracker.n_insert")
641 ->Set(dentry_stats.num_insert);
642 mount_point->statistics()
643 ->Lookup("dentry_tracker.n_remove")
644 ->Set(dentry_stats.num_remove);
645 mount_point->statistics()
646 ->Lookup("dentry_tracker.n_prune")
647 ->Set(dentry_stats.num_prune);
648 mount_point->statistics()
649 ->Lookup("page_cache_tracker.n_insert")
650 ->Set(page_cache_stats.n_insert);
651 mount_point->statistics()
652 ->Lookup("page_cache_tracker.n_remove")
653 ->Set(page_cache_stats.n_remove);
654 mount_point->statistics()
655 ->Lookup("page_cache_tracker.n_open_direct")
656 ->Set(page_cache_stats.n_open_direct);
657 mount_point->statistics()
658 ->Lookup("page_cache_tracker.n_open_flush")
659 ->Set(page_cache_stats.n_open_flush);
660 mount_point->statistics()
661 ->Lookup("page_cache_tracker.n_open_cached")
662 ->Set(page_cache_stats.n_open_cached);
663
664 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
665 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(
666 file_system->cache_mgr());
667 result += "\nCache Mode: ";
668 switch (cache_mgr->cache_mode()) {
669 case PosixCacheManager::kCacheReadWrite:
670 result += "read-write";
671 break;
672 case PosixCacheManager::kCacheReadOnly:
673 result += "read-only";
674 break;
675 default:
676 result += "unknown";
677 }
678 }
679 bool drainout_mode;
680 bool maintenance_mode;
681 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
682 result += "\nDrainout Mode: " + StringifyBool(drainout_mode) + "\n";
683 result += "Maintenance Mode: " + StringifyBool(maintenance_mode) + "\n";
684
685 if (file_system->IsNfsSource()) {
686 result += "\nNFS Map Statistics:\n";
687 result += file_system->nfs_maps()->GetStatistics();
688 }
689
690 result += "SQlite Statistics:\n";
691 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
692 result += " Number of allocations " + StringifyInt(current) + "\n";
693
694 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
695 result += " General purpose allocator " + StringifyInt(current / 1024)
696 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
697
698 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
699 result += " Largest malloc " + StringifyInt(highwater) + " Bytes\n";
700
701 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
702 result += " Page cache allocations " + StringifyInt(current) + " / "
703 + StringifyInt(highwater) + "\n";
704
705 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
706 result += " Page cache overflows " + StringifyInt(current / 1024)
707 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
708
709 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
710 result += " Largest page cache allocation " + StringifyInt(highwater)
711 + " Bytes\n";
712
713 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
714 result += " Scratch allocations " + StringifyInt(current) + " / "
715 + StringifyInt(highwater) + "\n";
716
717 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
718 result += " Scratch overflows " + StringifyInt(current) + " / "
719 + StringifyInt(highwater) + "\n";
720
721 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
722 result += " Largest scratch allocation " + StringifyInt(highwater / 1024)
723 + " KB\n";
724
725 result += "\nPer-Connection Memory Statistics:\n"
726 + mount_point->catalog_mgr()->PrintAllMemStatistics();
727
728 result += "\nLatency distribution of system calls:\n";
729
730 result += "Lookup\n" + file_system->hist_fs_lookup()->ToString();
731 result += "Forget\n" + file_system->hist_fs_forget()->ToString();
732 result += "Multi-Forget\n"
733 + file_system->hist_fs_forget_multi()->ToString();
734 result += "Getattr\n" + file_system->hist_fs_getattr()->ToString();
735 result += "Readlink\n" + file_system->hist_fs_readlink()->ToString();
736 result += "Opendir\n" + file_system->hist_fs_opendir()->ToString();
737 result += "Releasedir\n" + file_system->hist_fs_releasedir()->ToString();
738 result += "Readdir\n" + file_system->hist_fs_readdir()->ToString();
739 result += "Open\n" + file_system->hist_fs_open()->ToString();
740 result += "Read\n" + file_system->hist_fs_read()->ToString();
741 result += "Release\n" + file_system->hist_fs_release()->ToString();
742
743 result += "\nRaw Counters:\n"
744 + mount_point->statistics()->PrintList(
745 perf::Statistics::kPrintHeader);
746
747 talk_mgr->Answer(con_fd, result);
748 } else if (line == "reset error counters") {
749 file_system->ResetErrorCounters();
750 talk_mgr->Answer(con_fd, "OK\n");
751 } else if (line == "pid") {
752 const string pid_str = StringifyInt(cvmfs::pid_) + "\n";
753 talk_mgr->Answer(con_fd, pid_str);
754 } else if (line == "pid cachemgr") {
755 const string pid_str = StringifyInt(file_system->cache_mgr()
756 ->quota_mgr()
757 ->GetPid())
758 + "\n";
759 talk_mgr->Answer(con_fd, pid_str);
760 } else if (line == "pid watchdog") {
761 const string pid_str = StringifyInt(Watchdog::GetPid()) + "\n";
762 talk_mgr->Answer(con_fd, pid_str);
763 } else if (line == "parameters") {
764 talk_mgr->Answer(con_fd, file_system->options_mgr()->Dump());
765 } else if (line == "hotpatch history") {
766 string history_str = StringifyTime(cvmfs::loader_exports_->boot_time,
767 true)
768 + " (start of CernVM-FS loader "
769 + cvmfs::loader_exports_->loader_version + ")\n";
770 for (loader::EventList::const_iterator
771 i = cvmfs::loader_exports_->history.begin(),
772 iEnd = cvmfs::loader_exports_->history.end();
773 i != iEnd;
774 ++i) {
775 history_str += StringifyTime((*i)->timestamp, true)
776 + " (loaded CernVM-FS Fuse Module " + (*i)->so_version
777 + ")\n";
778 }
779 talk_mgr->Answer(con_fd, history_str);
780 } else if (line == "vfs inodes") {
781 string result;
782 glue::InodeTracker::Cursor cursor(
783 mount_point->inode_tracker()->BeginEnumerate());
784 uint64_t inode;
785 while (mount_point->inode_tracker()->NextInode(&cursor, &inode)) {
786 result += StringifyInt(inode) + "\n";
787 }
788 mount_point->inode_tracker()->EndEnumerate(&cursor);
789 talk_mgr->Answer(con_fd, result);
790 } else if (line == "vfs entries") {
791 string result;
792 glue::InodeTracker::Cursor cursor(
793 mount_point->inode_tracker()->BeginEnumerate());
794 uint64_t inode_parent;
795 NameString name;
796 while (mount_point->inode_tracker()->NextEntry(&cursor, &inode_parent,
797 &name)) {
798 result += "<" + StringifyInt(inode_parent) + ">/" + name.ToString()
799 + "\n";
800 }
801 mount_point->inode_tracker()->EndEnumerate(&cursor);
802 talk_mgr->Answer(con_fd, result);
803 } else if (line == "version") {
804 const string version_str = string(CVMFS_VERSION)
805 + " (CernVM-FS Fuse Module)\n"
806 + cvmfs::loader_exports_->loader_version
807 + " (Loader)\n";
808 talk_mgr->Answer(con_fd, version_str);
809 } else if (line == "version patchlevel") {
810 talk_mgr->Answer(con_fd, string(CVMFS_PATCH_LEVEL) + "\n");
811 } else if (line == "tear down to read-only") {
812 if (file_system->cache_mgr()->id() != kPosixCacheManager) {
813 talk_mgr->Answer(con_fd, "not supported\n");
814 } else {
815 // hack
816 cvmfs::UnregisterQuotaListener();
817 file_system->TearDown2ReadOnly();
818 talk_mgr->Answer(con_fd, "In read-only mode\n");
819 }
820 } else if (line == "latency") {
821 const string result = talk_mgr->FormatLatencies(*mount_point,
822 file_system);
823 talk_mgr->Answer(con_fd, result);
824 } else if (line == "metrics prometheus") {
825 const string result = talk_mgr->FormatPrometheusMetrics(*mount_point,
826 file_system);
827 talk_mgr->Answer(con_fd, result);
828 } else {
829 talk_mgr->Answer(con_fd, "unknown command\n");
830 }
831 }
832
833 return NULL;
834 } // NOLINT(readability/fn_size)
835
836 string TalkManager::FormatLatencies(const MountPoint &mount_point,
837 FileSystem *file_system) {
838 string result;
839 const unsigned int bufSize = 300;
840 char buffer[bufSize];
841
842 vector<float> qs;
843 qs.push_back(.1);
844 qs.push_back(.2);
845 qs.push_back(.25);
846 qs.push_back(.3);
847 qs.push_back(.4);
848 qs.push_back(.5);
849 qs.push_back(.6);
850 qs.push_back(.7);
851 qs.push_back(.75);
852 qs.push_back(.8);
853 qs.push_back(.9);
854 qs.push_back(.95);
855 qs.push_back(.99);
856 qs.push_back(.999);
857 qs.push_back(.9999);
858
859 const string repo(mount_point.fqrn());
860
861 unsigned int format_index = snprintf(
862 buffer, bufSize, "\"%s\",\"%s\",\"%s\",\"%s\"", "repository", "action",
863 "total_count", "time_unit");
864 for (unsigned int i = 0; i < qs.size(); i++) {
865 format_index += snprintf(buffer + format_index, bufSize - format_index,
866 ",%0.5f", qs[i]);
867 }
868 format_index += snprintf(buffer + format_index, bufSize - format_index, "\n");
869 assert(format_index < bufSize);
870
871 result += buffer;
872 memset(buffer, 0, sizeof(buffer));
873 format_index = 0;
874
875 vector<Log2Histogram *> hist;
876 vector<string> names;
877 hist.push_back(file_system->hist_fs_lookup());
878 names.push_back("lookup");
879 hist.push_back(file_system->hist_fs_forget());
880 names.push_back("forget");
881 hist.push_back(file_system->hist_fs_forget_multi());
882 names.push_back("forget_multi");
883 hist.push_back(file_system->hist_fs_getattr());
884 names.push_back("getattr");
885 hist.push_back(file_system->hist_fs_readlink());
886 names.push_back("readlink");
887 hist.push_back(file_system->hist_fs_opendir());
888 names.push_back("opendir");
889 hist.push_back(file_system->hist_fs_releasedir());
890 names.push_back("releasedir");
891 hist.push_back(file_system->hist_fs_readdir());
892 names.push_back("readdir");
893 hist.push_back(file_system->hist_fs_open());
894 names.push_back("open");
895 hist.push_back(file_system->hist_fs_read());
896 names.push_back("read");
897 hist.push_back(file_system->hist_fs_release());
898 names.push_back("release");
899
900 for (unsigned int j = 0; j < hist.size(); j++) {
901 Log2Histogram *h = hist[j];
902 unsigned int format_index = snprintf(
903 buffer, bufSize, "\"%s\",\"%s\",%" PRIu64 ",\"nanoseconds\"",
904 repo.c_str(), names[j].c_str(), h->N());
905 for (unsigned int i = 0; i < qs.size(); i++) {
906 format_index += snprintf(buffer + format_index, bufSize - format_index,
907 ",%u", h->GetQuantile(qs[i]));
908 }
909 format_index += snprintf(buffer + format_index, bufSize - format_index,
910 "\n");
911 assert(format_index < bufSize);
912
913 result += buffer;
914 memset(buffer, 0, sizeof(buffer));
915 format_index = 0;
916 }
917 return result;
918 }
919
920 string TalkManager::FormatPrometheusMetrics(MountPoint &mount_point,
921 FileSystem *file_system) {
922 string result;
923 const string fqrn = mount_point.fqrn();
924 const string mountpoint = cvmfs::loader_exports_->mount_point;
925
926 // Helper function to format a prometheus metric
927 class MetricFormatter {
928 public:
929 explicit MetricFormatter(string &result_ref) : result_(result_ref) {}
930 void operator()(const string &name, const string &type,
931 const string &help, const string &labels,
932 const string &value) {
933 result_ += "# HELP " + name + " " + help + "\n";
934 result_ += "# TYPE " + name + " " + type + "\n";
935 result_ += name + "{" + labels + "} " + value + "\n";
936 }
937 private:
938 string &result_;
939 };
940 MetricFormatter format_metric(result);
941
942 // Get cache information
943 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
944 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
945 const uint64_t size_unpinned = quota_mgr->GetSize();
946 const uint64_t size_pinned = quota_mgr->GetSizePinned();
947
948 format_metric("cvmfs_cached_bytes", "gauge",
949 "CVMFS currently cached bytes.",
950 "repo=\"" + fqrn + "\"", StringifyUint(size_unpinned));
951 format_metric("cvmfs_pinned_bytes", "gauge",
952 "CVMFS currently pinned bytes.",
953 "repo=\"" + fqrn + "\"", StringifyUint(size_pinned));
954 }
955
956 // Get cache limit from parameters
957 string cache_limit_str;
958 if (file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT", &cache_limit_str)) {
959 const uint64_t cache_limit_mb = String2Uint64(cache_limit_str);
960 const uint64_t cache_limit_bytes = cache_limit_mb * 1024 * 1024;
961 format_metric("cvmfs_total_cache_size_bytes", "gauge",
962 "CVMFS configured cache size via CVMFS_QUOTA_LIMIT.",
963 "repo=\"" + fqrn + "\"", StringifyUint(cache_limit_bytes));
964 }
965
966 // Get cache base directory for df information
967 string cache_base;
968 if (file_system->options_mgr()->GetValue("CVMFS_CACHE_BASE", &cache_base)) {
969 struct statvfs stat_info;
970 if (statvfs(cache_base.c_str(), &stat_info) == 0) {
971 const uint64_t total_size = static_cast<uint64_t>(stat_info.f_blocks) * stat_info.f_frsize;
972 const uint64_t avail_size = static_cast<uint64_t>(stat_info.f_bavail) * stat_info.f_frsize;
973
974 format_metric("cvmfs_physical_cache_size_bytes", "gauge",
975 "CVMFS cache volume physical size.",
976 "repo=\"" + fqrn + "\"", StringifyUint(total_size));
977 format_metric("cvmfs_physical_cache_avail_bytes", "gauge",
978 "CVMFS cache volume physical free space available.",
979 "repo=\"" + fqrn + "\"", StringifyUint(avail_size));
980 }
981 }
982
983 // Version and revision information
984 const string version = string(CVMFS_VERSION) + "." + string(CVMFS_PATCH_LEVEL);
985 const uint64_t revision = mount_point.catalog_mgr()->GetRevision();
986 format_metric("cvmfs_repo", "gauge",
987 "Shows the version of CVMFS used by this repository.",
988 "repo=\"" + fqrn + "\",mountpoint=\"" + mountpoint +
989 "\",version=\"" + version + "\",revision=\"" + StringifyUint(revision) + "\"",
990 "1");
991
992 // Statistics-based metrics
993 perf::Statistics *statistics = mount_point.statistics();
994
995 // Download statistics
996 const int64_t rx_bytes = statistics->Lookup("download.sz_transferred_bytes")->Get();
997 format_metric("cvmfs_rx_total", "counter",
998 "Shows the overall amount of downloaded bytes since mounting.",
999 "repo=\"" + fqrn + "\"", StringifyInt(rx_bytes));
1000
1001 const int64_t n_downloads = statistics->Lookup("fetch.n_downloads")->Get();
1002 format_metric("cvmfs_ndownload_total", "counter",
1003 "Shows the overall number of downloaded files since mounting.",
1004 "repo=\"" + fqrn + "\"", StringifyInt(n_downloads));
1005
1006 // Hit rate calculation
1007 const int64_t n_invocations = statistics->Lookup("fetch.n_invocations")->Get();
1008 if (n_invocations > 0) {
1009 const float hitrate = 100.0 * (1.0 - (static_cast<float>(n_downloads) /
1010 static_cast<float>(n_invocations)));
1011 format_metric("cvmfs_hitrate", "gauge",
1012 "CVMFS cache hit rate (%)",
1013 "repo=\"" + fqrn + "\"", StringifyDouble(hitrate));
1014 } else {
1015 format_metric("cvmfs_hitrate", "gauge",
1016 "CVMFS cache hit rate (%)",
1017 "repo=\"" + fqrn + "\"", "0");
1018 }
1019
1020 // Speed calculation
1021 const int64_t transfer_time = statistics->Lookup("download.sz_transfer_time")->Get();
1022 if (transfer_time > 0) {
1023 const int64_t speed = (1000 * (rx_bytes / 1024)) / transfer_time;
1024 format_metric("cvmfs_speed", "gauge",
1025 "Shows the average download speed.",
1026 "repo=\"" + fqrn + "\"", StringifyInt(speed));
1027 } else {
1028 format_metric("cvmfs_speed", "gauge",
1029 "Shows the average download speed.",
1030 "repo=\"" + fqrn + "\"", "0");
1031 }
1032
1033 // Uptime calculation
1034 const time_t now = time(NULL);
1035 const uint64_t uptime_seconds = now - cvmfs::loader_exports_->boot_time;
1036 const uint64_t mount_epoch_time = now - uptime_seconds;
1037 format_metric("cvmfs_uptime_seconds", "counter",
1038 "Shows the time since the repo was mounted.",
1039 "repo=\"" + fqrn + "\"", StringifyUint(uptime_seconds));
1040 format_metric("cvmfs_mount_epoch_timestamp", "counter",
1041 "Shows the epoch time the repo was mounted.",
1042 "repo=\"" + fqrn + "\"", StringifyUint(mount_epoch_time));
1043
1044 // Catalog expiry - access through the TalkManager's remounter member
1045 const time_t catalogs_valid_until = remounter_->catalogs_valid_until();
1046 if (catalogs_valid_until != MountPoint::kIndefiniteDeadline) {
1047 const int64_t expires_seconds = (catalogs_valid_until - now);
1048 format_metric("cvmfs_repo_expires_seconds", "gauge",
1049 "Shows the remaining life time of the mounted root file catalog in seconds.",
1050 "repo=\"" + fqrn + "\"", StringifyInt(expires_seconds));
1051 }
1052
1053 // I/O error count
1054 const uint64_t nioerr = file_system->io_error_info()->count();
1055 format_metric("cvmfs_nioerr_total", "counter",
1056 "Shows the total number of I/O errors encountered since mounting.",
1057 "repo=\"" + fqrn + "\"", StringifyUint(nioerr));
1058
1059 // Timeout information
1060 unsigned timeout_proxy, timeout_direct;
1061 mount_point.download_mgr()->GetTimeout(&timeout_proxy, &timeout_direct);
1062 format_metric("cvmfs_timeout", "gauge",
1063 "Shows the timeout for proxied connections in seconds.",
1064 "repo=\"" + fqrn + "\"", StringifyUint(timeout_proxy));
1065 format_metric("cvmfs_timeout_direct", "gauge",
1066 "Shows the timeout for direct connections in seconds.",
1067 "repo=\"" + fqrn + "\"", StringifyUint(timeout_direct));
1068
1069 // Last I/O error timestamp
1070 const int64_t timestamp_last_ioerr = file_system->io_error_info()->timestamp_last();
1071 format_metric("cvmfs_timestamp_last_ioerr", "counter",
1072 "Shows the timestamp of the last ioerror.",
1073 "repo=\"" + fqrn + "\"", StringifyInt(timestamp_last_ioerr));
1074
1075 // CPU usage from /proc/pid/stat
1076 const pid_t pid = cvmfs::pid_;
1077 const string proc_stat_path = "/proc/" + StringifyInt(pid) + "/stat";
1078 FILE *stat_file = fopen(proc_stat_path.c_str(), "r");
1079 if (stat_file) {
1080 char stat_line[1024];
1081 if (fgets(stat_line, sizeof(stat_line), stat_file)) {
1082 vector<string> stat_fields = SplitString(string(stat_line), ' ');
1083 if (stat_fields.size() > 15) {
1084 const uint64_t utime = String2Uint64(stat_fields[13]);
1085 const uint64_t stime = String2Uint64(stat_fields[14]);
1086 const long clock_tick = sysconf(_SC_CLK_TCK);
1087 if (clock_tick > 0) {
1088 const double user_seconds = static_cast<double>(utime) / clock_tick;
1089 const double system_seconds = static_cast<double>(stime) / clock_tick;
1090 format_metric("cvmfs_cpu_user_total", "counter",
1091 "CPU time used in userspace by CVMFS mount in seconds.",
1092 "repo=\"" + fqrn + "\"", StringifyDouble(user_seconds));
1093 format_metric("cvmfs_cpu_system_total", "counter",
1094 "CPU time used in the kernel system calls by CVMFS mount in seconds.",
1095 "repo=\"" + fqrn + "\"", StringifyDouble(system_seconds));
1096 }
1097 }
1098 }
1099 fclose(stat_file);
1100 }
1101
1102 // File descriptor and directory counts
1103 format_metric("cvmfs_usedfd", "gauge",
1104 "Shows the number of open directories currently used by file system clients.",
1105 "repo=\"" + fqrn + "\"",
1106 file_system->no_open_files()->ToString());
1107 format_metric("cvmfs_useddirp", "gauge",
1108 "Shows the number of file descriptors currently issued to file system clients.",
1109 "repo=\"" + fqrn + "\"",
1110 file_system->no_open_dirs()->ToString());
1111 format_metric("cvmfs_ndiropen", "gauge",
1112 "Shows the overall number of opened directories.",
1113 "repo=\"" + fqrn + "\"",
1114 file_system->n_fs_dir_open()->ToString());
1115
1116 // Inode max
1117 format_metric("cvmfs_inode_max", "gauge",
1118 "Shows the highest possible inode with the current set of loaded catalogs.",
1119 "repo=\"" + fqrn + "\"",
1120 StringifyInt(mount_point.inode_annotation()->GetGeneration() +
1121 mount_point.catalog_mgr()->inode_gauge()));
1122
1123 // Process ID
1124 format_metric("cvmfs_pid", "gauge",
1125 "Shows the process id of the CernVM-FS Fuse process.",
1126 "repo=\"" + fqrn + "\"", StringifyInt(pid));
1127
1128 // Catalog count
1129 const int n_catalogs = mount_point.catalog_mgr()->GetNumCatalogs();
1130 format_metric("cvmfs_nclg", "gauge",
1131 "Shows the number of currently loaded nested catalogs.",
1132 "repo=\"" + fqrn + "\"", StringifyInt(n_catalogs));
1133
1134 // Cleanup rate (24 hours)
1135 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
1136 const uint64_t period_s = 24 * 60 * 60;
1137 const uint64_t cleanup_rate = quota_mgr->GetCleanupRate(period_s);
1138 format_metric("cvmfs_ncleanup24", "gauge",
1139 "Shows the number of cache cleanups in the last 24 hours.",
1140 "repo=\"" + fqrn + "\"", StringifyUint(cleanup_rate));
1141 } else {
1142 format_metric("cvmfs_ncleanup24", "gauge",
1143 "Shows the number of cache cleanups in the last 24 hours.",
1144 "repo=\"" + fqrn + "\"", "-1");
1145 }
1146
1147 // Active proxy
1148 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
1149 unsigned current_group;
1150 mount_point.download_mgr()->GetProxyInfo(&proxy_chain, &current_group, NULL);
1151 string active_proxy = "DIRECT";
1152 if (proxy_chain.size() > 0 && current_group < proxy_chain.size() &&
1153 proxy_chain[current_group].size() > 0) {
1154 active_proxy = proxy_chain[current_group][0].url;
1155 }
1156 format_metric("cvmfs_active_proxy", "gauge",
1157 "Shows the active proxy in use for this mount.",
1158 "repo=\"" + fqrn + "\",proxy=\"" + active_proxy + "\"", "1");
1159
1160 // Proxy list metrics
1161 for (unsigned int i = 0; i < proxy_chain.size(); i++) {
1162 for (unsigned int j = 0; j < proxy_chain[i].size(); j++) {
1163 format_metric("cvmfs_proxy", "gauge",
1164 "Shows all registered proxies for this repository.",
1165 "repo=\"" + fqrn + "\",group=\"" + StringifyInt(i) +
1166 "\",url=\"" + proxy_chain[i][j].url + "\"", "1");
1167 }
1168 }
1169
1170 // Internal affairs metrics (excluding histograms)
1171
1172 // Update string counters manually (same as internal affairs does)
1173 mount_point.statistics()->Lookup("pathstring.n_instances")->Set(PathString::num_instances());
1174 mount_point.statistics()->Lookup("pathstring.n_overflows")->Set(PathString::num_overflows());
1175 mount_point.statistics()->Lookup("namestring.n_instances")->Set(NameString::num_instances());
1176 mount_point.statistics()->Lookup("namestring.n_overflows")->Set(NameString::num_overflows());
1177 mount_point.statistics()->Lookup("linkstring.n_instances")->Set(LinkString::num_instances());
1178 mount_point.statistics()->Lookup("linkstring.n_overflows")->Set(LinkString::num_overflows());
1179
1180 // String statistics
1181 const int64_t pathstring_instances = mount_point.statistics()->Lookup("pathstring.n_instances")->Get();
1182 const int64_t pathstring_overflows = mount_point.statistics()->Lookup("pathstring.n_overflows")->Get();
1183 const int64_t namestring_instances = mount_point.statistics()->Lookup("namestring.n_instances")->Get();
1184 const int64_t namestring_overflows = mount_point.statistics()->Lookup("namestring.n_overflows")->Get();
1185 const int64_t linkstring_instances = mount_point.statistics()->Lookup("linkstring.n_instances")->Get();
1186 const int64_t linkstring_overflows = mount_point.statistics()->Lookup("linkstring.n_overflows")->Get();
1187
1188 format_metric("cvmfs_pathstring_instances", "gauge",
1189 "Number of PathString instances.",
1190 "repo=\"" + fqrn + "\"", StringifyInt(pathstring_instances));
1191 format_metric("cvmfs_pathstring_overflows", "counter",
1192 "Number of PathString overflows.",
1193 "repo=\"" + fqrn + "\"", StringifyInt(pathstring_overflows));
1194 format_metric("cvmfs_namestring_instances", "gauge",
1195 "Number of NameString instances.",
1196 "repo=\"" + fqrn + "\"", StringifyInt(namestring_instances));
1197 format_metric("cvmfs_namestring_overflows", "counter",
1198 "Number of NameString overflows.",
1199 "repo=\"" + fqrn + "\"", StringifyInt(namestring_overflows));
1200 format_metric("cvmfs_linkstring_instances", "gauge",
1201 "Number of LinkString instances.",
1202 "repo=\"" + fqrn + "\"", StringifyInt(linkstring_instances));
1203 format_metric("cvmfs_linkstring_overflows", "counter",
1204 "Number of LinkString overflows.",
1205 "repo=\"" + fqrn + "\"", StringifyInt(linkstring_overflows));
1206
1207 // Tracker statistics (same as internal affairs does)
1208 glue::InodeTracker::Statistics inode_stats = mount_point.inode_tracker()->GetStatistics();
1209 const glue::DentryTracker::Statistics dentry_stats = mount_point.dentry_tracker()->GetStatistics();
1210 const glue::PageCacheTracker::Statistics page_cache_stats = mount_point.page_cache_tracker()->GetStatistics();
1211
1212 // Update statistics manually
1213 mount_point.statistics()->Lookup("inode_tracker.n_insert")->Set(atomic_read64(&inode_stats.num_inserts));
1214 mount_point.statistics()->Lookup("inode_tracker.n_remove")->Set(atomic_read64(&inode_stats.num_removes));
1215 mount_point.statistics()->Lookup("inode_tracker.no_reference")->Set(atomic_read64(&inode_stats.num_references));
1216 mount_point.statistics()->Lookup("inode_tracker.n_hit_inode")->Set(atomic_read64(&inode_stats.num_hits_inode));
1217 mount_point.statistics()->Lookup("inode_tracker.n_hit_path")->Set(atomic_read64(&inode_stats.num_hits_path));
1218 mount_point.statistics()->Lookup("inode_tracker.n_miss_path")->Set(atomic_read64(&inode_stats.num_misses_path));
1219 mount_point.statistics()->Lookup("dentry_tracker.n_insert")->Set(dentry_stats.num_insert);
1220 mount_point.statistics()->Lookup("dentry_tracker.n_remove")->Set(dentry_stats.num_remove);
1221 mount_point.statistics()->Lookup("dentry_tracker.n_prune")->Set(dentry_stats.num_prune);
1222 mount_point.statistics()->Lookup("page_cache_tracker.n_insert")->Set(page_cache_stats.n_insert);
1223 mount_point.statistics()->Lookup("page_cache_tracker.n_remove")->Set(page_cache_stats.n_remove);
1224 mount_point.statistics()->Lookup("page_cache_tracker.n_open_direct")->Set(page_cache_stats.n_open_direct);
1225 mount_point.statistics()->Lookup("page_cache_tracker.n_open_flush")->Set(page_cache_stats.n_open_flush);
1226 mount_point.statistics()->Lookup("page_cache_tracker.n_open_cached")->Set(page_cache_stats.n_open_cached);
1227
1228 // Inode tracker metrics
1229 format_metric("cvmfs_inode_tracker_inserts_total", "counter",
1230 "Number of inode tracker insertions.",
1231 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_inserts)));
1232 format_metric("cvmfs_inode_tracker_removes_total", "counter",
1233 "Number of inode tracker removals.",
1234 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_removes)));
1235 format_metric("cvmfs_inode_tracker_references", "gauge",
1236 "Number of inode tracker references.",
1237 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_references)));
1238 format_metric("cvmfs_inode_tracker_hits_inode_total", "counter",
1239 "Number of inode tracker inode hits.",
1240 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_hits_inode)));
1241 format_metric("cvmfs_inode_tracker_hits_path_total", "counter",
1242 "Number of inode tracker path hits.",
1243 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_hits_path)));
1244 format_metric("cvmfs_inode_tracker_misses_path_total", "counter",
1245 "Number of inode tracker path misses.",
1246 "repo=\"" + fqrn + "\"", StringifyInt(atomic_read64(&inode_stats.num_misses_path)));
1247
1248 // Dentry tracker metrics
1249 format_metric("cvmfs_dentry_tracker_inserts_total", "counter",
1250 "Number of dentry tracker insertions.",
1251 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_insert));
1252 format_metric("cvmfs_dentry_tracker_removes_total", "counter",
1253 "Number of dentry tracker removals.",
1254 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_remove));
1255 format_metric("cvmfs_dentry_tracker_prunes_total", "counter",
1256 "Number of dentry tracker prunes.",
1257 "repo=\"" + fqrn + "\"", StringifyInt(dentry_stats.num_prune));
1258
1259 // Page cache tracker metrics
1260 format_metric("cvmfs_page_cache_tracker_inserts_total", "counter",
1261 "Number of page cache tracker insertions.",
1262 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_insert));
1263 format_metric("cvmfs_page_cache_tracker_removes_total", "counter",
1264 "Number of page cache tracker removals.",
1265 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_remove));
1266 format_metric("cvmfs_page_cache_tracker_opens_direct_total", "counter",
1267 "Number of page cache tracker direct opens.",
1268 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_direct));
1269 format_metric("cvmfs_page_cache_tracker_opens_flush_total", "counter",
1270 "Number of page cache tracker flush opens.",
1271 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_flush));
1272 format_metric("cvmfs_page_cache_tracker_opens_cached_total", "counter",
1273 "Number of page cache tracker cached opens.",
1274 "repo=\"" + fqrn + "\"", StringifyInt(page_cache_stats.n_open_cached));
1275
1276 // Cache mode information
1277 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
1278 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(file_system->cache_mgr());
1279 int cache_mode_value = 0;
1280 switch (cache_mgr->cache_mode()) {
1281 case PosixCacheManager::kCacheReadWrite:
1282 cache_mode_value = 1;
1283 break;
1284 case PosixCacheManager::kCacheReadOnly:
1285 cache_mode_value = 2;
1286 break;
1287 default:
1288 cache_mode_value = 0;
1289 }
1290 format_metric("cvmfs_cache_mode", "gauge",
1291 "Cache mode (0=unknown, 1=read-write, 2=read-only).",
1292 "repo=\"" + fqrn + "\"", StringifyInt(cache_mode_value));
1293 }
1294
1295 // Drainout and maintenance mode
1296 bool drainout_mode;
1297 bool maintenance_mode;
1298 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
1299 format_metric("cvmfs_drainout_mode", "gauge",
1300 "Drainout mode status (0=false, 1=true).",
1301 "repo=\"" + fqrn + "\"", StringifyInt(drainout_mode ? 1 : 0));
1302 format_metric("cvmfs_maintenance_mode", "gauge",
1303 "Maintenance mode status (0=false, 1=true).",
1304 "repo=\"" + fqrn + "\"", StringifyInt(maintenance_mode ? 1 : 0));
1305
1306 // SQLite statistics
1307 int current, highwater;
1308
1309 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
1310 format_metric("cvmfs_sqlite_malloc_count", "gauge",
1311 "Number of SQLite allocations.",
1312 "repo=\"" + fqrn + "\"", StringifyInt(current));
1313
1314 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
1315 format_metric("cvmfs_sqlite_memory_used_bytes", "gauge",
1316 "SQLite general purpose allocator memory used.",
1317 "repo=\"" + fqrn + "\"", StringifyInt(current));
1318 format_metric("cvmfs_sqlite_memory_used_highwater_bytes", "gauge",
1319 "SQLite general purpose allocator memory used high water mark.",
1320 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1321
1322 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
1323 format_metric("cvmfs_sqlite_largest_malloc_bytes", "gauge",
1324 "SQLite largest malloc size.",
1325 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1326
1327 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
1328 format_metric("cvmfs_sqlite_pagecache_used", "gauge",
1329 "SQLite page cache allocations used.",
1330 "repo=\"" + fqrn + "\"", StringifyInt(current));
1331 format_metric("cvmfs_sqlite_pagecache_used_highwater", "gauge",
1332 "SQLite page cache allocations used high water mark.",
1333 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1334
1335 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
1336 format_metric("cvmfs_sqlite_pagecache_overflow_bytes", "gauge",
1337 "SQLite page cache overflow bytes.",
1338 "repo=\"" + fqrn + "\"", StringifyInt(current));
1339 format_metric("cvmfs_sqlite_pagecache_overflow_highwater_bytes", "gauge",
1340 "SQLite page cache overflow bytes high water mark.",
1341 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1342
1343 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
1344 format_metric("cvmfs_sqlite_largest_pagecache_bytes", "gauge",
1345 "SQLite largest page cache allocation size.",
1346 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1347
1348 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
1349 format_metric("cvmfs_sqlite_scratch_used", "gauge",
1350 "SQLite scratch allocations used.",
1351 "repo=\"" + fqrn + "\"", StringifyInt(current));
1352 format_metric("cvmfs_sqlite_scratch_used_highwater", "gauge",
1353 "SQLite scratch allocations used high water mark.",
1354 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1355
1356 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
1357 format_metric("cvmfs_sqlite_scratch_overflow", "gauge",
1358 "SQLite scratch overflows.",
1359 "repo=\"" + fqrn + "\"", StringifyInt(current));
1360 format_metric("cvmfs_sqlite_scratch_overflow_highwater", "gauge",
1361 "SQLite scratch overflows high water mark.",
1362 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1363
1364 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
1365 format_metric("cvmfs_sqlite_largest_scratch_bytes", "gauge",
1366 "SQLite largest scratch allocation size.",
1367 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1368
1369 // NFS statistics (if applicable)
1370 if (file_system->IsNfsSource()) {
1371 format_metric("cvmfs_nfs_mode", "gauge",
1372 "NFS mode enabled (1=true, 0=false).",
1373 "repo=\"" + fqrn + "\"", "1");
1374 // Note: NFS map statistics are complex strings, skipping detailed parsing for now
1375 } else {
1376 format_metric("cvmfs_nfs_mode", "gauge",
1377 "NFS mode enabled (1=true, 0=false).",
1378 "repo=\"" + fqrn + "\"", "0");
1379 }
1380
1381 return result;
1382 }
1383
1384 TalkManager::TalkManager(const string &socket_path,
1385 MountPoint *mount_point,
1386 FuseRemounter *remounter)
1387 : socket_path_(socket_path)
1388 , socket_fd_(-1)
1389 , mount_point_(mount_point)
1390 , remounter_(remounter)
1391 , spawned_(false) {
1392 memset(&thread_talk_, 0, sizeof(thread_talk_));
1393 }
1394
1395
1396 TalkManager::~TalkManager() {
1397 if (!socket_path_.empty()) {
1398 const int retval = unlink(socket_path_.c_str());
1399 if ((retval != 0) && (errno != ENOENT)) {
1400 LogCvmfs(kLogTalk, kLogSyslogWarn,
1401 "Could not remove cvmfs_io socket from cache directory (%d)",
1402 errno);
1403 }
1404 }
1405
1406 if (socket_fd_ >= 0) {
1407 shutdown(socket_fd_, SHUT_RDWR);
1408 close(socket_fd_);
1409 }
1410
1411 if (spawned_) {
1412 pthread_join(thread_talk_, NULL);
1413 LogCvmfs(kLogTalk, kLogDebug, "talk thread stopped");
1414 }
1415 }
1416
1417
1418 void TalkManager::Spawn() {
1419 const int retval = pthread_create(&thread_talk_, NULL, MainResponder, this);
1420 assert(retval == 0);
1421 spawned_ = true;
1422 }
1423