GCC Code Coverage Report


Directory: cvmfs/
File: cvmfs/talk.cc
Date: 2026-08-30 02:40:36
Exec Total Coverage
Lines: 0 931 0.0%
Branches: 0 2896 0.0%

Line Branch Exec Source
1 /**
2 * This file is part of the CernVM File System.
3 *
4 * Implements a socket interface to cvmfs. This way commands can be send
5 * to cvmfs. When cvmfs is running, the socket
6 * /var/cache/cvmfs2/$INSTANCE/cvmfs_io
7 * is available for command input and reply messages, resp.
8 *
9 * Cvmfs comes with the cvmfs_talk script, that handles writing and reading the
10 * socket.
11 *
12 * The talk module runs in a separate thread.
13 */
14
15 #include "talk.h"
16
17 #include <errno.h>
18 #include <inttypes.h>
19 #include <pthread.h>
20 #include <stdint.h>
21 #include <sys/socket.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <sys/uio.h>
25 #include <sys/un.h>
26 #include <unistd.h>
27
28 #include <cassert>
29 #include <cstdlib>
30 #include <cstring>
31 #include <memory>
32 #include <string>
33 #include <vector>
34
35 #include "cache.h"
36 #include "cache_posix.h"
37 #include "catalog_mgr_client.h"
38 #include "cvmfs.h"
39 #include "duplex_sqlite3.h"
40 #include "fuse_remount.h"
41 #include "glue_buffer.h"
42 #include "loader.h"
43 #include "lru_md.h"
44 #include "monitor.h"
45 #include "mountpoint.h"
46 #include "network/download.h"
47 #include "nfs_maps.h"
48 #include "options.h"
49 #include "quota.h"
50 #include "shortstring.h"
51 #include "statistics.h"
52 #include "tracer.h"
53 #include "util/logging.h"
54 #include "wpad.h"
55
56 using namespace std; // NOLINT
57
58
59 void TalkManager::Answer(int con_fd, const string &msg) {
60 (void)send(con_fd, &msg[0], msg.length(), MSG_NOSIGNAL);
61 }
62
63
64 void TalkManager::AnswerStringList(int con_fd, const vector<string> &list) {
65 string list_str;
66 for (unsigned i = 0; i < list.size(); ++i) {
67 list_str += list[i] + "\n";
68 }
69 Answer(con_fd, list_str);
70 }
71
72
73 TalkManager *TalkManager::Create(const string &socket_path,
74 MountPoint *mount_point,
75 FuseRemounter *remounter) {
76 std::unique_ptr<TalkManager> talk_manager(
77 new TalkManager(socket_path, mount_point, remounter));
78
79 talk_manager->socket_fd_ = MakeSocket(socket_path, 0660);
80 if (talk_manager->socket_fd_ == -1)
81 return NULL;
82 if (listen(talk_manager->socket_fd_, 1) == -1)
83 return NULL;
84
85 LogCvmfs(kLogTalk, kLogDebug, "socket created at %s (fd %d)",
86 socket_path.c_str(), talk_manager->socket_fd_);
87
88 return talk_manager.release();
89 }
90
91
92 string TalkManager::FormatMetalinkInfo(
93 download::DownloadManager *download_mgr) {
94 vector<string> metalink_chain;
95 unsigned active_metalink;
96
97 download_mgr->GetMetalinkInfo(&metalink_chain, &active_metalink);
98 if (metalink_chain.size() == 0)
99 return "No metalinks defined\n";
100
101 string metalink_str;
102 for (unsigned i = 0; i < metalink_chain.size(); ++i) {
103 metalink_str += " [" + StringifyInt(i) + "] " + metalink_chain[i] + "\n";
104 }
105 metalink_str += "Active metalink " + StringifyInt(active_metalink) + ": "
106 + metalink_chain[active_metalink] + "\n";
107 return metalink_str;
108 }
109
110 string TalkManager::FormatHostInfo(download::DownloadManager *download_mgr) {
111 vector<string> host_chain;
112 vector<int> rtt;
113 unsigned active_host;
114
115 download_mgr->GetHostInfo(&host_chain, &rtt, &active_host);
116 if (host_chain.size() == 0)
117 return "No hosts defined\n";
118
119 string host_str;
120 for (unsigned i = 0; i < host_chain.size(); ++i) {
121 host_str += " [" + StringifyInt(i) + "] " + host_chain[i] + " (";
122 if (rtt[i] == download::DownloadManager::kProbeUnprobed)
123 host_str += "unprobed";
124 else if (rtt[i] == download::DownloadManager::kProbeDown)
125 host_str += "host down";
126 else if (rtt[i] == download::DownloadManager::kProbeGeo)
127 host_str += "geographically ordered";
128 else
129 host_str += StringifyInt(rtt[i]) + " ms";
130 host_str += ")\n";
131 }
132 host_str += "Active host " + StringifyInt(active_host) + ": "
133 + host_chain[active_host] + "\n";
134 return host_str;
135 }
136
137 string TalkManager::FormatProxyInfo(download::DownloadManager *download_mgr) {
138 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
139 unsigned active_group;
140 unsigned fallback_group;
141
142 download_mgr->GetProxyInfo(&proxy_chain, &active_group, &fallback_group);
143 string proxy_str;
144 if (proxy_chain.size()) {
145 proxy_str += "Load-balance groups:\n";
146 for (unsigned i = 0; i < proxy_chain.size(); ++i) {
147 vector<string> urls;
148 for (unsigned j = 0; j < proxy_chain[i].size(); ++j) {
149 urls.push_back(proxy_chain[i][j].Print());
150 }
151 proxy_str += "[" + StringifyInt(i) + "] " + JoinStrings(urls, ", ")
152 + "\n";
153 }
154 proxy_str += "Active proxy: [" + StringifyInt(active_group) + "] "
155 + proxy_chain[active_group][0].url + "\n";
156 if (fallback_group < proxy_chain.size())
157 proxy_str += "First fallback group: [" + StringifyInt(fallback_group)
158 + "]\n";
159 } else {
160 proxy_str = "No proxies defined\n";
161 }
162 return proxy_str;
163 }
164
165
166 /**
167 * Listener thread on the socket.
168 * TODO(jblomer): create Format... helpers to shorten this method
169 */
170 void *TalkManager::MainResponder(void *data) {
171 TalkManager *talk_mgr = reinterpret_cast<TalkManager *>(data);
172 MountPoint *mount_point = talk_mgr->mount_point_;
173 FileSystem *file_system = mount_point->file_system();
174 FuseRemounter *remounter = talk_mgr->remounter_;
175 LogCvmfs(kLogTalk, kLogDebug, "talk thread started");
176
177 struct sockaddr_un remote;
178 socklen_t socket_size = sizeof(remote);
179 int con_fd = -1;
180 while (true) {
181 if (con_fd >= 0) {
182 shutdown(con_fd, SHUT_RDWR);
183 close(con_fd);
184 }
185 LogCvmfs(kLogTalk, kLogDebug, "accepting connections on socketfd %d",
186 talk_mgr->socket_fd_);
187 if ((con_fd = accept(talk_mgr->socket_fd_, (struct sockaddr *)&remote,
188 &socket_size))
189 < 0) {
190 LogCvmfs(kLogTalk, kLogDebug, "terminating talk thread (fd %d, errno %d)",
191 con_fd, errno);
192 break;
193 }
194
195 char buf[kMaxCommandSize];
196 int bytes_read;
197 if ((bytes_read = recv(con_fd, buf, sizeof(buf), 0)) <= 0)
198 continue;
199
200 if (buf[bytes_read - 1] == '\0')
201 bytes_read--;
202 const string line = string(buf, bytes_read);
203 LogCvmfs(kLogTalk, kLogDebug, "received %s (length %lu)", line.c_str(),
204 line.length());
205
206 if (line == "tracebuffer flush") {
207 mount_point->tracer()->Flush();
208 talk_mgr->Answer(con_fd, "OK\n");
209 } else if (line == "cache size") {
210 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
211 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
212 talk_mgr->Answer(con_fd, "Cache cannot report its size\n");
213 } else {
214 const uint64_t size_unpinned = quota_mgr->GetSize();
215 const uint64_t size_pinned = quota_mgr->GetSizePinned();
216 const string size_str = "Current cache size is "
217 + StringifyInt(size_unpinned / (1024 * 1024))
218 + "MB (" + StringifyInt(size_unpinned)
219 + " Bytes), pinned: "
220 + StringifyInt(size_pinned / (1024 * 1024))
221 + "MB (" + StringifyInt(size_pinned)
222 + " Bytes)\n";
223 talk_mgr->Answer(con_fd, size_str);
224 }
225 } else if (line == "cache instance") {
226 talk_mgr->Answer(con_fd, file_system->cache_mgr()->Describe());
227 } else if (line == "cache list") {
228 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
229 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
230 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
231 } else {
232 const vector<string> ls = quota_mgr->List();
233 talk_mgr->AnswerStringList(con_fd, ls);
234 }
235 } else if (line == "cache list pinned") {
236 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
237 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
238 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
239 } else {
240 const vector<string> ls_pinned = quota_mgr->ListPinned();
241 talk_mgr->AnswerStringList(con_fd, ls_pinned);
242 }
243 } else if (line == "cache list catalogs") {
244 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
245 if (!quota_mgr->HasCapability(QuotaManager::kCapList)) {
246 talk_mgr->Answer(con_fd, "Cache cannot list its entries\n");
247 } else {
248 const vector<string> ls_catalogs = quota_mgr->ListCatalogs();
249 talk_mgr->AnswerStringList(con_fd, ls_catalogs);
250 }
251 } else if (line.substr(0, 12) == "cleanup rate") {
252 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
253 if (!quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
254 talk_mgr->Answer(con_fd, "Unsupported by this cache\n");
255 } else {
256 if (line.length() < 14) {
257 talk_mgr->Answer(con_fd, "Usage: cleanup rate <period in mn>\n");
258 } else {
259 const uint64_t period_s = String2Uint64(line.substr(13)) * 60;
260 const uint64_t rate = quota_mgr->GetCleanupRate(period_s);
261 talk_mgr->Answer(con_fd, StringifyInt(rate) + "\n");
262 }
263 }
264 } else if (line.substr(0, 15) == "cache limit set") {
265 if (line.length() < 16) {
266 talk_mgr->Answer(con_fd, "Usage: cache limit set <MB>\n");
267 } else {
268 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
269 const uint64_t size = String2Uint64(line.substr(16));
270 if (size < 1000) {
271 talk_mgr->Answer(con_fd, "New limit too low (minimum 1000)\n");
272 } else {
273 if (quota_mgr->SetLimit(size * 1024 * 1024)) {
274 file_system->options_mgr()->SetValueFromTalk("CVMFS_QUOTA_LIMIT",
275 StringifyUint(size));
276 talk_mgr->Answer(con_fd, "OK\n");
277 } else {
278 talk_mgr->Answer(con_fd, "Limit not reset\n");
279 }
280 }
281 }
282 } else if (line == "cache limit get") {
283 std::string limit_from_options;
284 file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT",
285 &limit_from_options);
286 talk_mgr->Answer(con_fd, limit_from_options + "\n");
287 } else if (line.substr(0, 7) == "cleanup") {
288 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
289 if (!quota_mgr->HasCapability(QuotaManager::kCapShrink)) {
290 talk_mgr->Answer(con_fd, "Cache cannot trigger eviction\n");
291 } else {
292 if (line.length() < 9) {
293 talk_mgr->Answer(con_fd, "Usage: cleanup <MB>\n");
294 } else {
295 const uint64_t size = String2Uint64(line.substr(8)) * 1024 * 1024;
296 if (quota_mgr->Cleanup(size)) {
297 talk_mgr->Answer(con_fd, "OK\n");
298 } else {
299 talk_mgr->Answer(con_fd, "Not fully cleaned "
300 "(there might be pinned chunks)\n");
301 }
302 }
303 }
304 } else if (line.substr(0, 5) == "evict") {
305 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
306 if (line.length() < 7) {
307 talk_mgr->Answer(con_fd, "Usage: evict <path>\n");
308 } else {
309 const string path = line.substr(6);
310 const bool found_regular = cvmfs::Evict(path);
311 if (found_regular)
312 talk_mgr->Answer(con_fd, "OK\n");
313 else
314 talk_mgr->Answer(con_fd, "No such regular file\n");
315 }
316 } else if (line.substr(0, 3) == "pin") {
317 assert(mount_point->file_system()->type() == FileSystem::kFsFuse);
318 if (line.length() < 5) {
319 talk_mgr->Answer(con_fd, "Usage: pin <path>\n");
320 } else {
321 const string path = line.substr(4);
322 const bool found_regular = cvmfs::Pin(path);
323 if (found_regular)
324 talk_mgr->Answer(con_fd, "OK\n");
325 else
326 talk_mgr->Answer(con_fd, "No such regular file or pinning failed\n");
327 }
328 } else if (line == "mountpoint") {
329 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->mount_point + "\n");
330 } else if (line == "device id") {
331 if (cvmfs::loader_exports_->version >= 5)
332 talk_mgr->Answer(con_fd, cvmfs::loader_exports_->device_id + "\n");
333 else
334 talk_mgr->Answer(con_fd, "0:0\n");
335 } else if (line.substr(0, 13) == "send mount fd") {
336 // Hidden command intended to be used only by the cvmfs mount helper
337 if (line.length() < 15) {
338 talk_mgr->Answer(con_fd, "EINVAL\n");
339 } else {
340 const std::string socket_path = line.substr(14);
341 const bool retval = cvmfs::SendFuseFd(socket_path);
342 talk_mgr->Answer(con_fd, retval ? "OK\n" : "Failed\n");
343 LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslog,
344 "Attempt to send fuse connection info to new mount (via %s)%s",
345 socket_path.c_str(), retval ? "" : " -- failed!");
346 }
347 } else if (line.substr(0, 7) == "remount") {
348 FuseRemounter::Status status;
349 if (line == "remount sync")
350 status = remounter->CheckSynchronously();
351 else
352 status = remounter->Check();
353 switch (status) {
354 case FuseRemounter::kStatusFailGeneral:
355 talk_mgr->Answer(con_fd, "Failed\n");
356 break;
357 case FuseRemounter::kStatusFailNoSpace:
358 talk_mgr->Answer(con_fd, "Failed (no space)\n");
359 break;
360 case FuseRemounter::kStatusUp2Date:
361 talk_mgr->Answer(con_fd, "Catalog up to date\n");
362 break;
363 case FuseRemounter::kStatusDraining:
364 talk_mgr->Answer(con_fd, "New revision applied\n");
365 break;
366 case FuseRemounter::kStatusMaintenance:
367 talk_mgr->Answer(con_fd, "In maintenance mode\n");
368 break;
369 default:
370 talk_mgr->Answer(con_fd, "internal error\n");
371 }
372 } else if (line.substr(0, 6) == "chroot") {
373 if (line.length() < 8) {
374 talk_mgr->Answer(con_fd, "Usage: chroot <hash>\n");
375 } else {
376 const std::string root_hash = Trim(line.substr(7),
377 true /* trim_newline */);
378 const FuseRemounter::Status status = remounter->ChangeRoot(
379 MkFromHexPtr(shash::HexPtr(root_hash), shash::kSuffixCatalog));
380 switch (status) {
381 case FuseRemounter::kStatusUp2Date:
382 talk_mgr->Answer(con_fd, "OK\n");
383 break;
384 default:
385 talk_mgr->Answer(con_fd, "Failed\n");
386 break;
387 }
388 }
389 } else if (line == "detach nested catalogs") {
390 mount_point->catalog_mgr()->DetachNested();
391 talk_mgr->Answer(con_fd, "OK\n");
392 } else if (line == "revision") {
393 const string revision = StringifyInt(
394 mount_point->catalog_mgr()->GetRevision());
395 talk_mgr->Answer(con_fd, revision + "\n");
396 } else if (line == "max ttl info") {
397 const unsigned max_ttl = mount_point->GetMaxTtlMn();
398 if (max_ttl == 0) {
399 talk_mgr->Answer(con_fd, "unset\n");
400 } else {
401 const string max_ttl_str = StringifyInt(max_ttl) + " minutes\n";
402 talk_mgr->Answer(con_fd, max_ttl_str);
403 }
404 } else if (line.substr(0, 11) == "max ttl set") {
405 if (line.length() < 13) {
406 talk_mgr->Answer(con_fd, "Usage: max ttl set <minutes>\n");
407 } else {
408 const unsigned max_ttl = String2Uint64(line.substr(12));
409 mount_point->SetMaxTtlMn(max_ttl);
410 talk_mgr->Answer(con_fd, "OK\n");
411 }
412 } else if (line.substr(0, 14) == "nameserver get") {
413 const string dns_server = mount_point->download_mgr()->GetDnsServer();
414 const string reply = !dns_server.empty()
415 ? std::string("DNS server address: ")
416 + dns_server + "\n"
417 : std::string("DNS server not set.\n");
418 talk_mgr->Answer(con_fd, reply);
419 } else if (line.substr(0, 14) == "nameserver set") {
420 if (line.length() < 16) {
421 talk_mgr->Answer(con_fd, "Usage: nameserver set <host>\n");
422 } else {
423 const string host = line.substr(15);
424 mount_point->download_mgr()->SetDnsServer(host);
425 talk_mgr->Answer(con_fd, "OK\n");
426 }
427 } else if (line.substr(0, 22) == "__testing_freeze_cvmfs") {
428 const std::string fs_dir = line.substr(23) + "/dir";
429 mkdir(fs_dir.c_str(), 0700);
430 } else if (line == "external metalink info") {
431 const string external_metalink_info = talk_mgr->FormatMetalinkInfo(
432 mount_point->external_download_mgr());
433 talk_mgr->Answer(con_fd, external_metalink_info);
434 } else if (line == "metalink info") {
435 const string metalink_info = talk_mgr->FormatMetalinkInfo(
436 mount_point->download_mgr());
437 talk_mgr->Answer(con_fd, metalink_info);
438 } else if (line == "external host info") {
439 const string external_host_info = talk_mgr->FormatHostInfo(
440 mount_point->external_download_mgr());
441 talk_mgr->Answer(con_fd, external_host_info);
442 } else if (line == "host info") {
443 const string host_info = talk_mgr->FormatHostInfo(
444 mount_point->download_mgr());
445 talk_mgr->Answer(con_fd, host_info);
446 } else if (line == "host probe") {
447 mount_point->download_mgr()->ProbeHosts();
448 talk_mgr->Answer(con_fd, "OK\n");
449 } else if (line == "host probe geo") {
450 const bool retval = mount_point->download_mgr()->ProbeGeo();
451 if (retval)
452 talk_mgr->Answer(con_fd, "OK\n");
453 else
454 talk_mgr->Answer(con_fd, "Failed\n");
455 } else if (line == "external metalink switch") {
456 mount_point->external_download_mgr()->SwitchMetalink();
457 talk_mgr->Answer(con_fd, "OK\n");
458 } else if (line == "metalink switch") {
459 mount_point->download_mgr()->SwitchMetalink();
460 talk_mgr->Answer(con_fd, "OK\n");
461 } else if (line == "external host switch") {
462 mount_point->external_download_mgr()->SwitchHost();
463 talk_mgr->Answer(con_fd, "OK\n");
464 } else if (line == "host switch") {
465 mount_point->download_mgr()->SwitchHost();
466 talk_mgr->Answer(con_fd, "OK\n");
467 } else if (line.substr(0, 21) == "external metalink set") {
468 if (line.length() < 23) {
469 talk_mgr->Answer(con_fd, "Usage: external metalink set <URL>\n");
470 } else {
471 const std::string host = line.substr(22);
472 mount_point->external_download_mgr()->SetMetalinkChain(host);
473 talk_mgr->Answer(con_fd, "OK\n");
474 }
475 } else if (line.substr(0, 12) == "metalink set") {
476 if (line.length() < 14) {
477 talk_mgr->Answer(con_fd, "Usage: metalink set <URL>\n");
478 } else {
479 const std::string host = line.substr(13);
480 mount_point->download_mgr()->SetMetalinkChain(host);
481 talk_mgr->Answer(con_fd, "OK\n");
482 }
483 } else if (line.substr(0, 17) == "external host set") {
484 if (line.length() < 19) {
485 talk_mgr->Answer(con_fd, "Usage: external host set <URL>\n");
486 } else {
487 const std::string host = line.substr(18);
488 mount_point->external_download_mgr()->SetHostChain(host);
489 talk_mgr->Answer(con_fd, "OK\n");
490 }
491 } else if (line.substr(0, 8) == "host set") {
492 if (line.length() < 10) {
493 talk_mgr->Answer(con_fd, "Usage: host set <host list>\n");
494 } else {
495 const string hosts = line.substr(9);
496 mount_point->download_mgr()->SetHostChain(hosts);
497 talk_mgr->Answer(con_fd, "OK\n");
498 }
499 } else if (line == "external proxy info") {
500 const string external_proxy_info = talk_mgr->FormatProxyInfo(
501 mount_point->external_download_mgr());
502 talk_mgr->Answer(con_fd, external_proxy_info);
503 } else if (line == "proxy info") {
504 const string proxy_info = talk_mgr->FormatProxyInfo(
505 mount_point->download_mgr());
506 talk_mgr->Answer(con_fd, proxy_info);
507 } else if (line == "proxy rebalance") {
508 mount_point->download_mgr()->RebalanceProxies();
509 talk_mgr->Answer(con_fd, "OK\n");
510 } else if (line == "proxy group switch") {
511 mount_point->download_mgr()->SwitchProxyGroup();
512 talk_mgr->Answer(con_fd, "OK\n");
513 } else if (line.substr(0, 18) == "external proxy set") {
514 if (line.length() < 20) {
515 talk_mgr->Answer(con_fd, "Usage: external proxy set <proxy list>\n");
516 } else {
517 const string external_proxies = line.substr(19);
518 mount_point->external_download_mgr()->SetProxyChain(
519 external_proxies, "", download::DownloadManager::kSetProxyRegular);
520 talk_mgr->Answer(con_fd, "OK\n");
521 }
522 } else if (line.substr(0, 9) == "proxy set") {
523 if (line.length() < 11) {
524 talk_mgr->Answer(con_fd, "Usage: proxy set <proxy list>\n");
525 } else {
526 string proxies = line.substr(10);
527 proxies = download::ResolveProxyDescription(
528 proxies, "", mount_point->download_mgr());
529 if (proxies == "") {
530 talk_mgr->Answer(con_fd, "Failed, no valid proxies\n");
531 } else {
532 mount_point->download_mgr()->SetProxyChain(
533 proxies, "", download::DownloadManager::kSetProxyRegular);
534 talk_mgr->Answer(con_fd, "OK\n");
535 }
536 }
537 } else if (line.substr(0, 14) == "proxy fallback") {
538 if (line.length() < 15) {
539 talk_mgr->Answer(con_fd, "Usage: proxy fallback <proxy list>\n");
540 } else {
541 const string fallback_proxies = line.substr(15);
542 mount_point->download_mgr()->SetProxyChain(
543 "", fallback_proxies, download::DownloadManager::kSetProxyFallback);
544 talk_mgr->Answer(con_fd, "OK\n");
545 }
546 } else if (line == "timeout info") {
547 unsigned timeout;
548 unsigned timeout_direct;
549 mount_point->download_mgr()->GetTimeout(&timeout, &timeout_direct);
550 string timeout_str = "Timeout with proxy: ";
551 if (timeout)
552 timeout_str += StringifyInt(timeout) + "s\n";
553 else
554 timeout_str += "no timeout\n";
555 timeout_str += "Timeout without proxy: ";
556 if (timeout_direct)
557 timeout_str += StringifyInt(timeout_direct) + "s\n";
558 else
559 timeout_str += "no timeout\n";
560 talk_mgr->Answer(con_fd, timeout_str);
561 } else if (line.substr(0, 11) == "timeout set") {
562 if (line.length() < 13) {
563 talk_mgr->Answer(con_fd, "Usage: timeout set <proxy> <direct>\n");
564 } else {
565 uint64_t timeout;
566 uint64_t timeout_direct;
567 String2Uint64Pair(line.substr(12), &timeout, &timeout_direct);
568 mount_point->download_mgr()->SetTimeout(timeout, timeout_direct);
569 talk_mgr->Answer(con_fd, "OK\n");
570 }
571 } else if (line == "open catalogs") {
572 talk_mgr->Answer(con_fd, mount_point->catalog_mgr()->PrintHierarchy());
573 } else if (line == "drop metadata caches") {
574 // For testing
575 mount_point->inode_cache()->Pause();
576 mount_point->path_cache()->Pause();
577 mount_point->md5path_cache()->Pause();
578 mount_point->inode_cache()->Drop();
579 mount_point->path_cache()->Drop();
580 mount_point->md5path_cache()->Drop();
581 mount_point->inode_cache()->Resume();
582 mount_point->path_cache()->Resume();
583 mount_point->md5path_cache()->Resume();
584 talk_mgr->Answer(con_fd, "OK\n");
585 } else if (line == "internal affairs") {
586 int current;
587 int highwater;
588 string result;
589
590 result += "Inode Generation:\n " + cvmfs::PrintInodeGeneration();
591
592 // Manually setting the values of the ShortString counters
593 mount_point->statistics()
594 ->Lookup("pathstring.n_instances")
595 ->Set(PathString::num_instances());
596 mount_point->statistics()
597 ->Lookup("pathstring.n_overflows")
598 ->Set(PathString::num_overflows());
599 mount_point->statistics()
600 ->Lookup("namestring.n_instances")
601 ->Set(NameString::num_instances());
602 mount_point->statistics()
603 ->Lookup("namestring.n_overflows")
604 ->Set(NameString::num_overflows());
605 mount_point->statistics()
606 ->Lookup("linkstring.n_instances")
607 ->Set(LinkString::num_instances());
608 mount_point->statistics()
609 ->Lookup("linkstring.n_overflows")
610 ->Set(LinkString::num_overflows());
611
612 // Manually setting the inode tracker numbers
613 glue::InodeTracker::Statistics inode_stats = mount_point->inode_tracker()
614 ->GetStatistics();
615 const glue::DentryTracker::Statistics
616 dentry_stats = mount_point->dentry_tracker()->GetStatistics();
617 const glue::PageCacheTracker::Statistics
618 page_cache_stats = mount_point->page_cache_tracker()->GetStatistics();
619 mount_point->statistics()
620 ->Lookup("inode_tracker.n_insert")
621 ->Set(atomic_read64(&inode_stats.num_inserts));
622 mount_point->statistics()
623 ->Lookup("inode_tracker.n_remove")
624 ->Set(atomic_read64(&inode_stats.num_removes));
625 mount_point->statistics()
626 ->Lookup("inode_tracker.no_reference")
627 ->Set(atomic_read64(&inode_stats.num_references));
628 mount_point->statistics()
629 ->Lookup("inode_tracker.n_hit_inode")
630 ->Set(atomic_read64(&inode_stats.num_hits_inode));
631 mount_point->statistics()
632 ->Lookup("inode_tracker.n_hit_path")
633 ->Set(atomic_read64(&inode_stats.num_hits_path));
634 mount_point->statistics()
635 ->Lookup("inode_tracker.n_miss_path")
636 ->Set(atomic_read64(&inode_stats.num_misses_path));
637 mount_point->statistics()
638 ->Lookup("dentry_tracker.n_insert")
639 ->Set(dentry_stats.num_insert);
640 mount_point->statistics()
641 ->Lookup("dentry_tracker.n_remove")
642 ->Set(dentry_stats.num_remove);
643 mount_point->statistics()
644 ->Lookup("dentry_tracker.n_prune")
645 ->Set(dentry_stats.num_prune);
646 mount_point->statistics()
647 ->Lookup("page_cache_tracker.n_insert")
648 ->Set(page_cache_stats.n_insert);
649 mount_point->statistics()
650 ->Lookup("page_cache_tracker.n_remove")
651 ->Set(page_cache_stats.n_remove);
652 mount_point->statistics()
653 ->Lookup("page_cache_tracker.n_open_direct")
654 ->Set(page_cache_stats.n_open_direct);
655 mount_point->statistics()
656 ->Lookup("page_cache_tracker.n_open_flush")
657 ->Set(page_cache_stats.n_open_flush);
658 mount_point->statistics()
659 ->Lookup("page_cache_tracker.n_open_cached")
660 ->Set(page_cache_stats.n_open_cached);
661
662 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
663 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(
664 file_system->cache_mgr());
665 result += "\nCache Mode: ";
666 switch (cache_mgr->cache_mode()) {
667 case PosixCacheManager::kCacheReadWrite:
668 result += "read-write";
669 break;
670 case PosixCacheManager::kCacheReadOnly:
671 result += "read-only";
672 break;
673 default:
674 result += "unknown";
675 }
676 }
677 bool drainout_mode;
678 bool maintenance_mode;
679 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
680 result += "\nDrainout Mode: " + StringifyBool(drainout_mode) + "\n";
681 result += "Maintenance Mode: " + StringifyBool(maintenance_mode) + "\n";
682
683 if (file_system->IsNfsSource()) {
684 result += "\nNFS Map Statistics:\n";
685 result += file_system->nfs_maps()->GetStatistics();
686 }
687
688 result += "SQlite Statistics:\n";
689 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
690 result += " Number of allocations " + StringifyInt(current) + "\n";
691
692 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
693 result += " General purpose allocator " + StringifyInt(current / 1024)
694 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
695
696 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
697 result += " Largest malloc " + StringifyInt(highwater) + " Bytes\n";
698
699 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
700 result += " Page cache allocations " + StringifyInt(current) + " / "
701 + StringifyInt(highwater) + "\n";
702
703 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
704 result += " Page cache overflows " + StringifyInt(current / 1024)
705 + " KB / " + StringifyInt(highwater / 1024) + " KB\n";
706
707 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
708 result += " Largest page cache allocation " + StringifyInt(highwater)
709 + " Bytes\n";
710
711 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
712 result += " Scratch allocations " + StringifyInt(current) + " / "
713 + StringifyInt(highwater) + "\n";
714
715 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
716 result += " Scratch overflows " + StringifyInt(current) + " / "
717 + StringifyInt(highwater) + "\n";
718
719 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
720 result += " Largest scratch allocation " + StringifyInt(highwater / 1024)
721 + " KB\n";
722
723 result += "\nPer-Connection Memory Statistics:\n"
724 + mount_point->catalog_mgr()->PrintAllMemStatistics();
725
726 result += "\nLatency distribution of system calls:\n";
727
728 result += "Lookup\n" + file_system->hist_fs_lookup()->ToString();
729 result += "Forget\n" + file_system->hist_fs_forget()->ToString();
730 result += "Multi-Forget\n"
731 + file_system->hist_fs_forget_multi()->ToString();
732 result += "Getattr\n" + file_system->hist_fs_getattr()->ToString();
733 result += "Readlink\n" + file_system->hist_fs_readlink()->ToString();
734 result += "Opendir\n" + file_system->hist_fs_opendir()->ToString();
735 result += "Releasedir\n" + file_system->hist_fs_releasedir()->ToString();
736 result += "Readdir\n" + file_system->hist_fs_readdir()->ToString();
737 result += "Open\n" + file_system->hist_fs_open()->ToString();
738 result += "Read\n" + file_system->hist_fs_read()->ToString();
739 result += "Release\n" + file_system->hist_fs_release()->ToString();
740
741 result += "\nRaw Counters:\n"
742 + mount_point->statistics()->PrintList(
743 perf::Statistics::kPrintHeader);
744
745 talk_mgr->Answer(con_fd, result);
746 } else if (line == "reset error counters") {
747 file_system->ResetErrorCounters();
748 talk_mgr->Answer(con_fd, "OK\n");
749 } else if (line == "pid") {
750 const string pid_str = StringifyInt(cvmfs::pid_) + "\n";
751 talk_mgr->Answer(con_fd, pid_str);
752 } else if (line == "pid cachemgr") {
753 const string pid_str = StringifyInt(file_system->cache_mgr()
754 ->quota_mgr()
755 ->GetPid())
756 + "\n";
757 talk_mgr->Answer(con_fd, pid_str);
758 } else if (line == "pid watchdog") {
759 const string pid_str = StringifyInt(Watchdog::GetPid()) + "\n";
760 talk_mgr->Answer(con_fd, pid_str);
761 } else if (line == "parameters") {
762 talk_mgr->Answer(con_fd, file_system->options_mgr()->Dump());
763 } else if (line == "hotpatch history") {
764 string history_str = StringifyTime(cvmfs::loader_exports_->boot_time,
765 true)
766 + " (start of CernVM-FS loader "
767 + cvmfs::loader_exports_->loader_version + ")\n";
768 for (loader::EventList::const_iterator
769 i = cvmfs::loader_exports_->history.begin(),
770 iEnd = cvmfs::loader_exports_->history.end();
771 i != iEnd;
772 ++i) {
773 history_str += StringifyTime((*i)->timestamp, true)
774 + " (loaded CernVM-FS Fuse Module " + (*i)->so_version
775 + ")\n";
776 }
777 talk_mgr->Answer(con_fd, history_str);
778 } else if (line == "vfs inodes") {
779 string result;
780 glue::InodeTracker::Cursor cursor(
781 mount_point->inode_tracker()->BeginEnumerate());
782 uint64_t inode;
783 while (mount_point->inode_tracker()->NextInode(&cursor, &inode)) {
784 result += StringifyInt(inode) + "\n";
785 }
786 mount_point->inode_tracker()->EndEnumerate(&cursor);
787 talk_mgr->Answer(con_fd, result);
788 } else if (line == "vfs entries") {
789 string result;
790 glue::InodeTracker::Cursor cursor(
791 mount_point->inode_tracker()->BeginEnumerate());
792 uint64_t inode_parent;
793 NameString name;
794 while (mount_point->inode_tracker()->NextEntry(&cursor, &inode_parent,
795 &name)) {
796 result += "<" + StringifyInt(inode_parent) + ">/" + name.ToString()
797 + "\n";
798 }
799 mount_point->inode_tracker()->EndEnumerate(&cursor);
800 talk_mgr->Answer(con_fd, result);
801 } else if (line == "version") {
802 const string version_str = string(CVMFS_VERSION)
803 + " (CernVM-FS Fuse Module)\n"
804 + cvmfs::loader_exports_->loader_version
805 + " (Loader)\n";
806 talk_mgr->Answer(con_fd, version_str);
807 } else if (line == "version patchlevel") {
808 talk_mgr->Answer(con_fd, string(CVMFS_PATCH_LEVEL) + "\n");
809 } else if (line == "tear down to read-only") {
810 if (file_system->cache_mgr()->id() != kPosixCacheManager) {
811 talk_mgr->Answer(con_fd, "not supported\n");
812 } else {
813 // hack
814 cvmfs::UnregisterQuotaListener();
815 file_system->TearDown2ReadOnly();
816 talk_mgr->Answer(con_fd, "In read-only mode\n");
817 }
818 } else if (line == "latency") {
819 const string result = talk_mgr->FormatLatencies(*mount_point,
820 file_system);
821 talk_mgr->Answer(con_fd, result);
822 } else if (line == "metrics prometheus") {
823 const string result = talk_mgr->FormatPrometheusMetrics(*mount_point,
824 file_system);
825 talk_mgr->Answer(con_fd, result);
826 } else {
827 talk_mgr->Answer(con_fd, "unknown command\n");
828 }
829 }
830
831 return NULL;
832 } // NOLINT(readability/fn_size)
833
834 string TalkManager::FormatLatencies(const MountPoint &mount_point,
835 FileSystem *file_system) {
836 string result;
837 const unsigned int bufSize = 300;
838 char buffer[bufSize];
839
840 vector<float> qs;
841 qs.push_back(.1);
842 qs.push_back(.2);
843 qs.push_back(.25);
844 qs.push_back(.3);
845 qs.push_back(.4);
846 qs.push_back(.5);
847 qs.push_back(.6);
848 qs.push_back(.7);
849 qs.push_back(.75);
850 qs.push_back(.8);
851 qs.push_back(.9);
852 qs.push_back(.95);
853 qs.push_back(.99);
854 qs.push_back(.999);
855 qs.push_back(.9999);
856
857 const string repo(mount_point.fqrn());
858
859 unsigned int format_index = snprintf(
860 buffer, bufSize, "\"%s\",\"%s\",\"%s\",\"%s\"", "repository", "action",
861 "total_count", "time_unit");
862 for (unsigned int i = 0; i < qs.size(); i++) {
863 format_index += snprintf(buffer + format_index, bufSize - format_index,
864 ",%0.5f", qs[i]);
865 }
866 format_index += snprintf(buffer + format_index, bufSize - format_index, "\n");
867 assert(format_index < bufSize);
868
869 result += buffer;
870 memset(buffer, 0, sizeof(buffer));
871 format_index = 0;
872
873 vector<Log2Histogram *> hist;
874 vector<string> names;
875 hist.push_back(file_system->hist_fs_lookup());
876 names.push_back("lookup");
877 hist.push_back(file_system->hist_fs_forget());
878 names.push_back("forget");
879 hist.push_back(file_system->hist_fs_forget_multi());
880 names.push_back("forget_multi");
881 hist.push_back(file_system->hist_fs_getattr());
882 names.push_back("getattr");
883 hist.push_back(file_system->hist_fs_readlink());
884 names.push_back("readlink");
885 hist.push_back(file_system->hist_fs_opendir());
886 names.push_back("opendir");
887 hist.push_back(file_system->hist_fs_releasedir());
888 names.push_back("releasedir");
889 hist.push_back(file_system->hist_fs_readdir());
890 names.push_back("readdir");
891 hist.push_back(file_system->hist_fs_open());
892 names.push_back("open");
893 hist.push_back(file_system->hist_fs_read());
894 names.push_back("read");
895 hist.push_back(file_system->hist_fs_release());
896 names.push_back("release");
897
898 for (unsigned int j = 0; j < hist.size(); j++) {
899 Log2Histogram *h = hist[j];
900 unsigned int format_index = snprintf(
901 buffer, bufSize, "\"%s\",\"%s\",%" PRIu64 ",\"nanoseconds\"",
902 repo.c_str(), names[j].c_str(), h->N());
903 for (unsigned int i = 0; i < qs.size(); i++) {
904 format_index += snprintf(buffer + format_index, bufSize - format_index,
905 ",%u", h->GetQuantile(qs[i]));
906 }
907 format_index += snprintf(buffer + format_index, bufSize - format_index,
908 "\n");
909 assert(format_index < bufSize);
910
911 result += buffer;
912 memset(buffer, 0, sizeof(buffer));
913 format_index = 0;
914 }
915 return result;
916 }
917
918 string TalkManager::FormatPrometheusMetrics(MountPoint &mount_point,
919 FileSystem *file_system) {
920 string result;
921 const string fqrn = mount_point.fqrn();
922 const string mountpoint = cvmfs::loader_exports_->mount_point;
923
924 // Helper function to format a prometheus metric
925 class MetricFormatter {
926 public:
927 explicit MetricFormatter(string &result_ref) : result_(result_ref) { }
928 void operator()(const string &name, const string &type, const string &help,
929 const string &labels, const string &value) {
930 result_ += "# HELP " + name + " " + help + "\n";
931 result_ += "# TYPE " + name + " " + type + "\n";
932 result_ += name + "{" + labels + "} " + value + "\n";
933 }
934
935 private:
936 string &result_;
937 };
938 MetricFormatter format_metric(result);
939
940 // Get cache information
941 QuotaManager *quota_mgr = file_system->cache_mgr()->quota_mgr();
942 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectSize)) {
943 const uint64_t size_unpinned = quota_mgr->GetSize();
944 const uint64_t size_pinned = quota_mgr->GetSizePinned();
945
946 format_metric("cvmfs_cache_cached_bytes", "gauge",
947 "CVMFS currently cached bytes.", "repo=\"" + fqrn + "\"",
948 StringifyUint(size_unpinned));
949 format_metric("cvmfs_cache_pinned_bytes", "gauge",
950 "CVMFS currently pinned bytes.", "repo=\"" + fqrn + "\"",
951 StringifyUint(size_pinned));
952 }
953
954 // Get cache limit from parameters
955 string cache_limit_str;
956 if (file_system->options_mgr()->GetValue("CVMFS_QUOTA_LIMIT",
957 &cache_limit_str)) {
958 const uint64_t cache_limit_mb = String2Uint64(cache_limit_str);
959 const uint64_t cache_limit_bytes = cache_limit_mb * 1024 * 1024;
960 format_metric("cvmfs_cache_total_size_bytes", "gauge",
961 "CVMFS configured cache size via CVMFS_QUOTA_LIMIT.",
962 "repo=\"" + fqrn + "\"", StringifyUint(cache_limit_bytes));
963 }
964
965 // Get cache base directory for df information
966 string cache_base;
967 if (file_system->options_mgr()->GetValue("CVMFS_CACHE_BASE", &cache_base)) {
968 struct statvfs stat_info;
969 if (statvfs(cache_base.c_str(), &stat_info) == 0) {
970 const uint64_t total_size = static_cast<uint64_t>(stat_info.f_blocks)
971 * stat_info.f_frsize;
972 const uint64_t avail_size = static_cast<uint64_t>(stat_info.f_bavail)
973 * stat_info.f_frsize;
974
975 format_metric("cvmfs_cache_physical_size_bytes", "gauge",
976 "CVMFS cache volume physical size.",
977 "repo=\"" + fqrn + "\"", StringifyUint(total_size));
978 format_metric("cvmfs_cache_physical_avail_bytes", "gauge",
979 "CVMFS cache volume physical free space available.",
980 "repo=\"" + fqrn + "\"", StringifyUint(avail_size));
981 }
982 }
983
984 // Version and revision information
985 const string version = string(CVMFS_VERSION) + "."
986 + string(CVMFS_PATCH_LEVEL);
987 const uint64_t revision = mount_point.catalog_mgr()->GetRevision();
988 format_metric("cvmfs_repo", "gauge",
989 "Shows the version of CVMFS used by this repository.",
990 "repo=\"" + fqrn + "\",mountpoint=\"" + mountpoint
991 + "\",version=\"" + version + "\",revision=\""
992 + StringifyUint(revision) + "\"",
993 "1");
994
995 // Statistics-based metrics
996 perf::Statistics *statistics = mount_point.statistics();
997
998 // Download statistics
999 const int64_t rx_bytes = statistics->Lookup("download.sz_transferred_bytes")
1000 ->Get();
1001 format_metric("cvmfs_net_rx_bytes_total", "counter",
1002 "Shows the overall amount of downloaded bytes since mounting.",
1003 "repo=\"" + fqrn + "\"", StringifyInt(rx_bytes));
1004
1005 const int64_t n_downloads = statistics->Lookup("fetch.n_downloads")->Get();
1006 format_metric("cvmfs_net_ndownload_total", "counter",
1007 "Shows the overall number of downloaded files since mounting.",
1008 "repo=\"" + fqrn + "\"", StringifyInt(n_downloads));
1009
1010 // Hit rate calculation
1011 const int64_t n_invocations = statistics->Lookup("fetch.n_invocations")
1012 ->Get();
1013 if (n_invocations > 0) {
1014 const float hit_ratio = 1.0
1015 - (static_cast<float>(n_downloads)
1016 / static_cast<float>(n_invocations));
1017 format_metric("cvmfs_cache_hit_ratio", "gauge",
1018 "CVMFS cache hit ratio (0-1).", "repo=\"" + fqrn + "\"",
1019 StringifyDouble(hit_ratio));
1020 } else {
1021 format_metric("cvmfs_cache_hit_ratio", "gauge",
1022 "CVMFS cache hit ratio (0-1).", "repo=\"" + fqrn + "\"", "0");
1023 }
1024
1025 // Cumulative time spent transferring data. Combined with
1026 // cvmfs_net_rx_bytes_total this lets consumers compute their own average
1027 // download speed, rather than exposing a pre-computed rate with implicit
1028 // units. The underlying statistic is in milliseconds.
1029 const int64_t
1030 transfer_time_ms = statistics->Lookup("download.sz_transfer_time")->Get();
1031 format_metric("cvmfs_net_transfer_time_seconds_total", "counter",
1032 "Cumulative time spent downloading data since mounting.",
1033 "repo=\"" + fqrn + "\"",
1034 StringifyDouble(transfer_time_ms / 1000.0));
1035
1036 // Uptime calculation
1037 const time_t now = time(NULL);
1038 const uint64_t uptime_seconds = now - cvmfs::loader_exports_->boot_time;
1039 const uint64_t mount_epoch_time = now - uptime_seconds;
1040 format_metric("cvmfs_repo_uptime_seconds", "counter",
1041 "Shows the time since the repo was mounted.",
1042 "repo=\"" + fqrn + "\"", StringifyUint(uptime_seconds));
1043 format_metric("cvmfs_repo_mount_timestamp_seconds", "gauge",
1044 "Shows the epoch time the repo was mounted.",
1045 "repo=\"" + fqrn + "\"", StringifyUint(mount_epoch_time));
1046
1047 // Catalog expiry - access through the TalkManager's remounter member
1048 const time_t catalogs_valid_until = remounter_->catalogs_valid_until();
1049 if (catalogs_valid_until != MountPoint::kIndefiniteDeadline) {
1050 const int64_t expires_seconds = (catalogs_valid_until - now);
1051 format_metric("cvmfs_repo_expires_seconds", "gauge",
1052 "Shows the remaining life time of the mounted root file "
1053 "catalog in seconds.",
1054 "repo=\"" + fqrn + "\"", StringifyInt(expires_seconds));
1055 }
1056
1057 // I/O error count
1058 const uint64_t nioerr = file_system->io_error_info()->count();
1059 format_metric(
1060 "cvmfs_sys_nioerr_total", "counter",
1061 "Shows the total number of I/O errors encountered since mounting.",
1062 "repo=\"" + fqrn + "\"", StringifyUint(nioerr));
1063
1064 // Timeout information
1065 unsigned timeout_proxy, timeout_direct;
1066 mount_point.download_mgr()->GetTimeout(&timeout_proxy, &timeout_direct);
1067 format_metric("cvmfs_net_timeout_seconds", "gauge",
1068 "Shows the timeout for proxied connections in seconds.",
1069 "repo=\"" + fqrn + "\"", StringifyUint(timeout_proxy));
1070 format_metric("cvmfs_net_timeout_direct_seconds", "gauge",
1071 "Shows the timeout for direct connections in seconds.",
1072 "repo=\"" + fqrn + "\"", StringifyUint(timeout_direct));
1073
1074 // Last I/O error timestamp
1075 const int64_t timestamp_last_ioerr = file_system->io_error_info()
1076 ->timestamp_last();
1077 format_metric("cvmfs_sys_last_ioerr_timestamp_seconds", "gauge",
1078 "Shows the timestamp of the last ioerror.",
1079 "repo=\"" + fqrn + "\"", StringifyInt(timestamp_last_ioerr));
1080
1081 // CPU usage from /proc/pid/stat
1082 const pid_t pid = cvmfs::pid_;
1083 const string proc_stat_path = "/proc/" + StringifyInt(pid) + "/stat";
1084 FILE *stat_file = fopen(proc_stat_path.c_str(), "r");
1085 if (stat_file) {
1086 char stat_line[1024];
1087 if (fgets(stat_line, sizeof(stat_line), stat_file)) {
1088 vector<string> stat_fields = SplitString(string(stat_line), ' ');
1089 if (stat_fields.size() > 15) {
1090 const uint64_t utime = String2Uint64(stat_fields[13]);
1091 const uint64_t stime = String2Uint64(stat_fields[14]);
1092 const long clock_tick = sysconf(_SC_CLK_TCK);
1093 if (clock_tick > 0) {
1094 const double user_seconds = static_cast<double>(utime) / clock_tick;
1095 const double system_seconds = static_cast<double>(stime) / clock_tick;
1096 format_metric("cvmfs_sys_cpu_user_seconds_total", "counter",
1097 "CPU time used in userspace by CVMFS mount in seconds.",
1098 "repo=\"" + fqrn + "\"", StringifyDouble(user_seconds));
1099 format_metric("cvmfs_sys_cpu_system_seconds_total", "counter",
1100 "CPU time used in the kernel system calls by CVMFS "
1101 "mount in seconds.",
1102 "repo=\"" + fqrn + "\"",
1103 StringifyDouble(system_seconds));
1104 }
1105 }
1106 }
1107 fclose(stat_file);
1108 }
1109
1110 // File descriptor and directory counts
1111 format_metric("cvmfs_sys_usedfd", "gauge",
1112 "Shows the number of file descriptors currently issued to file "
1113 "system clients.",
1114 "repo=\"" + fqrn + "\"",
1115 file_system->no_open_files()->ToString());
1116 format_metric("cvmfs_sys_useddirp", "gauge",
1117 "Shows the number of open directories currently used by file "
1118 "system clients.",
1119 "repo=\"" + fqrn + "\"",
1120 file_system->no_open_dirs()->ToString());
1121 format_metric("cvmfs_sys_ndiropen", "gauge",
1122 "Shows the overall number of opened directories.",
1123 "repo=\"" + fqrn + "\"",
1124 file_system->n_fs_dir_open()->ToString());
1125
1126 // Inode max
1127 format_metric("cvmfs_sys_inode_max", "gauge",
1128 "Shows the highest possible inode with the current set of "
1129 "loaded catalogs.",
1130 "repo=\"" + fqrn + "\"",
1131 StringifyInt(mount_point.inode_annotation()->GetGeneration()
1132 + mount_point.catalog_mgr()->inode_gauge()));
1133
1134 // Process ID
1135 format_metric("cvmfs_sys_pid", "gauge",
1136 "Shows the process id of the CernVM-FS Fuse process.",
1137 "repo=\"" + fqrn + "\"", StringifyInt(pid));
1138
1139 // Catalog count
1140 const int n_catalogs = mount_point.catalog_mgr()->GetNumCatalogs();
1141 format_metric("cvmfs_repo_nclg", "gauge",
1142 "Shows the number of currently loaded nested catalogs.",
1143 "repo=\"" + fqrn + "\"", StringifyInt(n_catalogs));
1144
1145 // Cleanup rate (24 hours)
1146 if (quota_mgr->HasCapability(QuotaManager::kCapIntrospectCleanupRate)) {
1147 const uint64_t period_s = 24 * 60 * 60;
1148 const uint64_t cleanup_rate = quota_mgr->GetCleanupRate(period_s);
1149 format_metric("cvmfs_cache_ncleanup24", "gauge",
1150 "Shows the number of cache cleanups in the last 24 hours.",
1151 "repo=\"" + fqrn + "\"", StringifyUint(cleanup_rate));
1152 } else {
1153 format_metric("cvmfs_cache_ncleanup24", "gauge",
1154 "Shows the number of cache cleanups in the last 24 hours.",
1155 "repo=\"" + fqrn + "\"", "-1");
1156 }
1157
1158 // Active proxy
1159 vector<vector<download::DownloadManager::ProxyInfo> > proxy_chain;
1160 unsigned current_group;
1161 mount_point.download_mgr()->GetProxyInfo(&proxy_chain, &current_group, NULL);
1162 string active_proxy = "DIRECT";
1163 if (proxy_chain.size() > 0 && current_group < proxy_chain.size()
1164 && proxy_chain[current_group].size() > 0) {
1165 active_proxy = proxy_chain[current_group][0].url;
1166 }
1167 format_metric("cvmfs_net_active_proxy", "gauge",
1168 "Shows the active proxy in use for this mount.",
1169 "repo=\"" + fqrn + "\",proxy=\"" + active_proxy + "\"", "1");
1170
1171 // Proxy list metrics
1172 for (unsigned int i = 0; i < proxy_chain.size(); i++) {
1173 for (unsigned int j = 0; j < proxy_chain[i].size(); j++) {
1174 format_metric("cvmfs_net_proxy", "gauge",
1175 "Shows all registered proxies for this repository.",
1176 "repo=\"" + fqrn + "\",group=\"" + StringifyInt(i)
1177 + "\",url=\"" + proxy_chain[i][j].url + "\"",
1178 "1");
1179 }
1180 }
1181
1182 // Internal affairs metrics (excluding histograms)
1183
1184 // Update string counters manually (same as internal affairs does)
1185 mount_point.statistics()
1186 ->Lookup("pathstring.n_instances")
1187 ->Set(PathString::num_instances());
1188 mount_point.statistics()
1189 ->Lookup("pathstring.n_overflows")
1190 ->Set(PathString::num_overflows());
1191 mount_point.statistics()
1192 ->Lookup("namestring.n_instances")
1193 ->Set(NameString::num_instances());
1194 mount_point.statistics()
1195 ->Lookup("namestring.n_overflows")
1196 ->Set(NameString::num_overflows());
1197 mount_point.statistics()
1198 ->Lookup("linkstring.n_instances")
1199 ->Set(LinkString::num_instances());
1200 mount_point.statistics()
1201 ->Lookup("linkstring.n_overflows")
1202 ->Set(LinkString::num_overflows());
1203
1204 // String statistics
1205 const int64_t pathstring_instances = mount_point.statistics()
1206 ->Lookup("pathstring.n_instances")
1207 ->Get();
1208 const int64_t pathstring_overflows = mount_point.statistics()
1209 ->Lookup("pathstring.n_overflows")
1210 ->Get();
1211 const int64_t namestring_instances = mount_point.statistics()
1212 ->Lookup("namestring.n_instances")
1213 ->Get();
1214 const int64_t namestring_overflows = mount_point.statistics()
1215 ->Lookup("namestring.n_overflows")
1216 ->Get();
1217 const int64_t linkstring_instances = mount_point.statistics()
1218 ->Lookup("linkstring.n_instances")
1219 ->Get();
1220 const int64_t linkstring_overflows = mount_point.statistics()
1221 ->Lookup("linkstring.n_overflows")
1222 ->Get();
1223
1224 format_metric("cvmfs_internal_pathstring_instances", "gauge",
1225 "Number of PathString instances.", "repo=\"" + fqrn + "\"",
1226 StringifyInt(pathstring_instances));
1227 format_metric("cvmfs_internal_pathstring_overflows", "counter",
1228 "Number of PathString overflows.", "repo=\"" + fqrn + "\"",
1229 StringifyInt(pathstring_overflows));
1230 format_metric("cvmfs_internal_namestring_instances", "gauge",
1231 "Number of NameString instances.", "repo=\"" + fqrn + "\"",
1232 StringifyInt(namestring_instances));
1233 format_metric("cvmfs_internal_namestring_overflows", "counter",
1234 "Number of NameString overflows.", "repo=\"" + fqrn + "\"",
1235 StringifyInt(namestring_overflows));
1236 format_metric("cvmfs_internal_linkstring_instances", "gauge",
1237 "Number of LinkString instances.", "repo=\"" + fqrn + "\"",
1238 StringifyInt(linkstring_instances));
1239 format_metric("cvmfs_internal_linkstring_overflows", "counter",
1240 "Number of LinkString overflows.", "repo=\"" + fqrn + "\"",
1241 StringifyInt(linkstring_overflows));
1242
1243 // Tracker statistics (same as internal affairs does)
1244 glue::InodeTracker::Statistics inode_stats = mount_point.inode_tracker()
1245 ->GetStatistics();
1246 const glue::DentryTracker::Statistics
1247 dentry_stats = mount_point.dentry_tracker()->GetStatistics();
1248 const glue::PageCacheTracker::Statistics
1249 page_cache_stats = mount_point.page_cache_tracker()->GetStatistics();
1250
1251 // Update statistics manually
1252 mount_point.statistics()
1253 ->Lookup("inode_tracker.n_insert")
1254 ->Set(atomic_read64(&inode_stats.num_inserts));
1255 mount_point.statistics()
1256 ->Lookup("inode_tracker.n_remove")
1257 ->Set(atomic_read64(&inode_stats.num_removes));
1258 mount_point.statistics()
1259 ->Lookup("inode_tracker.no_reference")
1260 ->Set(atomic_read64(&inode_stats.num_references));
1261 mount_point.statistics()
1262 ->Lookup("inode_tracker.n_hit_inode")
1263 ->Set(atomic_read64(&inode_stats.num_hits_inode));
1264 mount_point.statistics()
1265 ->Lookup("inode_tracker.n_hit_path")
1266 ->Set(atomic_read64(&inode_stats.num_hits_path));
1267 mount_point.statistics()
1268 ->Lookup("inode_tracker.n_miss_path")
1269 ->Set(atomic_read64(&inode_stats.num_misses_path));
1270 mount_point.statistics()
1271 ->Lookup("dentry_tracker.n_insert")
1272 ->Set(dentry_stats.num_insert);
1273 mount_point.statistics()
1274 ->Lookup("dentry_tracker.n_remove")
1275 ->Set(dentry_stats.num_remove);
1276 mount_point.statistics()
1277 ->Lookup("dentry_tracker.n_prune")
1278 ->Set(dentry_stats.num_prune);
1279 mount_point.statistics()
1280 ->Lookup("page_cache_tracker.n_insert")
1281 ->Set(page_cache_stats.n_insert);
1282 mount_point.statistics()
1283 ->Lookup("page_cache_tracker.n_remove")
1284 ->Set(page_cache_stats.n_remove);
1285 mount_point.statistics()
1286 ->Lookup("page_cache_tracker.n_open_direct")
1287 ->Set(page_cache_stats.n_open_direct);
1288 mount_point.statistics()
1289 ->Lookup("page_cache_tracker.n_open_flush")
1290 ->Set(page_cache_stats.n_open_flush);
1291 mount_point.statistics()
1292 ->Lookup("page_cache_tracker.n_open_cached")
1293 ->Set(page_cache_stats.n_open_cached);
1294
1295 // Inode tracker metrics
1296 format_metric("cvmfs_internal_inode_tracker_inserts_total", "counter",
1297 "Number of inode tracker insertions.", "repo=\"" + fqrn + "\"",
1298 StringifyInt(atomic_read64(&inode_stats.num_inserts)));
1299 format_metric("cvmfs_internal_inode_tracker_removes_total", "counter",
1300 "Number of inode tracker removals.", "repo=\"" + fqrn + "\"",
1301 StringifyInt(atomic_read64(&inode_stats.num_removes)));
1302 format_metric("cvmfs_internal_inode_tracker_references", "gauge",
1303 "Number of inode tracker references.", "repo=\"" + fqrn + "\"",
1304 StringifyInt(atomic_read64(&inode_stats.num_references)));
1305 format_metric("cvmfs_internal_inode_tracker_hits_inode_total", "counter",
1306 "Number of inode tracker inode hits.", "repo=\"" + fqrn + "\"",
1307 StringifyInt(atomic_read64(&inode_stats.num_hits_inode)));
1308 format_metric("cvmfs_internal_inode_tracker_hits_path_total", "counter",
1309 "Number of inode tracker path hits.", "repo=\"" + fqrn + "\"",
1310 StringifyInt(atomic_read64(&inode_stats.num_hits_path)));
1311 format_metric("cvmfs_internal_inode_tracker_misses_path_total", "counter",
1312 "Number of inode tracker path misses.", "repo=\"" + fqrn + "\"",
1313 StringifyInt(atomic_read64(&inode_stats.num_misses_path)));
1314
1315 // Dentry tracker metrics
1316 format_metric("cvmfs_internal_dentry_tracker_inserts_total", "counter",
1317 "Number of dentry tracker insertions.", "repo=\"" + fqrn + "\"",
1318 StringifyInt(dentry_stats.num_insert));
1319 format_metric("cvmfs_internal_dentry_tracker_removes_total", "counter",
1320 "Number of dentry tracker removals.", "repo=\"" + fqrn + "\"",
1321 StringifyInt(dentry_stats.num_remove));
1322 format_metric("cvmfs_internal_dentry_tracker_prunes_total", "counter",
1323 "Number of dentry tracker prunes.", "repo=\"" + fqrn + "\"",
1324 StringifyInt(dentry_stats.num_prune));
1325
1326 // Page cache tracker metrics
1327 format_metric("cvmfs_internal_page_cache_tracker_inserts_total", "counter",
1328 "Number of page cache tracker insertions.",
1329 "repo=\"" + fqrn + "\"",
1330 StringifyInt(page_cache_stats.n_insert));
1331 format_metric("cvmfs_internal_page_cache_tracker_removes_total", "counter",
1332 "Number of page cache tracker removals.",
1333 "repo=\"" + fqrn + "\"",
1334 StringifyInt(page_cache_stats.n_remove));
1335 format_metric("cvmfs_internal_page_cache_tracker_opens_direct_total",
1336 "counter", "Number of page cache tracker direct opens.",
1337 "repo=\"" + fqrn + "\"",
1338 StringifyInt(page_cache_stats.n_open_direct));
1339 format_metric("cvmfs_internal_page_cache_tracker_opens_flush_total",
1340 "counter", "Number of page cache tracker flush opens.",
1341 "repo=\"" + fqrn + "\"",
1342 StringifyInt(page_cache_stats.n_open_flush));
1343 format_metric("cvmfs_internal_page_cache_tracker_opens_cached_total",
1344 "counter", "Number of page cache tracker cached opens.",
1345 "repo=\"" + fqrn + "\"",
1346 StringifyInt(page_cache_stats.n_open_cached));
1347
1348 // Cache mode information
1349 if (file_system->cache_mgr()->id() == kPosixCacheManager) {
1350 PosixCacheManager *cache_mgr = reinterpret_cast<PosixCacheManager *>(
1351 file_system->cache_mgr());
1352 int cache_mode_value = 0;
1353 switch (cache_mgr->cache_mode()) {
1354 case PosixCacheManager::kCacheReadWrite:
1355 cache_mode_value = 1;
1356 break;
1357 case PosixCacheManager::kCacheReadOnly:
1358 cache_mode_value = 2;
1359 break;
1360 default:
1361 cache_mode_value = 0;
1362 }
1363 format_metric("cvmfs_cache_mode", "gauge",
1364 "Cache mode (0=unknown, 1=read-write, 2=read-only).",
1365 "repo=\"" + fqrn + "\"", StringifyInt(cache_mode_value));
1366 }
1367
1368 // Drainout and maintenance mode
1369 bool drainout_mode;
1370 bool maintenance_mode;
1371 cvmfs::GetReloadStatus(&drainout_mode, &maintenance_mode);
1372 format_metric("cvmfs_sys_drainout_mode", "gauge",
1373 "Drainout mode status (0=false, 1=true).",
1374 "repo=\"" + fqrn + "\"", StringifyInt(drainout_mode ? 1 : 0));
1375 format_metric("cvmfs_sys_maintenance_mode", "gauge",
1376 "Maintenance mode status (0=false, 1=true).",
1377 "repo=\"" + fqrn + "\"",
1378 StringifyInt(maintenance_mode ? 1 : 0));
1379
1380 // SQLite statistics
1381 int current, highwater;
1382
1383 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &current, &highwater, 0);
1384 format_metric("cvmfs_internal_sqlite_malloc_count", "gauge",
1385 "Number of SQLite allocations.", "repo=\"" + fqrn + "\"",
1386 StringifyInt(current));
1387
1388 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &current, &highwater, 0);
1389 format_metric("cvmfs_internal_sqlite_memory_used_bytes", "gauge",
1390 "SQLite general purpose allocator memory used.",
1391 "repo=\"" + fqrn + "\"", StringifyInt(current));
1392 format_metric("cvmfs_internal_sqlite_memory_used_highwater_bytes", "gauge",
1393 "SQLite general purpose allocator memory used high water mark.",
1394 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1395
1396 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &current, &highwater, 0);
1397 format_metric("cvmfs_internal_sqlite_largest_malloc_bytes", "gauge",
1398 "SQLite largest malloc size.", "repo=\"" + fqrn + "\"",
1399 StringifyInt(highwater));
1400
1401 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &current, &highwater, 0);
1402 format_metric("cvmfs_internal_sqlite_pagecache_used", "gauge",
1403 "SQLite page cache allocations used.", "repo=\"" + fqrn + "\"",
1404 StringifyInt(current));
1405 format_metric("cvmfs_internal_sqlite_pagecache_used_highwater", "gauge",
1406 "SQLite page cache allocations used high water mark.",
1407 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1408
1409 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &current, &highwater, 0);
1410 format_metric("cvmfs_internal_sqlite_pagecache_overflow_bytes", "gauge",
1411 "SQLite page cache overflow bytes.", "repo=\"" + fqrn + "\"",
1412 StringifyInt(current));
1413 format_metric("cvmfs_internal_sqlite_pagecache_overflow_highwater_bytes",
1414 "gauge", "SQLite page cache overflow bytes high water mark.",
1415 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1416
1417 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &current, &highwater, 0);
1418 format_metric("cvmfs_internal_sqlite_largest_pagecache_bytes", "gauge",
1419 "SQLite largest page cache allocation size.",
1420 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1421
1422 sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &current, &highwater, 0);
1423 format_metric("cvmfs_internal_sqlite_scratch_used", "gauge",
1424 "SQLite scratch allocations used.", "repo=\"" + fqrn + "\"",
1425 StringifyInt(current));
1426 format_metric("cvmfs_internal_sqlite_scratch_used_highwater", "gauge",
1427 "SQLite scratch allocations used high water mark.",
1428 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1429
1430 sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &current, &highwater, 0);
1431 format_metric("cvmfs_internal_sqlite_scratch_overflow", "gauge",
1432 "SQLite scratch overflows.", "repo=\"" + fqrn + "\"",
1433 StringifyInt(current));
1434 format_metric("cvmfs_internal_sqlite_scratch_overflow_highwater", "gauge",
1435 "SQLite scratch overflows high water mark.",
1436 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1437
1438 sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &current, &highwater, 0);
1439 format_metric("cvmfs_internal_sqlite_largest_scratch_bytes", "gauge",
1440 "SQLite largest scratch allocation size.",
1441 "repo=\"" + fqrn + "\"", StringifyInt(highwater));
1442
1443 // NFS statistics (if applicable)
1444 if (file_system->IsNfsSource()) {
1445 format_metric("cvmfs_sys_nfs_mode", "gauge",
1446 "NFS mode enabled (1=true, 0=false).",
1447 "repo=\"" + fqrn + "\"", "1");
1448 // Note: NFS map statistics are complex strings, skipping detailed parsing
1449 // for now
1450 } else {
1451 format_metric("cvmfs_sys_nfs_mode", "gauge",
1452 "NFS mode enabled (1=true, 0=false).",
1453 "repo=\"" + fqrn + "\"", "0");
1454 }
1455
1456 return result;
1457 }
1458
1459 TalkManager::TalkManager(const string &socket_path,
1460 MountPoint *mount_point,
1461 FuseRemounter *remounter)
1462 : socket_path_(socket_path)
1463 , socket_fd_(-1)
1464 , mount_point_(mount_point)
1465 , remounter_(remounter)
1466 , spawned_(false) {
1467 memset(&thread_talk_, 0, sizeof(thread_talk_));
1468 }
1469
1470
1471 TalkManager::~TalkManager() {
1472 if (!socket_path_.empty()) {
1473 const int retval = unlink(socket_path_.c_str());
1474 if ((retval != 0) && (errno != ENOENT)) {
1475 LogCvmfs(kLogTalk, kLogSyslogWarn,
1476 "Could not remove cvmfs_io socket from cache directory (%d)",
1477 errno);
1478 }
1479 }
1480
1481 if (socket_fd_ >= 0) {
1482 shutdown(socket_fd_, SHUT_RDWR);
1483 close(socket_fd_);
1484 }
1485
1486 if (spawned_) {
1487 pthread_join(thread_talk_, NULL);
1488 LogCvmfs(kLogTalk, kLogDebug, "talk thread stopped");
1489 }
1490 }
1491
1492
1493 void TalkManager::Spawn() {
1494 const int retval = pthread_create(&thread_talk_, NULL, MainResponder, this);
1495 assert(retval == 0);
1496 spawned_ = true;
1497 }
1498