CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_balancer_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_CATALOG_BALANCER_IMPL_H_
6 #define CVMFS_CATALOG_BALANCER_IMPL_H_
7 
8 
9 #include <inttypes.h>
10 
11 #include <cassert>
12 #include <cstdlib>
13 #include <string>
14 #include <vector>
15 
16 #include "catalog_mgr.h"
18 #include "crypto/hash.h"
19 #include "directory_entry.h"
20 #include "util/logging.h"
21 
22 
23 using namespace std; // NOLINT
24 
25 namespace catalog {
26 
27 template <class CatalogMgrT>
28 DirectoryEntryBase
30  string name,
31  uid_t uid,
32  gid_t gid)
33 {
34  // Note that another entity needs to ensure that the object of an empty
35  // file is in the repository! It is currently done by the sync_mediator.
36  shash::Algorithms algorithm = catalog_mgr_->spooler_->GetHashAlgorithm();
37  shash::Any file_hash(algorithm);
38  void *empty_compressed;
39  uint64_t sz_empty_compressed;
40  bool retval = zlib::CompressMem2Mem(
41  NULL, 0, &empty_compressed, &sz_empty_compressed);
42  assert(retval);
43  shash::HashMem(static_cast<unsigned char *>(empty_compressed),
44  sz_empty_compressed, &file_hash);
45  free(empty_compressed);
46 
48  deb.name_ = NameString(name);
49  deb.mode_ = S_IFREG | S_IRUSR | S_IWUSR;
50  deb.checksum_ = file_hash;
51  deb.mtime_ = time(NULL);
52  deb.uid_ = uid;
53  deb.gid_ = gid;
54  return deb;
55 }
56 
57 template <class CatalogMgrT>
59  XattrList xattr;
60  DirectoryEntry parent;
61  bool retval;
62  retval = catalog_mgr_->LookupPath(PathString(path), kLookupDefault, &parent);
63  assert(retval);
64  DirectoryEntryBase cvmfscatalog =
65  MakeEmptyDirectoryEntryBase(".cvmfscatalog", parent.uid(),
66  parent.gid());
67  DirectoryEntryBase cvmfsautocatalog =
68  MakeEmptyDirectoryEntryBase(".cvmfsautocatalog", parent.uid(),
69  parent.gid());
70  string relative_path = path.substr(1);
71  catalog_mgr_->AddFile(cvmfscatalog, xattr, relative_path);
72  catalog_mgr_->AddFile(cvmfsautocatalog, xattr, relative_path);
73 }
74 
75 template <class CatalogMgrT>
77  if (catalog == NULL) {
78  // obtain a copy of the catalogs
79  vector<catalog_t*> catalogs = catalog_mgr_->GetCatalogs();
80  // we need to reverse the catalog list in order to analyze the
81  // last added ones first. This is necessary in the weird case the child
82  // catalog's underflow provokes an overflow in the father
83  reverse(catalogs.begin(), catalogs.end());
84  for (unsigned i = 0; i < catalogs.size(); ++i)
85  Balance(catalogs[i]);
86  return;
87  }
88  string catalog_path = catalog->mountpoint().ToString();
89  virtual_node_t root_node(catalog_path, catalog_mgr_);
90  root_node.ExtractChildren(catalog_mgr_);
91  // we have just recursively loaded the entire virtual tree!
92  PartitionOptimally(&root_node);
93 }
94 
95 template <class CatalogMgrT>
97  virtual_node_t *virtual_node) {
98  // postorder track of the fs-tree
99  for (unsigned i = 0; i < virtual_node->children.size(); ++i) {
100  virtual_node_t *virtual_child =
101  &virtual_node->children[i];
102  if (virtual_child->IsDirectory() && !virtual_child->IsCatalog())
103  PartitionOptimally(virtual_child);
104  }
105  virtual_node->FixWeight();
106  while (virtual_node->weight > catalog_mgr_->balance_weight_) {
107  virtual_node_t *heaviest_node =
108  MaxChild(virtual_node);
109  // we directly add a catalog in this node
110  // TODO(molina) apply heuristics here
111  if (heaviest_node != NULL &&
112  heaviest_node->weight >= catalog_mgr_->min_weight_) {
113  // the catalog now generated _cannot_ be overflowed because the tree is
114  // being traversed in postorder, handling the lightest nodes first
115  unsigned max_weight = heaviest_node->weight;
116  AddCatalogMarker(heaviest_node->path);
117  AddCatalog(heaviest_node);
118  virtual_node->weight -= (max_weight - 1);
119  } else {
120  // there is no possibility for any this directory's children
121  // to be a catalog
122  LogCvmfs(kLogPublish, kLogStdout, "Couldn't create a new nested catalog"
123  " in any subdirectory of '%s' even though"
124  " currently it is overflowed", virtual_node->path.c_str());
125  break;
126  }
127  }
128 }
129 
130 template <class CatalogMgrT>
133  virtual_node_t *virtual_node)
134 {
135  virtual_node_t *max_child = NULL;
136  unsigned max_weight = 0;
137  if (virtual_node->IsDirectory() &&
138  !virtual_node->IsCatalog() &&
139  !virtual_node->is_new_nested_catalog) {
140  for (unsigned i = 0; i < virtual_node->children.size(); ++i) {
141  virtual_node_t *child = &virtual_node->children[i];
142  if (child->IsDirectory() &&
143  !child->IsCatalog() &&
144  max_weight < child->weight) {
145  max_weight = child->weight;
146  max_child = child;
147  }
148  }
149  }
150  return max_child;
151 }
152 
153 template <class CatalogMgrT>
155  assert(child_node != NULL);
156  string new_catalog_path = child_node->path.substr(1);
157  catalog_mgr_->CreateNestedCatalog(new_catalog_path);
158  child_node->weight = 1;
159  child_node->is_new_nested_catalog = true;
160  LogCvmfs(kLogPublish, kLogStdout, "Automatic creation of nested"
161  " catalog in '%s'", child_node->path.c_str());
162 }
163 
164 template <class CatalogMgrT>
166  CatalogMgrT *catalog_mgr) {
167  DirectoryEntryList direntlist;
168  catalog_mgr->Listing(path, &direntlist);
169  for (unsigned i = 0; i < direntlist.size(); ++i) {
170  string child_path = path + "/" + direntlist[i].name().ToString();
171  children.push_back(virtual_node_t(
172  child_path, direntlist[i], catalog_mgr));
173  weight += children[i].weight;
174  }
175 }
176 
190 template <class CatalogMgrT>
192  weight = 1;
193  if (!IsCatalog() && IsDirectory()) {
194  for (unsigned i = 0; i < children.size(); ++i) {
195  weight += children[i].weight;
196  }
197  }
198 }
199 
200 } // namespace catalog
201 
202 #endif // CVMFS_CATALOG_BALANCER_IMPL_H_
ShortString< kDefaultMaxName, 1 > NameString
Definition: shortstring.h:218
const std::vector< CatalogT * > & GetCatalogs() const
Definition: catalog_mgr.h:343
gid_t gid_
assert((mem||(size==0))&&"Out Of Memory")
bool LookupPath(const PathString &path, const LookupOptions options, DirectoryEntry *entry)
CatalogMgrT::catalog_t catalog_t
const unsigned kLookupDefault
Definition: catalog_mgr.h:43
catalog::WritableCatalogManager * catalog_mgr_
Definition: repository.h:385
char algorithm
Algorithms
Definition: hash.h:41
NameString name_
std::vector< DirectoryEntry > DirectoryEntryList
time_t mtime_
void HashMem(const unsigned char *buffer, const unsigned buffer_size, Any *any_digest)
Definition: hash.cc:255
unsigned int mode_
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:217
uid_t uid() const
bool CompressMem2Mem(const void *buf, const int64_t size, void **out_buf, uint64_t *out_size)
Definition: compression.cc:739
gid_t gid() const
void ExtractChildren(CatalogMgrT *catalog_mgr)
shash::Any checksum_
std::vector< VirtualNode > children
void AddFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
void CreateNestedCatalog(const std::string &mountpoint)
uid_t uid_
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528