CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_merge_tool_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_RECEIVER_CATALOG_MERGE_TOOL_IMPL_H_
6 #define CVMFS_RECEIVER_CATALOG_MERGE_TOOL_IMPL_H_
7 
8 #include <string>
9 
10 #include "catalog.h"
11 #include "hash.h"
12 #include "lease_path_util.h"
13 #include "logging.h"
14 #include "manifest.h"
15 #include "options.h"
16 #include "upload.h"
17 #include "util/exception.h"
18 #include "util/posix.h"
19 #include "util/raii_temp_dir.h"
20 
21 inline PathString MakeRelative(const PathString& path) {
22  std::string rel_path;
23  std::string abs_path = path.ToString();
24  if (abs_path[0] == '/') {
25  rel_path = abs_path.substr(1);
26  } else {
27  rel_path = abs_path;
28  }
29  return PathString(rel_path);
30 }
31 
33  if (entry->linkcount() > 1) {
35  "CatalogMergeTool - Hardlink found: %s. Hardlinks are not "
36  "supported when publishing through repository gateway and "
37  "will be split.", entry->name().c_str());
38  entry->set_linkcount(1);
39  }
40 }
41 
42 inline void AbortIfHardlinked(const catalog::DirectoryEntry& entry) {
43  if (entry.linkcount() > 1) {
45  "CatalogMergeTool - Removal of file %s with linkcount > 1 is "
46  "not supported. Aborting",
47  entry.name().c_str());
48  }
49 }
50 
51 namespace receiver {
52 
53 template <typename RwCatalogMgr, typename RoCatalogMgr>
55  const Params& params, std::string* new_manifest_path, uint64_t *final_rev) {
57  perf::StatisticsTemplate stats_tmpl("publish", statistics_);
58  counters_ = new perf::FsCounters(stats_tmpl);
59 
60  UniquePtr<RaiiTempDir> raii_temp_dir(RaiiTempDir::Create(temp_dir_prefix_));
61  if (needs_setup_) {
62  upload::SpoolerDefinition definition(
63  params.spooler_configuration, params.hash_alg, params.compression_alg,
65  params.min_chunk_size, params.avg_chunk_size, params.max_chunk_size,
66  "dummy_token", "dummy_key");
67  spooler = upload::Spooler::Construct(definition, &stats_tmpl);
68  const std::string temp_dir = raii_temp_dir->dir();
69  output_catalog_mgr_ = new RwCatalogMgr(
70  manifest_->catalog_hash(), repo_path_, temp_dir, spooler,
71  download_manager_, params.enforce_limits, params.nested_kcatalog_limit,
73  params.use_autocatalogs, params.max_weight, params.min_weight);
74  output_catalog_mgr_->Init();
75  }
76 
78 
79  ret &= CreateNewManifest(new_manifest_path);
80 
81  *final_rev = manifest_->revision();
82 
83  output_catalog_mgr_.Destroy();
84 
85  return ret;
86 }
87 
88 template <typename RwCatalogMgr, typename RoCatalogMgr>
90  const PathString& path) {
91  const PathString rel_path = MakeRelative(path);
92 
93  // Ignore any paths that are not either within the lease path or
94  // above the lease path
95  return !(IsSubPath(lease_path_, rel_path) ||
96  IsSubPath(rel_path, lease_path_));
97 }
98 
99 template <typename RwCatalogMgr, typename RoCatalogMgr>
101  const PathString& path) {
102  const PathString rel_path = MakeRelative(path);
103 
104  // Do not report any changes occurring outside the lease path (which
105  // will be due to other concurrent writers)
106  return IsSubPath(lease_path_, rel_path);
107 }
108 
109 template <typename RwCatalogMgr, typename RoCatalogMgr>
111  const PathString& path, const catalog::DirectoryEntry& entry,
112  const XattrList& xattrs, const FileChunkList& chunks) {
113  const PathString rel_path = MakeRelative(path);
114 
115  const std::string parent_path =
116  std::strchr(rel_path.c_str(), '/') ? GetParentPath(rel_path).c_str() : "";
117 
118  if (entry.IsDirectory()) {
119  output_catalog_mgr_->AddDirectory(entry, xattrs, parent_path);
120  if (entry.IsNestedCatalogMountpoint()) {
121  output_catalog_mgr_->CreateNestedCatalog(std::string(rel_path.c_str()));
122  }
123  perf::Inc(counters_->n_directories_added);
124  } else if (entry.IsRegular() || entry.IsLink()) {
125  catalog::DirectoryEntry modified_entry = entry;
126  SplitHardlink(&modified_entry);
127  const catalog::DirectoryEntryBase* base_entry =
128  static_cast<const catalog::DirectoryEntryBase*>(&modified_entry);
129  if (entry.IsChunkedFile()) {
130  assert(!chunks.IsEmpty());
131  output_catalog_mgr_->AddChunkedFile(*base_entry, xattrs, parent_path,
132  chunks);
133  } else {
134  output_catalog_mgr_->AddFile(*base_entry, xattrs, parent_path);
135  }
136  if (entry.IsLink())
137  perf::Inc(counters_->n_symlinks_added);
138  else
139  perf::Inc(counters_->n_files_added);
140  perf::Xadd(counters_->sz_added_bytes, entry.size());
141  }
142 }
143 
144 template <typename RwCatalogMgr, typename RoCatalogMgr>
146  const PathString& path, const catalog::DirectoryEntry& entry) {
147  const PathString rel_path = MakeRelative(path);
148 
149  if (entry.IsDirectory()) {
150  if (entry.IsNestedCatalogMountpoint()) {
151  output_catalog_mgr_->RemoveNestedCatalog(std::string(rel_path.c_str()),
152  false);
153  }
154 
155  output_catalog_mgr_->RemoveDirectory(rel_path.c_str());
156  perf::Inc(counters_->n_directories_removed);
157  } else if (entry.IsRegular() || entry.IsLink()) {
158  AbortIfHardlinked(entry);
159  output_catalog_mgr_->RemoveFile(rel_path.c_str());
160 
161  if (entry.IsLink())
162  perf::Inc(counters_->n_symlinks_removed);
163  else
164  perf::Inc(counters_->n_files_removed);
165 
166  perf::Xadd(counters_->sz_removed_bytes, entry.size());
167  }
168 }
169 
170 template <typename RwCatalogMgr, typename RoCatalogMgr>
172  const PathString& path, const catalog::DirectoryEntry& entry1,
173  const catalog::DirectoryEntry& entry2, const XattrList& xattrs,
174  const FileChunkList& chunks) {
175  const PathString rel_path = MakeRelative(path);
176 
177  const std::string parent_path =
178  std::strchr(rel_path.c_str(), '/') ? GetParentPath(rel_path).c_str() : "";
179 
180  if (entry1.IsNestedCatalogMountpoint() &&
181  entry2.IsNestedCatalogMountpoint()) {
182  // From nested catalog to nested catalog
183  RoCatalogMgr *new_catalog_mgr =
185  PathString mountpoint;
186  shash::Any new_hash;
187  uint64_t new_size;
188  const bool found = new_catalog_mgr->LookupNested(path, &mountpoint,
189  &new_hash, &new_size);
190  if (!found || !new_size) {
192  "CatalogMergeTool - nested catalog %s not found. Aborting",
193  rel_path.c_str());
194  }
195  output_catalog_mgr_->SwapNestedCatalog(rel_path.ToString(), new_hash,
196  new_size);
197  return false; // skip recursion into nested catalog mountpoints
198  } else if (entry1.IsDirectory() && entry2.IsDirectory()) {
199  // From directory to directory
200  const catalog::DirectoryEntryBase* base_entry =
201  static_cast<const catalog::DirectoryEntryBase*>(&entry2);
202  output_catalog_mgr_->TouchDirectory(*base_entry, xattrs, rel_path.c_str());
203  if (!entry1.IsNestedCatalogMountpoint() &&
204  entry2.IsNestedCatalogMountpoint()) {
205  output_catalog_mgr_->CreateNestedCatalog(std::string(rel_path.c_str()));
206  } else if (entry1.IsNestedCatalogMountpoint() &&
207  !entry2.IsNestedCatalogMountpoint()) {
208  output_catalog_mgr_->RemoveNestedCatalog(std::string(rel_path.c_str()));
209  }
210  perf::Inc(counters_->n_directories_changed);
211  } else if ((entry1.IsRegular() || entry1.IsLink()) && entry2.IsDirectory()) {
212  // From file to directory
213  AbortIfHardlinked(entry1);
214  output_catalog_mgr_->RemoveFile(rel_path.c_str());
215  output_catalog_mgr_->AddDirectory(entry2, xattrs, parent_path);
216  if (entry2.IsNestedCatalogMountpoint()) {
217  output_catalog_mgr_->CreateNestedCatalog(std::string(rel_path.c_str()));
218  }
219  if (entry1.IsLink())
220  perf::Inc(counters_->n_symlinks_removed);
221  else
222  perf::Inc(counters_->n_files_removed);
223  perf::Xadd(counters_->sz_removed_bytes, entry1.size());
224  perf::Inc(counters_->n_directories_added);
225 
226  } else if (entry1.IsDirectory() && (entry2.IsRegular() || entry2.IsLink())) {
227  // From directory to file
228  if (entry1.IsNestedCatalogMountpoint()) {
229  // we merge the nested catalog with its parent, it will be the recursive
230  // procedure that will take care of deleting all the files.
231  output_catalog_mgr_->RemoveNestedCatalog(std::string(rel_path.c_str()),
232  /* merge = */ true);
233  }
234 
235  catalog::DirectoryEntry modified_entry = entry2;
236  SplitHardlink(&modified_entry);
237  const catalog::DirectoryEntryBase* base_entry =
238  static_cast<const catalog::DirectoryEntryBase*>(&modified_entry);
239 
240  output_catalog_mgr_->RemoveDirectory(rel_path.c_str());
241 
242  if (entry2.IsChunkedFile()) {
243  assert(!chunks.IsEmpty());
244  output_catalog_mgr_->AddChunkedFile(*base_entry, xattrs, parent_path,
245  chunks);
246  } else {
247  output_catalog_mgr_->AddFile(*base_entry, xattrs, parent_path);
248  }
249 
250  perf::Inc(counters_->n_directories_removed);
251  if (entry2.IsLink())
252  perf::Inc(counters_->n_symlinks_added);
253  else
254  perf::Inc(counters_->n_files_added);
255  perf::Xadd(counters_->sz_added_bytes, entry2.size());
256 
257  } else if ((entry1.IsRegular() || entry1.IsLink()) &&
258  (entry2.IsRegular() || entry2.IsLink())) {
259  // From file to file
260  AbortIfHardlinked(entry1);
261  catalog::DirectoryEntry modified_entry = entry2;
262  SplitHardlink(&modified_entry);
263  const catalog::DirectoryEntryBase* base_entry =
264  static_cast<const catalog::DirectoryEntryBase*>(&modified_entry);
265  output_catalog_mgr_->RemoveFile(rel_path.c_str());
266  if (entry2.IsChunkedFile()) {
267  assert(!chunks.IsEmpty());
268  output_catalog_mgr_->AddChunkedFile(*base_entry, xattrs, parent_path,
269  chunks);
270  } else {
271  output_catalog_mgr_->AddFile(*base_entry, xattrs, parent_path);
272  }
273 
274  if (entry1.IsRegular() && entry2.IsRegular()) {
275  perf::Inc(counters_->n_files_changed);
276  } else if (entry1.IsRegular() && entry2.IsLink()) {
277  perf::Inc(counters_->n_files_removed);
278  perf::Inc(counters_->n_symlinks_added);
279  } else if (entry1.IsLink() && entry2.IsRegular()) {
280  perf::Inc(counters_->n_symlinks_removed);
281  perf::Inc(counters_->n_files_added);
282  } else {
283  perf::Inc(counters_->n_symlinks_changed);
284  }
285  perf::Xadd(counters_->sz_removed_bytes, entry1.size());
286  perf::Xadd(counters_->sz_added_bytes, entry2.size());
287  }
288  return true;
289 }
290 
291 template <typename RwCatalogMgr, typename RoCatalogMgr>
293  std::string* new_manifest_path) {
294  if (!output_catalog_mgr_->Commit(false, 0, manifest_)) {
296  "CatalogMergeTool - Could not commit output catalog");
297  return false;
298  }
299 
300  const std::string new_path = CreateTempPath(temp_dir_prefix_, 0600);
301 
302  if (!manifest_->Export(new_path)) {
304  "CatalogMergeTool - Could not export new manifest");
305  }
306 
307  *new_manifest_path = new_path;
308 
309  return true;
310 }
311 
312 } // namespace receiver
313 
314 #endif // CVMFS_RECEIVER_CATALOG_MERGE_TOOL_IMPL_H_
uint32_t linkcount() const
bool IsSubPath(const PathString &parent, const PathString &path)
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool Export(const std::string &path) const
Definition: manifest.cc:209
int64_t Xadd(class Counter *counter, const int64_t delta)
Definition: statistics.h:51
bool IsDirectory() const
size_t min_weight
Definition: params.h:35
size_t avg_chunk_size
Definition: params.h:26
std::string spooler_configuration
Definition: params.h:19
bool IsChunkedFile() const
#define PANIC(...)
Definition: exception.h:26
uint64_t size() const
void set_linkcount(const uint32_t linkcount)
zlib::Algorithms compression_alg
Definition: params.h:22
size_t nested_kcatalog_limit
Definition: params.h:29
bool use_autocatalogs
Definition: params.h:33
perf::Statistics * statistics_
Definition: repository.h:138
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1054
virtual bool ReportModification(const PathString &path, const catalog::DirectoryEntry &old_entry, const catalog::DirectoryEntry &new_entry, const XattrList &xattrs, const FileChunkList &chunks)
assert((mem||(size==0))&&"Out Of Memory")
std::string GetParentPath(const std::string &path)
Definition: posix.cc:129
void AbortIfHardlinked(const catalog::DirectoryEntry &entry)
bool IsNestedCatalogMountpoint() const
uint64_t revision() const
Definition: manifest.h:121
PathString MakeRelative(const PathString &path)
bool use_file_chunking
Definition: params.h:24
NameString name() const
bool IsLink() const
size_t max_weight
Definition: params.h:34
size_t max_chunk_size
Definition: params.h:27
bool IsRegular() const
virtual bool IsReportablePath(const PathString &path)
RoCatalogMgr * GetNewCatalogMgr()
bool Run(const Params &params, std::string *new_manifest_path, uint64_t *final_rev)
virtual bool IsIgnoredPath(const PathString &path)
static RaiiTempDir * Create(const std::string &prefix)
Definition: raii_temp_dir.cc:9
shash::Any catalog_hash() const
Definition: manifest.h:124
void Inc(class Counter *counter)
Definition: statistics.h:50
bool IsEmpty() const
Definition: bigvector.h:67
manifest::Manifest * manifest_
Definition: repository.h:147
std::string ToString() const
Definition: shortstring.h:113
size_t min_chunk_size
Definition: params.h:25
void SplitHardlink(catalog::DirectoryEntry *entry)
size_t file_mbyte_limit
Definition: params.h:31
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:189
bool CreateNewManifest(std::string *new_manifest_path)
size_t root_kcatalog_limit
Definition: params.h:30
bool Run(const PathString &path)
virtual void ReportAddition(const PathString &path, const catalog::DirectoryEntry &entry, const XattrList &xattrs, const FileChunkList &chunks)
bool generate_legacy_bulk_chunks
Definition: params.h:23
const char * c_str() const
Definition: shortstring.h:117
bool enforce_limits
Definition: params.h:28
virtual void ReportRemoval(const PathString &path, const catalog::DirectoryEntry &entry)
shash::Algorithms hash_alg
Definition: params.h:20