CernVM-FS  2.12.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_merge_tool_impl.h
Go to the documentation of this file.
1 
5 #ifndef CVMFS_RECEIVER_CATALOG_MERGE_TOOL_IMPL_H_
6 #define CVMFS_RECEIVER_CATALOG_MERGE_TOOL_IMPL_H_
7 
8 #include <string>
9 
10 #include "catalog.h"
11 #include "crypto/hash.h"
12 #include "lease_path_util.h"
13 #include "manifest.h"
14 #include "options.h"
15 #include "upload.h"
16 #include "util/exception.h"
17 #include "util/logging.h"
18 #include "util/posix.h"
19 #include "util/raii_temp_dir.h"
20 
21 inline PathString MakeRelative(const PathString& path) {
22  std::string rel_path;
23  std::string abs_path = path.ToString();
24  if (abs_path[0] == '/') {
25  rel_path = abs_path.substr(1);
26  } else {
27  rel_path = abs_path;
28  }
29  return PathString(rel_path);
30 }
31 
33  if (entry->linkcount() > 1) {
35  "CatalogMergeTool - Hardlink found: %s. Hardlinks are not "
36  "supported when publishing through repository gateway and "
37  "will be split.", entry->name().c_str());
38  entry->set_linkcount(1);
39  }
40 }
41 
42 inline void AbortIfHardlinked(const catalog::DirectoryEntry& entry) {
43  if (entry.linkcount() > 1) {
45  "CatalogMergeTool - Removal of file %s with linkcount > 1 is "
46  "not supported. Aborting",
47  entry.name().c_str());
48  }
49 }
50 
51 namespace receiver {
52 
53 template <typename RwCatalogMgr, typename RoCatalogMgr>
55  const Params& params, std::string* new_manifest_path, uint64_t *final_rev) {
57  perf::StatisticsTemplate stats_tmpl("publish", statistics_);
58  counters_ = new perf::FsCounters(stats_tmpl);
59 
60  UniquePtr<RaiiTempDir> raii_temp_dir(RaiiTempDir::Create(temp_dir_prefix_));
61  if (needs_setup_) {
62  upload::SpoolerDefinition definition(
63  params.spooler_configuration, params.hash_alg, params.compression_alg,
65  params.min_chunk_size, params.avg_chunk_size, params.max_chunk_size,
66  "dummy_token", "dummy_key");
67  spooler = upload::Spooler::Construct(definition, &stats_tmpl);
68  const std::string temp_dir = raii_temp_dir->dir();
69  output_catalog_mgr_ = new RwCatalogMgr(
70  manifest_->catalog_hash(), repo_path_, temp_dir, spooler.weak_ref(),
71  download_manager_, params.enforce_limits, params.nested_kcatalog_limit,
73  params.use_autocatalogs, params.max_weight, params.min_weight);
74  output_catalog_mgr_->Init();
75  }
76 
78 
79  ret &= CreateNewManifest(new_manifest_path);
80 
81  *final_rev = manifest_->revision();
82 
83  output_catalog_mgr_.Destroy();
84 
85  return ret;
86 }
87 
88 template <typename RwCatalogMgr, typename RoCatalogMgr>
90  const PathString& path) {
91  const PathString rel_path = MakeRelative(path);
92 
93  // Ignore any paths that are not either within the lease path or
94  // above the lease path
95  return !(IsSubPath(lease_path_, rel_path) ||
96  IsSubPath(rel_path, lease_path_));
97 }
98 
99 template <typename RwCatalogMgr, typename RoCatalogMgr>
101  const PathString& path) {
102  const PathString rel_path = MakeRelative(path);
103 
104  // Do not report any changes occurring outside the lease path (which
105  // will be due to other concurrent writers)
106  return IsSubPath(lease_path_, rel_path);
107 }
108 
109 template <typename RwCatalogMgr, typename RoCatalogMgr>
111  const PathString& path, const catalog::DirectoryEntry& entry,
112  const XattrList& xattrs, const FileChunkList& chunks) {
113  const PathString rel_path = MakeRelative(path);
114 
115  const std::string parent_path =
116  std::strchr(rel_path.c_str(), '/') ? GetParentPath(rel_path).c_str() : "";
117 
118  if (entry.IsDirectory()) {
119  output_catalog_mgr_->AddDirectory(entry, xattrs, parent_path);
120  if (entry.IsNestedCatalogMountpoint()) {
121  output_catalog_mgr_->CreateNestedCatalog(std::string(rel_path.c_str()));
122  }
123  perf::Inc(counters_->n_directories_added);
124  } else if (entry.IsRegular() || entry.IsLink()) {
125  catalog::DirectoryEntry modified_entry = entry;
126  SplitHardlink(&modified_entry);
127  const catalog::DirectoryEntryBase* base_entry =
128  static_cast<const catalog::DirectoryEntryBase*>(&modified_entry);
129  if (entry.IsChunkedFile()) {
130  assert(!chunks.IsEmpty());
131  output_catalog_mgr_->AddChunkedFile(*base_entry, xattrs, parent_path,
132  chunks);
133  } else {
134  output_catalog_mgr_->AddFile(*base_entry, xattrs, parent_path);
135  }
136  if (entry.IsLink()) {
137  perf::Inc(counters_->n_symlinks_added);
138  } else {
139  perf::Inc(counters_->n_files_added);
140  }
141  perf::Xadd(counters_->sz_added_bytes, static_cast<int64_t>(entry.size()));
142  }
143 }
144 
145 template <typename RwCatalogMgr, typename RoCatalogMgr>
147  const PathString& path, const catalog::DirectoryEntry& entry) {
148  const PathString rel_path = MakeRelative(path);
149 
150  if (entry.IsDirectory()) {
151  if (entry.IsNestedCatalogMountpoint()) {
152  output_catalog_mgr_->RemoveNestedCatalog(std::string(rel_path.c_str()),
153  false);
154  }
155 
156  output_catalog_mgr_->RemoveDirectory(rel_path.c_str());
157  perf::Inc(counters_->n_directories_removed);
158  } else if (entry.IsRegular() || entry.IsLink()) {
159  AbortIfHardlinked(entry);
160  output_catalog_mgr_->RemoveFile(rel_path.c_str());
161 
162  if (entry.IsLink()) {
163  perf::Inc(counters_->n_symlinks_removed);
164  } else {
165  perf::Inc(counters_->n_files_removed);
166  }
167 
168  perf::Xadd(counters_->sz_removed_bytes, static_cast<int64_t>(entry.size()));
169  }
170 }
171 
172 template <typename RwCatalogMgr, typename RoCatalogMgr>
174  const PathString& path, const catalog::DirectoryEntry& entry1,
175  const catalog::DirectoryEntry& entry2, const XattrList& xattrs,
176  const FileChunkList& chunks) {
177  const PathString rel_path = MakeRelative(path);
178 
179  const std::string parent_path =
180  std::strchr(rel_path.c_str(), '/') ? GetParentPath(rel_path).c_str() : "";
181 
182  if (entry1.IsNestedCatalogMountpoint() &&
183  entry2.IsNestedCatalogMountpoint()) {
184  // From nested catalog to nested catalog
185  RoCatalogMgr *new_catalog_mgr =
187  PathString mountpoint;
188  shash::Any new_hash;
189  uint64_t new_size;
190  const bool found = new_catalog_mgr->LookupNested(path, &mountpoint,
191  &new_hash, &new_size);
192  if (!found || !new_size) {
194  "CatalogMergeTool - nested catalog %s not found. Aborting",
195  rel_path.c_str());
196  }
197  output_catalog_mgr_->SwapNestedCatalog(rel_path.ToString(), new_hash,
198  new_size);
199  return false; // skip recursion into nested catalog mountpoints
200  } else if (entry1.IsDirectory() && entry2.IsDirectory()) {
201  // From directory to directory
202  const catalog::DirectoryEntryBase* base_entry =
203  static_cast<const catalog::DirectoryEntryBase*>(&entry2);
204  output_catalog_mgr_->TouchDirectory(*base_entry, xattrs, rel_path.c_str());
205  if (!entry1.IsNestedCatalogMountpoint() &&
206  entry2.IsNestedCatalogMountpoint()) {
207  output_catalog_mgr_->CreateNestedCatalog(std::string(rel_path.c_str()));
208  } else if (entry1.IsNestedCatalogMountpoint() &&
209  !entry2.IsNestedCatalogMountpoint()) {
210  output_catalog_mgr_->RemoveNestedCatalog(std::string(rel_path.c_str()));
211  }
212  perf::Inc(counters_->n_directories_changed);
213  } else if ((entry1.IsRegular() || entry1.IsLink()) && entry2.IsDirectory()) {
214  // From file to directory
215  AbortIfHardlinked(entry1);
216  output_catalog_mgr_->RemoveFile(rel_path.c_str());
217  output_catalog_mgr_->AddDirectory(entry2, xattrs, parent_path);
218  if (entry2.IsNestedCatalogMountpoint()) {
219  output_catalog_mgr_->CreateNestedCatalog(std::string(rel_path.c_str()));
220  }
221  if (entry1.IsLink()) {
222  perf::Inc(counters_->n_symlinks_removed);
223  } else {
224  perf::Inc(counters_->n_files_removed);
225  }
226  perf::Xadd(counters_->sz_removed_bytes,
227  static_cast<int64_t>(entry1.size()));
228  perf::Inc(counters_->n_directories_added);
229 
230  } else if (entry1.IsDirectory() && (entry2.IsRegular() || entry2.IsLink())) {
231  // From directory to file
232  if (entry1.IsNestedCatalogMountpoint()) {
233  // we merge the nested catalog with its parent, it will be the recursive
234  // procedure that will take care of deleting all the files.
235  output_catalog_mgr_->RemoveNestedCatalog(std::string(rel_path.c_str()),
236  /* merge = */ true);
237  }
238 
239  catalog::DirectoryEntry modified_entry = entry2;
240  SplitHardlink(&modified_entry);
241  const catalog::DirectoryEntryBase* base_entry =
242  static_cast<const catalog::DirectoryEntryBase*>(&modified_entry);
243 
244  output_catalog_mgr_->RemoveDirectory(rel_path.c_str());
245 
246  if (entry2.IsChunkedFile()) {
247  assert(!chunks.IsEmpty());
248  output_catalog_mgr_->AddChunkedFile(*base_entry, xattrs, parent_path,
249  chunks);
250  } else {
251  output_catalog_mgr_->AddFile(*base_entry, xattrs, parent_path);
252  }
253 
254  perf::Inc(counters_->n_directories_removed);
255  if (entry2.IsLink()) {
256  perf::Inc(counters_->n_symlinks_added);
257  } else {
258  perf::Inc(counters_->n_files_added);
259  }
260  perf::Xadd(counters_->sz_added_bytes, static_cast<int64_t>(entry2.size()));
261 
262  } else if ((entry1.IsRegular() || entry1.IsLink()) &&
263  (entry2.IsRegular() || entry2.IsLink())) {
264  // From file to file
265  AbortIfHardlinked(entry1);
266  catalog::DirectoryEntry modified_entry = entry2;
267  SplitHardlink(&modified_entry);
268  const catalog::DirectoryEntryBase* base_entry =
269  static_cast<const catalog::DirectoryEntryBase*>(&modified_entry);
270  output_catalog_mgr_->RemoveFile(rel_path.c_str());
271  if (entry2.IsChunkedFile()) {
272  assert(!chunks.IsEmpty());
273  output_catalog_mgr_->AddChunkedFile(*base_entry, xattrs, parent_path,
274  chunks);
275  } else {
276  output_catalog_mgr_->AddFile(*base_entry, xattrs, parent_path);
277  }
278 
279  if (entry1.IsRegular() && entry2.IsRegular()) {
280  perf::Inc(counters_->n_files_changed);
281  } else if (entry1.IsRegular() && entry2.IsLink()) {
282  perf::Inc(counters_->n_files_removed);
283  perf::Inc(counters_->n_symlinks_added);
284  } else if (entry1.IsLink() && entry2.IsRegular()) {
285  perf::Inc(counters_->n_symlinks_removed);
286  perf::Inc(counters_->n_files_added);
287  } else {
288  perf::Inc(counters_->n_symlinks_changed);
289  }
290  perf::Xadd(counters_->sz_removed_bytes,
291  static_cast<int64_t>(entry1.size()));
292  perf::Xadd(counters_->sz_added_bytes, static_cast<int64_t>(entry2.size()));
293  }
294  return true;
295 }
296 
297 template <typename RwCatalogMgr, typename RoCatalogMgr>
299  std::string* new_manifest_path) {
300  if (!output_catalog_mgr_->Commit(false, 0, manifest_)) {
302  "CatalogMergeTool - Could not commit output catalog");
303  return false;
304  }
305 
306  const std::string new_path = CreateTempPath(temp_dir_prefix_, 0600);
307 
308  if (!manifest_->Export(new_path)) {
310  "CatalogMergeTool - Could not export new manifest");
311  }
312 
313  *new_manifest_path = new_path;
314 
315  return true;
316 }
317 
318 } // namespace receiver
319 
320 #endif // CVMFS_RECEIVER_CATALOG_MERGE_TOOL_IMPL_H_
uint32_t linkcount() const
bool IsSubPath(const PathString &parent, const PathString &path)
bool Export(const std::string &path) const
Definition: manifest.cc:223
int64_t Xadd(class Counter *counter, const int64_t delta)
Definition: statistics.h:51
bool IsDirectory() const
size_t min_weight
Definition: params.h:36
size_t avg_chunk_size
Definition: params.h:27
std::string spooler_configuration
Definition: params.h:20
T * weak_ref() const
Definition: pointer.h:42
bool IsChunkedFile() const
#define PANIC(...)
Definition: exception.h:29
uint64_t size() const
void set_linkcount(const uint32_t linkcount)
zlib::Algorithms compression_alg
Definition: params.h:23
size_t nested_kcatalog_limit
Definition: params.h:30
bool use_autocatalogs
Definition: params.h:34
perf::Statistics * statistics_
Definition: repository.h:139
std::string CreateTempPath(const std::string &path_prefix, const int mode)
Definition: posix.cc:1034
virtual bool ReportModification(const PathString &path, const catalog::DirectoryEntry &old_entry, const catalog::DirectoryEntry &new_entry, const XattrList &xattrs, const FileChunkList &chunks)
assert((mem||(size==0))&&"Out Of Memory")
void AbortIfHardlinked(const catalog::DirectoryEntry &entry)
bool IsNestedCatalogMountpoint() const
uint64_t revision() const
Definition: manifest.h:129
PathString MakeRelative(const PathString &path)
bool use_file_chunking
Definition: params.h:25
NameString name() const
bool IsLink() const
size_t max_weight
Definition: params.h:35
size_t max_chunk_size
Definition: params.h:28
bool IsRegular() const
virtual bool IsReportablePath(const PathString &path)
RoCatalogMgr * GetNewCatalogMgr()
bool Run(const Params &params, std::string *new_manifest_path, uint64_t *final_rev)
virtual bool IsIgnoredPath(const PathString &path)
static RaiiTempDir * Create(const std::string &prefix)
Definition: raii_temp_dir.cc:9
shash::Any catalog_hash() const
Definition: manifest.h:132
void Inc(class Counter *counter)
Definition: statistics.h:50
bool IsEmpty() const
Definition: bigvector.h:72
manifest::Manifest * manifest_
Definition: repository.h:148
std::string ToString() const
Definition: shortstring.h:141
size_t min_chunk_size
Definition: params.h:26
void SplitHardlink(catalog::DirectoryEntry *entry)
size_t file_mbyte_limit
Definition: params.h:32
ShortString< kDefaultMaxPath, 0 > PathString
Definition: shortstring.h:217
PathString GetParentPath(const PathString &path)
Definition: shortstring.cc:15
bool CreateNewManifest(std::string *new_manifest_path)
size_t root_kcatalog_limit
Definition: params.h:31
bool Run(const PathString &path)
virtual void ReportAddition(const PathString &path, const catalog::DirectoryEntry &entry, const XattrList &xattrs, const FileChunkList &chunks)
bool generate_legacy_bulk_chunks
Definition: params.h:24
const char * c_str() const
Definition: shortstring.h:145
bool enforce_limits
Definition: params.h:29
virtual void ReportRemoval(const PathString &path, const catalog::DirectoryEntry &entry)
shash::Algorithms hash_alg
Definition: params.h:21
CVMFS_EXPORT void LogCvmfs(const LogSource source, const int mask, const char *format,...)
Definition: logging.cc:528