CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
catalog_mgr_rw.h
Go to the documentation of this file.
1 
29 #ifndef CVMFS_CATALOG_MGR_RW_H_
30 #define CVMFS_CATALOG_MGR_RW_H_
31 
32 #include <pthread.h>
33 #include <stdint.h>
34 
35 #include <map>
36 #include <set>
37 #include <string>
38 
39 #include "catalog_mgr_ro.h"
40 #include "catalog_rw.h"
41 #include "file_chunk.h"
42 #include "upload_spooler_result.h"
43 #include "util_concurrency.h"
44 #include "xattr.h"
45 
46 class XattrList;
47 namespace upload {
48 class Spooler;
49 }
50 
51 namespace download {
52 class DownloadManager;
53 }
54 
55 namespace manifest {
56 class Manifest;
57 }
58 
59 namespace perf {
60 class Statistics;
61 }
62 
63 namespace catalog {
64 template <class CatalogMgrT>
65 class CatalogBalancer;
66 }
67 
68 namespace catalog {
69 
72  // TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me
73  // after refactoring of the swissknife utility.
74  friend class VirtualCatalog;
75 
76  public:
78  const std::string &stratum0,
79  const std::string &dir_temp,
80  upload::Spooler *spooler,
81  download::DownloadManager *download_manager,
82  bool enforce_limits,
83  const unsigned nested_kcatalog_limit,
84  const unsigned root_kcatalog_limit,
85  const unsigned file_mbyte_limit,
87  bool is_balanceable,
88  unsigned max_weight,
89  unsigned min_weight);
91  static manifest::Manifest *CreateRepository(const std::string &dir_temp,
92  const bool volatile_content,
93  const std::string &voms_authz,
94  upload::Spooler *spooler);
95 
96  // DirectoryEntry handling
97  void AddFile(const DirectoryEntryBase &entry,
98  const XattrList &xattrs,
99  const std::string &parent_directory)
100  {
101  AddFile(DirectoryEntry(entry), xattrs, parent_directory);
102  }
103  void AddChunkedFile(const DirectoryEntryBase &entry,
104  const XattrList &xattrs,
105  const std::string &parent_directory,
106  const FileChunkList &file_chunks);
107  void RemoveFile(const std::string &file_path);
108 
109  void AddDirectory(const DirectoryEntryBase &entry,
110  const XattrList &xattrs,
111  const std::string &parent_directory);
112  void TouchDirectory(const DirectoryEntryBase &entry,
113  const XattrList &xattrs,
114  const std::string &directory_path);
115  void RemoveDirectory(const std::string &directory_path);
116 
117  void Clone(const std::string from, const std::string to);
118  void CloneTree(const std::string &from_dir, const std::string &to_dir);
119 
120  // Hardlink group handling
121  void AddHardlinkGroup(const DirectoryEntryBaseList &entries,
122  const XattrList &xattrs,
123  const std::string &parent_directory,
124  const FileChunkList &file_chunks);
125  void ShrinkHardlinkGroup(const std::string &remove_path);
126 
127  // Nested catalog handling
128  void CreateNestedCatalog(const std::string &mountpoint);
129  void RemoveNestedCatalog(const std::string &mountpoint,
130  const bool merge = true);
131  void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash,
132  const uint64_t new_size);
133  bool IsTransitionPoint(const std::string &mountpoint);
134  WritableCatalog *GetHostingCatalog(const std::string &path);
135 
136  inline bool IsBalanceable() const { return is_balanceable_; }
140  void PrecalculateListings();
141 
142  void SetTTL(const uint64_t new_ttl);
143  bool SetVOMSAuthz(const std::string &voms_authz);
144  bool Commit(const bool stop_for_tweaks,
145  const uint64_t manual_revision,
147 
148  void Balance() {
149  if (IsBalanceable()) {
150  DoBalance();
151  } else {
152  LogCvmfs(kLogCatalog, kLogVerboseMsg, "Not balancing the catalog "
153  "manager because it is not balanceable");
154  }
155  }
156 
157  protected:
159 
160  Catalog *CreateCatalog(const PathString &mountpoint,
161  const shash::Any &catalog_hash,
162  Catalog *parent_catalog);
163  void ActivateCatalog(Catalog *catalog);
164 
165  void AddFile(const DirectoryEntry &entry,
166  const XattrList &xattrs,
167  const std::string &parent_directory);
168 
169  private:
170  bool FindCatalog(const std::string &path,
171  WritableCatalog **result,
172  DirectoryEntry *dirent = NULL);
173  void DoBalance();
174  void FixWeight(WritableCatalog *catalog);
175 
176  void CloneTreeImpl(const PathString &source_dir,
177  const std::string &dest_parent_dir,
178  const NameString &dest_name);
179 
180  struct CatalogInfo {
181  uint64_t ttl;
182  size_t size;
184  unsigned int revision;
185  };
186 
190  };
191 
192  CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks);
193  void FinalizeCatalog(WritableCatalog *catalog,
194  const bool stop_for_tweaks);
196 
199  result);
200  assert(dirty);
201  }
203  WritableCatalogList *result) const;
204 
205  void CatalogUploadCallback(const upload::SpoolerResult &result,
206  const CatalogUploadContext clg_upload_context);
207 
208  private:
209  inline void SyncLock() { pthread_mutex_lock(sync_lock_); }
210  inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); }
211 
212  //****************************************************************************
213  // Workaround -- Serialized Catalog Committing
215  const unsigned int number_of_dirty_catalogs =
217  assert(number_of_dirty_catalogs <= result->size());
218  }
219  int GetModifiedCatalogsRecursively(const Catalog *catalog,
220  WritableCatalogList *result) const;
222  const upload::SpoolerResult &result,
223  const CatalogUploadContext unused);
224  CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks);
225  //****************************************************************************
226 
227  // defined in catalog_mgr_rw.cc
228  static const std::string kCatalogFilename;
229 
230  // private lock of WritableCatalogManager
231  pthread_mutex_t *sync_lock_;
232  upload::Spooler *spooler_;
233 
234  pthread_mutex_t *catalog_processing_lock_;
235  std::map<std::string, WritableCatalog*> catalog_processing_map_;
236 
237  // TODO(jblomer): catalog limits should become its own struct
242 
247 
251  const bool is_balanceable_;
252 
259  const unsigned max_weight_;
260 
268  const unsigned min_weight_;
269 
275  const unsigned balance_weight_;
276 }; // class WritableCatalogManager
277 
278 } // namespace catalog
279 
280 #endif // CVMFS_CATALOG_MGR_RW_H_
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
void AddChunkedFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
const manifest::Manifest * manifest() const
Definition: repository.h:123
void CloneTree(const std::string &from_dir, const std::string &to_dir)
void AddHardlinkGroup(const DirectoryEntryBaseList &entries, const XattrList &xattrs, const std::string &parent_directory, const FileChunkList &file_chunks)
void AddDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
void RemoveDirectory(const std::string &directory_path)
WritableCatalog * GetHostingCatalog(const std::string &path)
WritableCatalogManager(const shash::Any &base_hash, const std::string &stratum0, const std::string &dir_temp, upload::Spooler *spooler, download::DownloadManager *download_manager, bool enforce_limits, const unsigned nested_kcatalog_limit, const unsigned root_kcatalog_limit, const unsigned file_mbyte_limit, perf::Statistics *statistics, bool is_balanceable, unsigned max_weight, unsigned min_weight)
void FixWeight(WritableCatalog *catalog)
void ScheduleCatalogProcessing(WritableCatalog *catalog)
std::map< std::string, WritableCatalog * > catalog_processing_map_
void SetTTL(const uint64_t new_ttl)
int GetModifiedCatalogsRecursively(const Catalog *catalog, WritableCatalogList *result) const
assert((mem||(size==0))&&"Out Of Memory")
bool FindCatalog(const std::string &path, WritableCatalog **result, DirectoryEntry *dirent=NULL)
bool Commit(const bool stop_for_tweaks, const uint64_t manual_revision, manifest::Manifest *manifest)
pthread_mutex_t * catalog_processing_lock_
std::vector< WritableCatalog * > WritableCatalogList
Definition: catalog_rw.h:203
void CloneTreeImpl(const PathString &source_dir, const std::string &dest_parent_dir, const NameString &dest_name)
bool IsTransitionPoint(const std::string &mountpoint)
void CatalogUploadCallback(const upload::SpoolerResult &result, const CatalogUploadContext clg_upload_context)
void TouchDirectory(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &directory_path)
static manifest::Manifest * CreateRepository(const std::string &dir_temp, const bool volatile_content, const std::string &voms_authz, upload::Spooler *spooler)
void ActivateCatalog(Catalog *catalog)
void ShrinkHardlinkGroup(const std::string &remove_path)
bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, WritableCatalogList *result) const
const std::string & dir_temp() const
CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks)
CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks)
bool SetVOMSAuthz(const std::string &voms_authz)
void Clone(const std::string from, const std::string to)
void RemoveFile(const std::string &file_path)
void CatalogUploadSerializedCallback(const upload::SpoolerResult &result, const CatalogUploadContext unused)
void RemoveNestedCatalog(const std::string &mountpoint, const bool merge=true)
static const std::string kCatalogFilename
std::vector< DirectoryEntryBase > DirectoryEntryBaseList
const int kLogVerboseMsg
void AddFile(const DirectoryEntryBase &entry, const XattrList &xattrs, const std::string &parent_directory)
void SwapNestedCatalog(const string &mountpoint, const shash::Any &new_hash, const uint64_t new_size)
static void size_t size
Definition: smalloc.h:47
void GetModifiedCatalogLeafs(WritableCatalogList *result) const
Catalog * CreateCatalog(const PathString &mountpoint, const shash::Any &catalog_hash, Catalog *parent_catalog)
void FinalizeCatalog(WritableCatalog *catalog, const bool stop_for_tweaks)
void CreateNestedCatalog(const std::string &mountpoint)
void GetModifiedCatalogs(WritableCatalogList *result) const
const shash::Any & base_hash() const