CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
sync_mediator.h
Go to the documentation of this file.
1 
24 #ifndef CVMFS_SYNC_MEDIATOR_H_
25 #define CVMFS_SYNC_MEDIATOR_H_
26 
27 #include <pthread.h>
28 
29 #include <map>
30 #include <set>
31 #include <stack>
32 #include <string>
33 #include <vector>
34 
35 #include "catalog_mgr_rw.h"
36 #include "compression.h"
37 #include "file_chunk.h"
38 #include "platform.h"
39 #include "publish/repository.h"
40 #include "statistics.h"
41 #include "swissknife_sync.h"
42 #include "sync_item.h"
43 #include "util/pointer.h"
44 #include "util/shared_ptr.h"
45 #include "xattr.h"
46 
47 namespace manifest {
48 class Manifest;
49 }
50 
51 struct Counters;
52 
53 namespace publish {
54 
55 class SyncDiffReporter : public DiffListener {
56  public:
57  enum PrintAction {
60  };
61 
62  explicit SyncDiffReporter(PrintAction print_action = kPrintChanges,
63  unsigned int processing_dot_interval = 100)
64  : print_action_(print_action),
65  processing_dot_interval_(processing_dot_interval),
66  changed_items_(0) {}
67 
68  virtual void OnInit(const history::History::Tag &from_tag,
69  const history::History::Tag &to_tag);
70  virtual void OnStats(const catalog::DeltaCounters &delta);
71 
72  virtual void OnAdd(const std::string &path,
73  const catalog::DirectoryEntry &entry);
74  virtual void OnRemove(const std::string &path,
75  const catalog::DirectoryEntry &entry);
76  virtual void OnModify(const std::string &path,
77  const catalog::DirectoryEntry &entry_from,
78  const catalog::DirectoryEntry &entry_to);
79  void CommitReport();
80 
81  private:
82  void PrintDots();
83  void AddImpl(const std::string &path);
84  void RemoveImpl(const std::string &path);
85  void ModifyImpl(const std::string &path);
86 
89  unsigned int changed_items_;
90 };
91 
98 struct HardlinkGroup {
99  explicit HardlinkGroup(SharedPtr<SyncItem> item) : master(item) {
100  hardlinks[master->GetRelativePath()] = item;
101  }
102 
104  hardlinks[entry->GetRelativePath()] = entry;
105  }
106 
110 };
111 
113  public:
114  virtual ~AbstractSyncMediator() = 0;
115 
116  virtual void RegisterUnionEngine(SyncUnion *engine) = 0;
117 
118  virtual void Add(SharedPtr<SyncItem> entry) = 0;
119  virtual void Touch(SharedPtr<SyncItem> entry) = 0;
120  virtual void Remove(SharedPtr<SyncItem> entry) = 0;
121  virtual void Replace(SharedPtr<SyncItem> entry) = 0;
122  virtual void Clone(const std::string from, const std::string to) = 0;
123 
124  virtual void AddUnmaterializedDirectory(SharedPtr<SyncItem> entry) = 0;
125 
126  virtual void EnterDirectory(SharedPtr<SyncItem> entry) = 0;
127  virtual void LeaveDirectory(SharedPtr<SyncItem> entry) = 0;
128 
129  virtual bool Commit(manifest::Manifest *manifest) = 0;
130 
131  virtual bool IsExternalData() const = 0;
132  virtual bool IsDirectIo() const = 0;
133  virtual zlib::Algorithms GetCompressionAlgorithm() const = 0;
134 };
135 
139 typedef std::map<uint64_t, HardlinkGroup> HardlinkGroupMap;
140 
150 class SyncMediator : public virtual AbstractSyncMediator {
151  public:
152  static const unsigned int processing_dot_interval = 100;
153 
155  const SyncParameters *params,
156  perf::StatisticsTemplate statistics);
157  void RegisterUnionEngine(SyncUnion *engine);
158  // Final class, it is not meant to be derived any further
159  ~SyncMediator();
160 
161  void Add(SharedPtr<SyncItem> entry);
162  void Touch(SharedPtr<SyncItem> entry);
163  void Remove(SharedPtr<SyncItem> entry);
164  void Replace(SharedPtr<SyncItem> entry);
165  void Clone(const std::string from, const std::string to);
166 
168 
171 
173 
174  // The sync union engine uses this information to create properly initialized
175  // sync items
176  bool IsExternalData() const { return params_->external_data; }
177  bool IsDirectIo() const { return params_->direct_io; }
179  return params_->compression_alg;
180  }
181 
182  private:
183  typedef std::stack<HardlinkGroupMap> HardlinkGroupMapStack;
184  typedef std::vector<HardlinkGroup> HardlinkGroupList;
185 
187 
188  // Called after figuring out the type of a path (file, symlink, dir)
189  void AddFile(SharedPtr<SyncItem> entry);
190  void RemoveFile(SharedPtr<SyncItem> entry);
191 
192  void AddDirectory(SharedPtr<SyncItem> entry);
195 
198 
200  void TouchingFileCallback(const std::string &parent_dir,
201  const std::string &file_name);
202  void TouchingSymlinkCallback(const std::string &parent_dir,
203  const std::string &link_name);
204  void TouchDirectoryCallback(const std::string &parent_dir,
205  const std::string &dir_name);
207  void RemoveFileCallback(const std::string &parent_dir,
208  const std::string &file_name);
209  void RemoveSymlinkCallback(const std::string &parent_dir,
210  const std::string &link_name);
211  void RemoveCharacterDeviceCallback(const std::string &parent_dir,
212  const std::string &link_name);
213  void RemoveBlockDeviceCallback(const std::string &parent_dir,
214  const std::string &link_name);
215  void RemoveFifoCallback(const std::string &parent_dir,
216  const std::string &link_name);
217  void RemoveSocketCallback(const std::string &parent_dir,
218  const std::string &link_name);
219  void RemoveDirectoryCallback(const std::string &parent_dir,
220  const std::string &dir_name);
221  bool IgnoreFileCallback(const std::string &parent_dir,
222  const std::string &file_name);
223  // Called by file system traversal
224  void EnterAddedDirectoryCallback(const std::string &parent_dir,
225  const std::string &dir_name);
226  void LeaveAddedDirectoryCallback(const std::string &parent_dir,
227  const std::string &dir_name);
229  bool AddDirectoryCallback(const std::string &parent_dir,
230  const std::string &dir_name);
231  void AddFileCallback(const std::string &parent_dir,
232  const std::string &file_name);
233  void AddCharacterDeviceCallback(const std::string &parent_dir,
234  const std::string &file_name);
235  void AddBlockDeviceCallback(const std::string &parent_dir,
236  const std::string &file_name);
237  void AddFifoCallback(const std::string &parent_dir,
238  const std::string &file_name);
239  void AddSocketCallback(const std::string &parent_dir,
240  const std::string &file_name);
241  void AddSymlinkCallback(const std::string &parent_dir,
242  const std::string &link_name);
243  SharedPtr<SyncItem> CreateSyncItem(const std::string &relative_parent_path,
244  const std::string &filename,
245  const SyncItemType entry_type) const;
246 
247  // Called by Upload Spooler
248  void PublishFilesCallback(const upload::SpoolerResult &result);
250 
251  // Hardlink handling
254  void LegacyRegularHardlinkCallback(const std::string &parent_dir,
255  const std::string &file_name);
256  void LegacySymlinkHardlinkCallback(const std::string &parent_dir,
257  const std::string &file_name);
258  void LegacyCharacterDeviceHardlinkCallback(const std::string &parent_dir,
259  const std::string &file_name);
260  void LegacyBlockDeviceHardlinkCallback(const std::string &parent_dir,
261  const std::string &file_name);
262  void LegacyFifoHardlinkCallback(const std::string &parent_dir,
263  const std::string &file_name);
264  void LegacySocketHardlinkCallback(const std::string &parent_dir,
265  const std::string &file_name);
269 
270  void AddLocalHardlinkGroups(const HardlinkGroupMap &hardlinks);
271  void AddHardlinkGroup(const HardlinkGroup &group);
272 
275 
277 
286 
291  pthread_mutex_t lock_file_queue_;
293 
295 
297  mutable unsigned int changed_items_;
298 
304 
306 }; // class SyncMediator
307 
308 } // namespace publish
309 
310 #endif // CVMFS_SYNC_MEDIATOR_H_
bool Commit(manifest::Manifest *manifest)
void RemoveSocketCallback(const std::string &parent_dir, const std::string &link_name)
void ModifyImpl(const std::string &path)
const manifest::Manifest * manifest() const
Definition: repository.h:123
virtual void LeaveDirectory(SharedPtr< SyncItem > entry)=0
void AddDirectoryRecursively(SharedPtr< SyncItem > entry)
virtual void AddUnmaterializedDirectory(SharedPtr< SyncItem > entry)=0
void TouchingFileCallback(const std::string &parent_dir, const std::string &file_name)
virtual void Replace(SharedPtr< SyncItem > entry)=0
HardlinkGroupMapStack hardlink_stack_
void AddBlockDeviceCallback(const std::string &parent_dir, const std::string &file_name)
void RemoveFile(SharedPtr< SyncItem > entry)
void CompleteHardlinks(SharedPtr< SyncItem > entry)
virtual zlib::Algorithms GetCompressionAlgorithm() const =0
void RemoveNestedCatalog(SharedPtr< SyncItem > directory)
void TouchingSymlinkCallback(const std::string &parent_dir, const std::string &link_name)
bool IsDirectIo() const
void CreateNestedCatalog(SharedPtr< SyncItem > directory)
UniquePtr< SyncDiffReporter > reporter_
virtual void OnInit(const history::History::Tag &from_tag, const history::History::Tag &to_tag)
virtual void OnModify(const std::string &path, const catalog::DirectoryEntry &entry_from, const catalog::DirectoryEntry &entry_to)
void RemoveDirectoryRecursively(SharedPtr< SyncItem > entry)
HardlinkGroupMap & GetHardlinkMap()
void LegacyCharacterDeviceHardlinkCallback(const std::string &parent_dir, const std::string &file_name)
virtual void Touch(SharedPtr< SyncItem > entry)=0
void Replace(SharedPtr< SyncItem > entry)
unsigned int changed_items_
void PublishHardlinksCallback(const upload::SpoolerResult &result)
void EnterDirectory(SharedPtr< SyncItem > entry)
zlib::Algorithms compression_alg
virtual void RegisterUnionEngine(SyncUnion *engine)=0
SharedPtr< SyncItem > CreateSyncItem(const std::string &relative_parent_path, const std::string &filename, const SyncItemType entry_type) const
UniquePtr< perf::FsCounters > counters_
void LegacySymlinkHardlinkCallback(const std::string &parent_dir, const std::string &file_name)
void AddDirectory(SharedPtr< SyncItem > entry)
void RemoveImpl(const std::string &path)
HardlinkGroupList hardlink_queue_
void RemoveCharacterDeviceCallback(const std::string &parent_dir, const std::string &link_name)
void AddFileCallback(const std::string &parent_dir, const std::string &file_name)
virtual bool IsExternalData() const =0
void LegacySocketHardlinkCallback(const std::string &parent_dir, const std::string &file_name)
std::map< std::string, SharedPtr< SyncItem > > SyncItemList
Definition: sync_item.h:322
void RemoveFileCallback(const std::string &parent_dir, const std::string &file_name)
void TouchDirectoryCallback(const std::string &parent_dir, const std::string &dir_name)
void PublishFilesCallback(const upload::SpoolerResult &result)
void Clone(const std::string from, const std::string to)
void AddFile(SharedPtr< SyncItem > entry)
void InsertLegacyHardlink(SharedPtr< SyncItem > entry)
Algorithms
Definition: compression.h:44
void AddUnmaterializedDirectory(SharedPtr< SyncItem > entry)
unsigned int processing_dot_interval_
Definition: sync_mediator.h:88
SyncUnion * union_engine_
SyncItemList file_queue_
const SyncParameters * params_
void RemoveFifoCallback(const std::string &parent_dir, const std::string &link_name)
virtual bool IsDirectIo() const =0
virtual void OnStats(const catalog::DeltaCounters &delta)
SyncMediator(catalog::WritableCatalogManager *catalog_manager, const SyncParameters *params, perf::StatisticsTemplate statistics)
void AddImpl(const std::string &path)
void AddSocketCallback(const std::string &parent_dir, const std::string &file_name)
static const unsigned int processing_dot_interval
void Touch(SharedPtr< SyncItem > entry)
void AddFifoCallback(const std::string &parent_dir, const std::string &file_name)
void AddSymlinkCallback(const std::string &parent_dir, const std::string &link_name)
void EnterAddedDirectoryCallback(const std::string &parent_dir, const std::string &dir_name)
SyncDiffReporter(PrintAction print_action=kPrintChanges, unsigned int processing_dot_interval=100)
Definition: sync_mediator.h:62
uint64_t GetTemporaryHardlinkGroupNumber(SharedPtr< SyncItem > entry) const
void Add(SharedPtr< SyncItem > entry)
bool AddDirectoryCallback(const std::string &parent_dir, const std::string &dir_name)
bool IgnoreFileCallback(const std::string &parent_dir, const std::string &file_name)
virtual void Add(SharedPtr< SyncItem > entry)=0
void LegacyFifoHardlinkCallback(const std::string &parent_dir, const std::string &file_name)
void AddCharacterDeviceCallback(const std::string &parent_dir, const std::string &file_name)
void AddLocalHardlinkGroups(const HardlinkGroupMap &hardlinks)
pthread_mutex_t lock_file_queue_
void InsertHardlink(SharedPtr< SyncItem > entry)
catalog::WritableCatalogManager * catalog_manager_
void LeaveDirectory(SharedPtr< SyncItem > entry)
virtual void OnRemove(const std::string &path, const catalog::DirectoryEntry &entry)
std::vector< HardlinkGroup > HardlinkGroupList
SyncItemType
Definition: sync_item.h:29
virtual void Clone(const std::string from, const std::string to)=0
virtual void Remove(SharedPtr< SyncItem > entry)=0
virtual bool Commit(manifest::Manifest *manifest)=0
void RemoveSymlinkCallback(const std::string &parent_dir, const std::string &link_name)
void RemoveBlockDeviceCallback(const std::string &parent_dir, const std::string &link_name)
bool IsExternalData() const
std::stack< HardlinkGroupMap > HardlinkGroupMapStack
zlib::Algorithms GetCompressionAlgorithm() const
virtual void EnterDirectory(SharedPtr< SyncItem > entry)=0
void LegacyBlockDeviceHardlinkCallback(const std::string &parent_dir, const std::string &file_name)
void RemoveDirectoryCallback(const std::string &parent_dir, const std::string &dir_name)
void TouchDirectory(SharedPtr< SyncItem > entry)
void EnsureAllowed(SharedPtr< SyncItem > entry)
void LegacyRegularHardlinkCallback(const std::string &parent_dir, const std::string &file_name)
void RegisterUnionEngine(SyncUnion *engine)
void AddHardlinkGroup(const HardlinkGroup &group)
void TouchDirectoryRecursively(SharedPtr< SyncItem > entry)
void RemoveDirectory(SharedPtr< SyncItem > entry)
std::map< uint64_t, HardlinkGroup > HardlinkGroupMap
virtual void OnAdd(const std::string &path, const catalog::DirectoryEntry &entry)
void Remove(SharedPtr< SyncItem > entry)
void LeaveAddedDirectoryCallback(const std::string &parent_dir, const std::string &dir_name)