| Line |
Branch |
Exec |
Source |
| 1 |
|
|
/** |
| 2 |
|
|
* This file is part of the CernVM File System. |
| 3 |
|
|
* |
| 4 |
|
|
* The SyncMediator is an intermediate layer between the UnionSync |
| 5 |
|
|
* implementation and the CatalogManager. |
| 6 |
|
|
* It's main responsibility is to unwind file system intrinsics as |
| 7 |
|
|
* deleting all files in a deleted directory. Furthermore newly |
| 8 |
|
|
* created directories are recursed and all included files are |
| 9 |
|
|
* added as a whole (performance improvement). |
| 10 |
|
|
* |
| 11 |
|
|
* Furthermore it keeps track of hard link relations. As we cannot store the |
| 12 |
|
|
* transient inode of the union file system in cvmfs, we just keep track of |
| 13 |
|
|
* hard link relations itself. Inodes will be assigned at run time of the CVMFS |
| 14 |
|
|
* client taking these relations into account. |
| 15 |
|
|
* |
| 16 |
|
|
* Another responsibility of this class is the creation and destruction |
| 17 |
|
|
* of nested catalogs. If a .cvmfscatalog magic file is encountered, |
| 18 |
|
|
* either on delete or add, it will be treated as nested catalog change. |
| 19 |
|
|
* |
| 20 |
|
|
* New and modified files are piped to external processes for hashing and |
| 21 |
|
|
* compression. Results come back in a pipe. |
| 22 |
|
|
*/ |
| 23 |
|
|
|
| 24 |
|
|
#ifndef CVMFS_SYNC_MEDIATOR_H_ |
| 25 |
|
|
#define CVMFS_SYNC_MEDIATOR_H_ |
| 26 |
|
|
|
| 27 |
|
|
#include <pthread.h> |
| 28 |
|
|
|
| 29 |
|
|
#include <map> |
| 30 |
|
|
#include <stack> |
| 31 |
|
|
#include <string> |
| 32 |
|
|
#include <vector> |
| 33 |
|
|
|
| 34 |
|
|
#include "catalog_mgr_rw.h" |
| 35 |
|
|
#include "compression/compression.h" |
| 36 |
|
|
#include "file_chunk.h" |
| 37 |
|
|
#include "publish/repository.h" |
| 38 |
|
|
#include "statistics.h" |
| 39 |
|
|
#include "swissknife_sync.h" |
| 40 |
|
|
#include "sync_item.h" |
| 41 |
|
|
#include "util/pointer.h" |
| 42 |
|
|
#include "util/shared_ptr.h" |
| 43 |
|
|
#include "xattr.h" |
| 44 |
|
|
|
| 45 |
|
|
namespace manifest { |
| 46 |
|
|
class Manifest; |
| 47 |
|
|
} |
| 48 |
|
|
|
| 49 |
|
|
namespace publish { |
| 50 |
|
|
|
| 51 |
|
|
class SyncDiffReporter : public DiffListener { |
| 52 |
|
|
public: |
| 53 |
|
|
enum PrintAction { |
| 54 |
|
|
kPrintDots, |
| 55 |
|
|
kPrintChanges |
| 56 |
|
|
}; |
| 57 |
|
|
|
| 58 |
|
✗ |
explicit SyncDiffReporter(PrintAction print_action = kPrintChanges, |
| 59 |
|
|
unsigned int processing_dot_interval = 100) |
| 60 |
|
✗ |
: print_action_(print_action) |
| 61 |
|
✗ |
, processing_dot_interval_(processing_dot_interval) |
| 62 |
|
✗ |
, changed_items_(0) { } |
| 63 |
|
|
|
| 64 |
|
|
virtual void OnInit(const history::History::Tag &from_tag, |
| 65 |
|
|
const history::History::Tag &to_tag); |
| 66 |
|
|
virtual void OnStats(const catalog::DeltaCounters &delta); |
| 67 |
|
|
|
| 68 |
|
|
virtual void OnAdd(const std::string &path, |
| 69 |
|
|
const catalog::DirectoryEntry &entry); |
| 70 |
|
|
virtual void OnRemove(const std::string &path, |
| 71 |
|
|
const catalog::DirectoryEntry &entry); |
| 72 |
|
|
virtual void OnModify(const std::string &path, |
| 73 |
|
|
const catalog::DirectoryEntry &entry_from, |
| 74 |
|
|
const catalog::DirectoryEntry &entry_to); |
| 75 |
|
|
void CommitReport(); |
| 76 |
|
|
|
| 77 |
|
|
private: |
| 78 |
|
|
void PrintDots(); |
| 79 |
|
|
void AddImpl(const std::string &path); |
| 80 |
|
|
void RemoveImpl(const std::string &path); |
| 81 |
|
|
void ModifyImpl(const std::string &path); |
| 82 |
|
|
|
| 83 |
|
|
PrintAction print_action_; |
| 84 |
|
|
unsigned int processing_dot_interval_; |
| 85 |
|
|
unsigned int changed_items_; |
| 86 |
|
|
}; |
| 87 |
|
|
|
| 88 |
|
|
/** |
| 89 |
|
|
* If we encounter a file with linkcount > 1 it will be added to a HardlinkGroup |
| 90 |
|
|
* After processing all files, the HardlinkGroups are populated with |
| 91 |
|
|
* related hardlinks |
| 92 |
|
|
* Assertion: linkcount == HardlinkGroup::hardlinks.size() at the end |
| 93 |
|
|
*/ |
| 94 |
|
|
struct HardlinkGroup { |
| 95 |
|
✗ |
explicit HardlinkGroup(const SharedPtr<SyncItem> &item) : master(item) { |
| 96 |
|
✗ |
hardlinks[master->GetRelativePath()] = item; |
| 97 |
|
|
} |
| 98 |
|
|
|
| 99 |
|
✗ |
void AddHardlink(const SharedPtr<SyncItem> &entry) { |
| 100 |
|
✗ |
hardlinks[entry->GetRelativePath()] = entry; |
| 101 |
|
|
} |
| 102 |
|
|
|
| 103 |
|
|
SharedPtr<SyncItem> master; |
| 104 |
|
|
SyncItemList hardlinks; |
| 105 |
|
|
FileChunkList file_chunks; |
| 106 |
|
|
}; |
| 107 |
|
|
|
| 108 |
|
|
class AbstractSyncMediator { |
| 109 |
|
|
public: |
| 110 |
|
|
virtual ~AbstractSyncMediator() = 0; |
| 111 |
|
|
|
| 112 |
|
|
virtual void RegisterUnionEngine(SyncUnion *engine) = 0; |
| 113 |
|
|
|
| 114 |
|
|
virtual void Add(SharedPtr<SyncItem> entry) = 0; |
| 115 |
|
|
virtual void Touch(SharedPtr<SyncItem> entry) = 0; |
| 116 |
|
|
virtual void Remove(SharedPtr<SyncItem> entry, bool fast_delete = false) = 0; |
| 117 |
|
|
virtual void Replace(SharedPtr<SyncItem> entry) = 0; |
| 118 |
|
|
virtual void Clone(const std::string from, const std::string to) = 0; |
| 119 |
|
|
|
| 120 |
|
|
virtual void AddUnmaterializedDirectory(SharedPtr<SyncItem> entry) = 0; |
| 121 |
|
|
|
| 122 |
|
|
virtual void EnterDirectory(SharedPtr<SyncItem> entry) = 0; |
| 123 |
|
|
virtual void LeaveDirectory(SharedPtr<SyncItem> entry) = 0; |
| 124 |
|
|
|
| 125 |
|
|
virtual bool Commit(manifest::Manifest *manifest) = 0; |
| 126 |
|
|
|
| 127 |
|
|
virtual bool IsExternalData() const = 0; |
| 128 |
|
|
virtual bool IsDirectIo() const = 0; |
| 129 |
|
|
virtual zlib::Algorithms GetCompressionAlgorithm() const = 0; |
| 130 |
|
|
}; |
| 131 |
|
|
|
| 132 |
|
|
/** |
| 133 |
|
|
* Mapping of inode number to the related HardlinkGroup. |
| 134 |
|
|
*/ |
| 135 |
|
|
typedef std::map<uint64_t, HardlinkGroup> HardlinkGroupMap; |
| 136 |
|
|
|
| 137 |
|
|
/** |
| 138 |
|
|
* List of bundle specification files found in a certain directory |
| 139 |
|
|
* (non-recursive). |
| 140 |
|
|
*/ |
| 141 |
|
|
typedef std::vector<std::string> BundleSpecs; |
| 142 |
|
|
|
| 143 |
|
|
/** |
| 144 |
|
|
* The SyncMediator refines the input received from a concrete UnionSync object. |
| 145 |
|
|
* For example, it resolves the insertion and deletion of complete directories |
| 146 |
|
|
* by recursing them. It works as a mediator between the union file system and |
| 147 |
|
|
* forwards the correct database commands to the catalog handler to sync the |
| 148 |
|
|
* changes into the repository. |
| 149 |
|
|
* Furthermore it sends new and modified files to the spooler for compression |
| 150 |
|
|
* and hashing. |
| 151 |
|
|
*/ |
| 152 |
|
|
class SyncMediator : public virtual AbstractSyncMediator { |
| 153 |
|
|
public: |
| 154 |
|
|
static const unsigned int processing_dot_interval = 100; |
| 155 |
|
|
|
| 156 |
|
|
SyncMediator(catalog::WritableCatalogManager *catalog_manager, |
| 157 |
|
|
const SyncParameters *params, |
| 158 |
|
|
perf::StatisticsTemplate statistics); |
| 159 |
|
|
void RegisterUnionEngine(SyncUnion *engine); |
| 160 |
|
|
// Final class, it is not meant to be derived any further |
| 161 |
|
|
~SyncMediator(); |
| 162 |
|
|
|
| 163 |
|
|
void Add(SharedPtr<SyncItem> entry); |
| 164 |
|
|
void Touch(SharedPtr<SyncItem> entry); |
| 165 |
|
|
void Remove(SharedPtr<SyncItem> entry, bool fast_delete = false); |
| 166 |
|
|
void Replace(SharedPtr<SyncItem> entry); |
| 167 |
|
|
void Clone(const std::string from, const std::string to); |
| 168 |
|
|
|
| 169 |
|
|
void AddUnmaterializedDirectory(SharedPtr<SyncItem> entry); |
| 170 |
|
|
|
| 171 |
|
|
void EnterDirectory(SharedPtr<SyncItem> entry); |
| 172 |
|
|
void LeaveDirectory(SharedPtr<SyncItem> entry); |
| 173 |
|
|
|
| 174 |
|
|
bool Commit(manifest::Manifest *manifest); |
| 175 |
|
|
|
| 176 |
|
|
// The sync union engine uses this information to create properly initialized |
| 177 |
|
|
// sync items |
| 178 |
|
✗ |
bool IsExternalData() const { return params_->external_data; } |
| 179 |
|
✗ |
bool IsDirectIo() const { return params_->direct_io; } |
| 180 |
|
✗ |
zlib::Algorithms GetCompressionAlgorithm() const { |
| 181 |
|
✗ |
return params_->compression_alg; |
| 182 |
|
|
} |
| 183 |
|
|
|
| 184 |
|
|
private: |
| 185 |
|
|
typedef std::stack<HardlinkGroupMap> HardlinkGroupMapStack; |
| 186 |
|
|
typedef std::vector<HardlinkGroup> HardlinkGroupList; |
| 187 |
|
|
|
| 188 |
|
|
void EnsureAllowed(SharedPtr<SyncItem> entry); |
| 189 |
|
|
|
| 190 |
|
|
// Called after figuring out the type of a path (file, symlink, dir) |
| 191 |
|
|
void AddFile(SharedPtr<SyncItem> entry); |
| 192 |
|
|
void RemoveFile(SharedPtr<SyncItem> entry); |
| 193 |
|
|
|
| 194 |
|
|
void AddDirectory(SharedPtr<SyncItem> entry); |
| 195 |
|
|
void RemoveDirectory(SharedPtr<SyncItem> entry); |
| 196 |
|
|
void TouchDirectory(SharedPtr<SyncItem> entry); |
| 197 |
|
|
|
| 198 |
|
|
void CreateNestedCatalog(SharedPtr<SyncItem> directory); |
| 199 |
|
|
void RemoveNestedCatalog(SharedPtr<SyncItem> directory); |
| 200 |
|
|
|
| 201 |
|
|
void TouchDirectoryRecursively(SharedPtr<SyncItem> entry); |
| 202 |
|
|
void TouchingFileCallback(const std::string &parent_dir, |
| 203 |
|
|
const std::string &file_name); |
| 204 |
|
|
void TouchingSymlinkCallback(const std::string &parent_dir, |
| 205 |
|
|
const std::string &link_name); |
| 206 |
|
|
void TouchDirectoryCallback(const std::string &parent_dir, |
| 207 |
|
|
const std::string &dir_name); |
| 208 |
|
|
void RemoveDirectoryRecursively(SharedPtr<SyncItem> entry, |
| 209 |
|
|
bool fast_delete = false); |
| 210 |
|
|
void RemoveFileCallback(const std::string &parent_dir, |
| 211 |
|
|
const std::string &file_name); |
| 212 |
|
|
void RemoveSymlinkCallback(const std::string &parent_dir, |
| 213 |
|
|
const std::string &link_name); |
| 214 |
|
|
void RemoveCharacterDeviceCallback(const std::string &parent_dir, |
| 215 |
|
|
const std::string &link_name); |
| 216 |
|
|
void RemoveBlockDeviceCallback(const std::string &parent_dir, |
| 217 |
|
|
const std::string &link_name); |
| 218 |
|
|
void RemoveFifoCallback(const std::string &parent_dir, |
| 219 |
|
|
const std::string &link_name); |
| 220 |
|
|
void RemoveSocketCallback(const std::string &parent_dir, |
| 221 |
|
|
const std::string &link_name); |
| 222 |
|
|
void RemoveDirectoryCallback(const std::string &parent_dir, |
| 223 |
|
|
const std::string &dir_name); |
| 224 |
|
|
bool IgnoreFileCallback(const std::string &parent_dir, |
| 225 |
|
|
const std::string &file_name); |
| 226 |
|
|
// Called by file system traversal |
| 227 |
|
|
void EnterAddedDirectoryCallback(const std::string &parent_dir, |
| 228 |
|
|
const std::string &dir_name); |
| 229 |
|
|
void LeaveAddedDirectoryCallback(const std::string &parent_dir, |
| 230 |
|
|
const std::string &dir_name); |
| 231 |
|
|
void AddDirectoryRecursively(SharedPtr<SyncItem> entry); |
| 232 |
|
|
bool AddDirectoryCallback(const std::string &parent_dir, |
| 233 |
|
|
const std::string &dir_name); |
| 234 |
|
|
void AddFileCallback(const std::string &parent_dir, |
| 235 |
|
|
const std::string &file_name); |
| 236 |
|
|
void AddCharacterDeviceCallback(const std::string &parent_dir, |
| 237 |
|
|
const std::string &file_name); |
| 238 |
|
|
void AddBlockDeviceCallback(const std::string &parent_dir, |
| 239 |
|
|
const std::string &file_name); |
| 240 |
|
|
void AddFifoCallback(const std::string &parent_dir, |
| 241 |
|
|
const std::string &file_name); |
| 242 |
|
|
void AddSocketCallback(const std::string &parent_dir, |
| 243 |
|
|
const std::string &file_name); |
| 244 |
|
|
void AddSymlinkCallback(const std::string &parent_dir, |
| 245 |
|
|
const std::string &link_name); |
| 246 |
|
|
SharedPtr<SyncItem> CreateSyncItem(const std::string &relative_parent_path, |
| 247 |
|
|
const std::string &filename, |
| 248 |
|
|
const SyncItemType entry_type) const; |
| 249 |
|
|
|
| 250 |
|
|
// Called by Upload Spooler |
| 251 |
|
|
void PublishFilesCallback(const upload::SpoolerResult &result); |
| 252 |
|
|
void PublishHardlinksCallback(const upload::SpoolerResult &result); |
| 253 |
|
|
|
| 254 |
|
|
std::string GetBundleTriggerPath(SharedPtr<SyncItem> bundle_spec_entry) const; |
| 255 |
|
|
void InsertBundleSpec(SharedPtr<SyncItem> entry); |
| 256 |
|
|
void AddBundleSpecs(); |
| 257 |
|
|
|
| 258 |
|
|
// Hardlink handling |
| 259 |
|
|
void CompleteHardlinks(SharedPtr<SyncItem> entry); |
| 260 |
|
✗ |
HardlinkGroupMap &GetHardlinkMap() { return hardlink_stack_.top(); } |
| 261 |
|
|
void LegacyRegularHardlinkCallback(const std::string &parent_dir, |
| 262 |
|
|
const std::string &file_name); |
| 263 |
|
|
void LegacySymlinkHardlinkCallback(const std::string &parent_dir, |
| 264 |
|
|
const std::string &file_name); |
| 265 |
|
|
void LegacyCharacterDeviceHardlinkCallback(const std::string &parent_dir, |
| 266 |
|
|
const std::string &file_name); |
| 267 |
|
|
void LegacyBlockDeviceHardlinkCallback(const std::string &parent_dir, |
| 268 |
|
|
const std::string &file_name); |
| 269 |
|
|
void LegacyFifoHardlinkCallback(const std::string &parent_dir, |
| 270 |
|
|
const std::string &file_name); |
| 271 |
|
|
void LegacySocketHardlinkCallback(const std::string &parent_dir, |
| 272 |
|
|
const std::string &file_name); |
| 273 |
|
|
void InsertLegacyHardlink(SharedPtr<SyncItem> entry); |
| 274 |
|
|
uint64_t GetTemporaryHardlinkGroupNumber(SharedPtr<SyncItem> entry) const; |
| 275 |
|
|
void InsertHardlink(SharedPtr<SyncItem> entry); |
| 276 |
|
|
|
| 277 |
|
|
void AddLocalHardlinkGroups(const HardlinkGroupMap &hardlinks); |
| 278 |
|
|
void AddHardlinkGroup(const HardlinkGroup &group); |
| 279 |
|
|
|
| 280 |
|
|
catalog::WritableCatalogManager *catalog_manager_; |
| 281 |
|
|
SyncUnion *union_engine_; |
| 282 |
|
|
|
| 283 |
|
|
bool handle_hardlinks_; |
| 284 |
|
|
|
| 285 |
|
|
/** |
| 286 |
|
|
* When fast_delete is requested but the target path is not a nested catalog |
| 287 |
|
|
* transition point, we fall back to filesystem traversal. This flag ensures |
| 288 |
|
|
* that any nested catalog transition points encountered during the traversal |
| 289 |
|
|
* are still fast-deleted instead of being recursively traversed. |
| 290 |
|
|
*/ |
| 291 |
|
|
bool recursive_fast_delete_; |
| 292 |
|
|
|
| 293 |
|
|
/** |
| 294 |
|
|
* Hardlinks are supported as long as they all reside in the same directory. |
| 295 |
|
|
* If a recursion enters a directory, we push an empty HardlinkGroupMap to |
| 296 |
|
|
* keep track of the hardlinks of this directory. |
| 297 |
|
|
* When leaving a directory (i.e. it is completely processed) the stack is |
| 298 |
|
|
* popped and the HardlinkGroupMap is processed. |
| 299 |
|
|
*/ |
| 300 |
|
|
HardlinkGroupMapStack hardlink_stack_; |
| 301 |
|
|
|
| 302 |
|
|
/** |
| 303 |
|
|
* Files that trigger preloading a file bundle have a .cvmfsbundle-<filename> |
| 304 |
|
|
* bundle specification file. We don't know in which order we encounter the |
| 305 |
|
|
* main file and the bundle specification file. So we pick up all the bundle |
| 306 |
|
|
* specifications first and compare them to the present files at the end of |
| 307 |
|
|
* the transaction. |
| 308 |
|
|
*/ |
| 309 |
|
|
BundleSpecs bundle_specs_; |
| 310 |
|
|
|
| 311 |
|
|
/** |
| 312 |
|
|
* New and modified files are sent to an external spooler for hashing and |
| 313 |
|
|
* compression. A spooler callback adds them to the catalogs, once processed. |
| 314 |
|
|
*/ |
| 315 |
|
|
pthread_mutex_t lock_file_queue_; |
| 316 |
|
|
SyncItemList file_queue_; |
| 317 |
|
|
|
| 318 |
|
|
HardlinkGroupList hardlink_queue_; |
| 319 |
|
|
|
| 320 |
|
|
const SyncParameters *params_; |
| 321 |
|
|
mutable unsigned int changed_items_; |
| 322 |
|
|
|
| 323 |
|
|
/** |
| 324 |
|
|
* By default, files have no extended attributes. |
| 325 |
|
|
*/ |
| 326 |
|
|
XattrList default_xattrs_; |
| 327 |
|
|
UniquePtr<perf::FsCounters> counters_; |
| 328 |
|
|
|
| 329 |
|
|
UniquePtr<SyncDiffReporter> reporter_; |
| 330 |
|
|
}; // class SyncMediator |
| 331 |
|
|
|
| 332 |
|
|
} // namespace publish |
| 333 |
|
|
|
| 334 |
|
|
#endif // CVMFS_SYNC_MEDIATOR_H_ |
| 335 |
|
|
|