1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System |
3 |
|
|
*/ |
4 |
|
|
|
5 |
|
|
#ifndef CVMFS_SYNC_ITEM_H_ |
6 |
|
|
#define CVMFS_SYNC_ITEM_H_ |
7 |
|
|
|
8 |
|
|
#include <sys/types.h> |
9 |
|
|
|
10 |
|
|
#if !defined(__APPLE__) |
11 |
|
|
#include <sys/sysmacros.h> |
12 |
|
|
#endif // __APPLE__ |
13 |
|
|
|
14 |
|
|
#include <cstring> |
15 |
|
|
#include <map> |
16 |
|
|
#include <string> |
17 |
|
|
|
18 |
|
|
#include "directory_entry.h" |
19 |
|
|
#include "duplex_libarchive.h" |
20 |
|
|
#include "file_chunk.h" |
21 |
|
|
#include "hash.h" |
22 |
|
|
#include "platform.h" |
23 |
|
|
#include "util/shared_ptr.h" |
24 |
|
|
|
25 |
|
|
class IngestionSource; |
26 |
|
|
|
27 |
|
|
namespace publish { |
28 |
|
|
|
29 |
|
|
enum SyncItemType { |
30 |
|
|
kItemDir, |
31 |
|
|
kItemFile, |
32 |
|
|
kItemSymlink, |
33 |
|
|
kItemCharacterDevice, |
34 |
|
|
kItemBlockDevice, |
35 |
|
|
kItemFifo, |
36 |
|
|
kItemSocket, |
37 |
|
|
kItemNew, |
38 |
|
|
kItemMarker, |
39 |
|
|
kItemUnknown, |
40 |
|
|
}; |
41 |
|
|
|
42 |
|
|
class SyncUnion; |
43 |
|
|
/** |
44 |
|
|
* Every directory entry emitted by the FileSystemTraversal is wrapped in a |
45 |
|
|
* SyncItem structure by the factory method SyncUnion::CreateSyncItem(). |
46 |
|
|
* |
47 |
|
|
* Since we are dealing with a union file system setup, this class represents |
48 |
|
|
* potentially three concrete files: |
49 |
|
|
* - <read-only path>/<filename> | cf. rdonly_stat_ |
50 |
|
|
* - <scratch (read-write) branch>/<filename> | cf. scratch_stat_ |
51 |
|
|
* - <union volume path>/<filename> | cf. union_stat_ |
52 |
|
|
* |
53 |
|
|
* This class caches stat calls to the underlying files in different branches of |
54 |
|
|
* the union file system and hides some interpretation details. |
55 |
|
|
*/ |
56 |
|
|
class SyncItem { |
57 |
|
|
// only SyncUnion can create SyncItems (see SyncUnion::CreateSyncItem). |
58 |
|
|
// SyncUnionTarball can create SyncItemTar and SyncItemDummyDir. |
59 |
|
|
|
60 |
|
|
public: |
61 |
|
|
SyncItem(); |
62 |
|
|
virtual ~SyncItem(); |
63 |
|
|
|
64 |
|
|
inline bool IsDirectory() const { return IsType(kItemDir); } |
65 |
|
|
inline bool WasDirectory() const { return WasType(kItemDir); } |
66 |
|
|
inline bool IsRegularFile() const { return IsType(kItemFile); } |
67 |
|
|
inline bool WasRegularFile() const { return WasType(kItemFile); } |
68 |
|
|
inline bool IsSymlink() const { return IsType(kItemSymlink); } |
69 |
|
|
inline bool WasSymlink() const { return WasType(kItemSymlink); } |
70 |
|
|
inline bool IsNew() const { return WasType(kItemNew); } |
71 |
|
|
inline bool IsTouched() const { |
72 |
|
|
return (GetRdOnlyFiletype() == GetUnionFiletype()) && |
73 |
|
|
(GetRdOnlyFiletype() == GetScratchFiletype()) && |
74 |
|
|
(GetUnionFiletype() == GetScratchFiletype()); |
75 |
|
|
} |
76 |
|
|
inline bool IsCharacterDevice() const { return IsType(kItemCharacterDevice); } |
77 |
|
|
inline bool IsBlockDevice() const { return IsType(kItemBlockDevice); } |
78 |
|
|
inline bool IsFifo() const { return IsType(kItemFifo); } |
79 |
|
|
inline bool IsSocket() const { return IsType(kItemSocket); } |
80 |
|
|
inline bool IsGraftMarker() const { return IsType(kItemMarker); } |
81 |
|
|
inline bool IsExternalData() const { return external_data_; } |
82 |
|
|
|
83 |
|
|
inline bool IsWhiteout() const { return whiteout_; } |
84 |
|
|
inline bool IsCatalogMarker() const { return filename_ == ".cvmfscatalog"; } |
85 |
|
|
inline bool IsOpaqueDirectory() const { return IsDirectory() && opaque_; } |
86 |
|
|
|
87 |
|
|
inline bool IsSpecialFile() const { |
88 |
|
|
return IsCharacterDevice() || IsBlockDevice() || IsFifo() || IsSocket(); |
89 |
|
|
} |
90 |
|
|
inline bool WasSpecialFile() const { |
91 |
|
|
return WasType(kItemCharacterDevice) || |
92 |
|
|
WasType(kItemBlockDevice) || |
93 |
|
|
WasType(kItemFifo) || |
94 |
|
|
WasType(kItemSocket); |
95 |
|
|
} |
96 |
|
|
|
97 |
|
|
inline unsigned int GetRdevMajor() const { |
98 |
|
|
assert(IsSpecialFile()); |
99 |
|
|
StatUnion(true); return major(union_stat_.stat.st_rdev); |
100 |
|
|
} |
101 |
|
|
|
102 |
|
|
inline unsigned int GetRdevMinor() const { |
103 |
|
|
assert(IsSpecialFile()); |
104 |
|
|
StatUnion(true); return minor(union_stat_.stat.st_rdev); |
105 |
|
|
} |
106 |
|
|
|
107 |
|
|
bool HasCatalogMarker() const { return has_catalog_marker_; } |
108 |
|
|
bool HasGraftMarker() const { return graft_marker_present_; } |
109 |
|
|
bool IsValidGraft() const { return valid_graft_; } |
110 |
|
|
bool IsChunkedGraft() const { return graft_chunklist_; } |
111 |
|
|
|
112 |
|
|
inline const FileChunkList* GetGraftChunks() const {return graft_chunklist_;} |
113 |
|
|
inline shash::Any GetContentHash() const { return content_hash_; } |
114 |
|
|
inline void SetContentHash(const shash::Any &hash) { content_hash_ = hash; } |
115 |
|
|
inline bool HasContentHash() const { return !content_hash_.IsNull(); } |
116 |
|
|
void SetExternalData(bool val) {external_data_ = val;} |
117 |
|
|
|
118 |
|
|
inline zlib::Algorithms GetCompressionAlgorithm() const { |
119 |
|
|
return compression_algorithm_; |
120 |
|
|
} |
121 |
|
|
inline void SetCompressionAlgorithm(const zlib::Algorithms &alg) { |
122 |
|
|
compression_algorithm_ = alg; |
123 |
|
|
} |
124 |
|
|
|
125 |
|
|
/** |
126 |
|
|
* Generates a DirectoryEntry that can be directly stored into a catalog db. |
127 |
|
|
* Note: this sets the inode fields to kInvalidInode as well as the link |
128 |
|
|
* count to 1 if MaskHardlink() has been called before (cf. OverlayFS) |
129 |
|
|
* @return a DirectoryEntry structure to be written into a catalog |
130 |
|
|
*/ |
131 |
|
|
virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent() const = 0; |
132 |
|
|
|
133 |
|
|
inline std::string GetRelativePath() const { |
134 |
|
|
return (relative_parent_path_.empty()) ? |
135 |
|
|
filename_ : |
136 |
|
|
relative_parent_path_ + (filename_.empty() ? "" : ("/" + filename_)); |
137 |
|
|
} |
138 |
|
|
|
139 |
|
|
std::string GetRdOnlyPath() const; |
140 |
|
|
std::string GetUnionPath() const; |
141 |
|
|
std::string GetScratchPath() const; |
142 |
|
|
|
143 |
|
|
void MarkAsWhiteout(const std::string &actual_filename); |
144 |
|
|
void MarkAsOpaqueDirectory(); |
145 |
|
|
|
146 |
|
|
/** |
147 |
|
|
* Union file systems (i.e. OverlayFS) might not properly support hardlinks, |
148 |
|
|
* forcing us to ignore them during publishing. A 'masked hardlink' will be |
149 |
|
|
* treated as a normal file (linkcount == 1). Hence, any created hardlinks |
150 |
|
|
* will be broken up into individual files with differing inodes. |
151 |
|
|
*/ |
152 |
|
|
inline void MaskHardlink() { masked_hardlink_ = true; } |
153 |
|
|
inline bool HasHardlinks() const { |
154 |
|
|
return !masked_hardlink_ && GetUnionLinkcount() > 1; |
155 |
|
|
} |
156 |
|
|
|
157 |
|
|
unsigned int GetRdOnlyLinkcount() const; |
158 |
|
|
uint64_t GetRdOnlyInode() const; |
159 |
|
|
unsigned int GetUnionLinkcount() const; |
160 |
|
|
uint64_t GetUnionInode() const; |
161 |
|
|
uint64_t GetScratchSize() const; |
162 |
|
|
uint64_t GetRdOnlySize() const; |
163 |
|
|
|
164 |
|
|
inline std::string filename() const { return filename_; } |
165 |
|
|
inline std::string relative_parent_path() const { |
166 |
|
|
return relative_parent_path_; |
167 |
|
|
} |
168 |
|
|
|
169 |
|
|
virtual IngestionSource *CreateIngestionSource() const = 0; |
170 |
|
|
virtual void MakePlaceholderDirectory() const = 0; |
171 |
|
|
void SetCatalogMarker() { has_catalog_marker_ = true; } |
172 |
|
|
|
173 |
|
|
bool operator==(const SyncItem &other) const { |
174 |
|
|
return ((relative_parent_path_ == other.relative_parent_path_) && |
175 |
|
|
(filename_ == other.filename_)); |
176 |
|
|
} |
177 |
|
|
|
178 |
|
|
protected: |
179 |
|
|
/** |
180 |
|
|
* create a new SyncItem |
181 |
|
|
* Note: SyncItems cannot be created by any using code. SyncUnion will take |
182 |
|
|
* care of their creating through a factory method to make sure they |
183 |
|
|
* are initialised correctly (whiteout, hardlink handling, ...) |
184 |
|
|
* |
185 |
|
|
* @param dirPath the RELATIVE path to the file |
186 |
|
|
* @param filename the name of the file ;-) |
187 |
|
|
* @param entryType well... |
188 |
|
|
*/ |
189 |
|
|
SyncItem(const std::string &relative_parent_path, |
190 |
|
|
const std::string &filename, |
191 |
|
|
const SyncUnion *union_engine, |
192 |
|
|
const SyncItemType entry_type); |
193 |
|
|
|
194 |
|
|
inline platform_stat64 GetUnionStat() const { |
195 |
|
|
StatUnion(); |
196 |
|
|
return union_stat_.stat; |
197 |
|
|
} |
198 |
|
|
|
199 |
|
|
SyncItemType GetRdOnlyFiletype() const; |
200 |
|
|
SyncItemType GetUnionFiletype() const; |
201 |
|
|
|
202 |
|
|
virtual SyncItemType GetScratchFiletype() const = 0; |
203 |
|
|
|
204 |
|
|
/** |
205 |
|
|
* Checks if the SyncItem _is_ the given file type (file, dir, symlink, ...) |
206 |
|
|
* in the union file system volume. Hence: After the publish operation, the |
207 |
|
|
* file will be this type in CVMFS. |
208 |
|
|
* @param expected_type the file type to be checked against |
209 |
|
|
* @return true if file type matches the expected type |
210 |
|
|
*/ |
211 |
|
|
virtual bool IsType(const SyncItemType expected_type) const = 0; |
212 |
|
|
|
213 |
|
|
/** |
214 |
|
|
* Checks if the SyncItem _was_ the given file type (file, dir, symlink, ...) |
215 |
|
|
* in CVMFS (or the lower layer of the union file system). Hence: Before the |
216 |
|
|
* current transaction the file _was_ this type in CVMFS. |
217 |
|
|
* @param expected_type the file type to be checked against |
218 |
|
|
* @return true if file type was the expected type in CVMFS |
219 |
|
|
*/ |
220 |
|
|
inline bool WasType(const SyncItemType expected_type) const { |
221 |
|
|
if (rdonly_type_ == kItemUnknown) { |
222 |
|
|
rdonly_type_ = GetRdOnlyFiletype(); |
223 |
|
|
} |
224 |
|
|
return rdonly_type_ == expected_type; |
225 |
|
|
} |
226 |
|
|
|
227 |
|
|
/** |
228 |
|
|
* Structure to cache stat calls to the different file locations. |
229 |
|
|
*/ |
230 |
|
|
struct EntryStat { |
231 |
|
|
EntryStat() : obtained(false), error_code(0) { |
232 |
|
|
memset(&stat, 0, sizeof(stat)); |
233 |
|
|
} |
234 |
|
|
|
235 |
|
|
inline SyncItemType GetSyncItemType() const { |
236 |
|
|
assert(obtained); |
237 |
|
|
if (S_ISREG(stat.st_mode)) return kItemFile; |
238 |
|
|
if (S_ISLNK(stat.st_mode)) return kItemSymlink; |
239 |
|
|
if (S_ISDIR(stat.st_mode)) return kItemDir; |
240 |
|
|
if (S_ISFIFO(stat.st_mode)) return kItemFifo; |
241 |
|
|
if (S_ISSOCK(stat.st_mode)) return kItemSocket; |
242 |
|
|
if (S_ISCHR(stat.st_mode)) return kItemCharacterDevice; |
243 |
|
|
if (S_ISBLK(stat.st_mode)) return kItemBlockDevice; |
244 |
|
|
return kItemUnknown; |
245 |
|
|
} |
246 |
|
|
|
247 |
|
|
bool obtained; /**< false at the beginning, true after first stat call */ |
248 |
|
|
int error_code; /**< errno value of the stat call */ |
249 |
|
|
platform_stat64 stat; |
250 |
|
|
}; |
251 |
|
|
|
252 |
|
|
static void StatGeneric(const std::string &path, |
253 |
|
|
EntryStat *info, |
254 |
|
|
const bool refresh); |
255 |
|
|
SyncItemType GetGenericFiletype(const EntryStat &stat) const; |
256 |
|
|
void CheckMarkerFiles(); |
257 |
|
|
|
258 |
|
|
mutable SyncItemType rdonly_type_; |
259 |
|
|
mutable EntryStat scratch_stat_; |
260 |
|
|
|
261 |
|
|
ssize_t graft_size_; |
262 |
|
|
|
263 |
|
|
// The hash of regular file's content |
264 |
|
|
shash::Any content_hash_; |
265 |
|
|
|
266 |
|
|
mutable SyncItemType scratch_type_; |
267 |
|
|
|
268 |
|
|
private: |
269 |
|
|
void CheckCatalogMarker(); |
270 |
|
|
|
271 |
|
|
std::string filename_; |
272 |
|
|
|
273 |
|
|
std::string GetGraftMarkerPath() const; |
274 |
|
|
void CheckGraft(); |
275 |
|
|
|
276 |
|
|
const SyncUnion *union_engine_; /**< this SyncUnion created this object */ |
277 |
|
|
|
278 |
|
|
mutable EntryStat rdonly_stat_; |
279 |
|
|
mutable EntryStat union_stat_; |
280 |
|
|
|
281 |
|
|
bool whiteout_; /**< SyncUnion marked this as whiteout */ |
282 |
|
|
bool opaque_; /**< SyncUnion marked this as opaque dir*/ |
283 |
|
|
bool masked_hardlink_; /**< SyncUnion masked out the linkcount */ |
284 |
|
|
bool has_catalog_marker_; /**< directory containing .cvmfscatalog */ |
285 |
|
|
bool valid_graft_; /**< checksum and size in graft marker */ |
286 |
|
|
bool graft_marker_present_; /**< .cvmfsgraft-$filename exists */ |
287 |
|
|
|
288 |
|
|
bool external_data_; |
289 |
|
|
std::string relative_parent_path_; |
290 |
|
|
|
291 |
|
|
/** |
292 |
|
|
* Chunklist from graft. Not initialized by default to save memory. |
293 |
|
|
*/ |
294 |
|
|
FileChunkList *graft_chunklist_; |
295 |
|
|
|
296 |
|
|
// The compression algorithm for the file |
297 |
|
|
zlib::Algorithms compression_algorithm_; |
298 |
|
|
|
299 |
|
|
// Lazy evaluation and caching of results of file stats |
300 |
|
|
inline void StatRdOnly(const bool refresh = false) const { |
301 |
|
|
StatGeneric(GetRdOnlyPath(), &rdonly_stat_, refresh); |
302 |
|
|
} |
303 |
|
|
inline void StatUnion(const bool refresh = false) const { |
304 |
|
|
StatGeneric(GetUnionPath(), &union_stat_, refresh); |
305 |
|
|
} |
306 |
|
|
virtual void StatScratch(const bool refresh = false) const = 0; |
307 |
|
|
}; |
308 |
|
|
|
309 |
|
|
typedef std::map<std::string, SharedPtr<SyncItem> > SyncItemList; |
310 |
|
|
|
311 |
|
|
class SyncItemNative : public SyncItem { |
312 |
|
|
friend class SyncUnion; |
313 |
|
|
virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent() const; |
314 |
|
|
virtual IngestionSource *CreateIngestionSource() const; |
315 |
|
|
virtual void MakePlaceholderDirectory() const { assert(false); } |
316 |
|
|
virtual SyncItemType GetScratchFiletype() const; |
317 |
|
|
virtual bool IsType(const SyncItemType expected_type) const; |
318 |
|
|
virtual void StatScratch(const bool refresh = false) const { |
319 |
|
|
StatGeneric(GetScratchPath(), &scratch_stat_, refresh); |
320 |
|
|
} |
321 |
|
|
|
322 |
|
|
protected: |
323 |
|
|
SyncItemNative(const std::string &relative_parent_path, |
324 |
|
|
const std::string &filename, const SyncUnion *union_engine, |
325 |
|
|
const SyncItemType entry_type) |
326 |
|
|
: SyncItem(relative_parent_path, filename, union_engine, entry_type) { |
327 |
|
|
CheckMarkerFiles(); |
328 |
|
|
} |
329 |
|
|
}; |
330 |
|
|
|
331 |
|
|
} // namespace publish |
332 |
|
|
|
333 |
|
|
#endif // CVMFS_SYNC_ITEM_H_ |