Directory: | cvmfs/ |
---|---|
File: | cvmfs/sync_item.h |
Date: | 2024-04-28 02:33:07 |
Exec | Total | Coverage | |
---|---|---|---|
Lines: | 24 | 95 | 25.3% |
Branches: | 12 | 79 | 15.2% |
Line | Branch | Exec | Source |
---|---|---|---|
1 | /** | ||
2 | * This file is part of the CernVM File System | ||
3 | */ | ||
4 | |||
5 | #ifndef CVMFS_SYNC_ITEM_H_ | ||
6 | #define CVMFS_SYNC_ITEM_H_ | ||
7 | |||
8 | #include <sys/types.h> | ||
9 | |||
10 | #if !defined(__APPLE__) | ||
11 | #include <sys/sysmacros.h> | ||
12 | #endif // __APPLE__ | ||
13 | |||
14 | #include <cstring> | ||
15 | #include <map> | ||
16 | #include <string> | ||
17 | |||
18 | #include "crypto/hash.h" | ||
19 | #include "directory_entry.h" | ||
20 | #include "duplex_libarchive.h" | ||
21 | #include "file_chunk.h" | ||
22 | #include "util/platform.h" | ||
23 | #include "util/shared_ptr.h" | ||
24 | |||
25 | class IngestionSource; | ||
26 | |||
27 | namespace publish { | ||
28 | |||
29 | enum SyncItemType { | ||
30 | kItemDir, | ||
31 | kItemFile, | ||
32 | kItemSymlink, | ||
33 | kItemCharacterDevice, | ||
34 | kItemBlockDevice, | ||
35 | kItemFifo, | ||
36 | kItemSocket, | ||
37 | kItemNew, | ||
38 | kItemMarker, | ||
39 | kItemUnknown, | ||
40 | }; | ||
41 | |||
42 | class SyncUnion; | ||
43 | /** | ||
44 | * Every directory entry emitted by the FileSystemTraversal is wrapped in a | ||
45 | * SyncItem structure by the factory method SyncUnion::CreateSyncItem(). | ||
46 | * | ||
47 | * Since we are dealing with a union file system setup, this class represents | ||
48 | * potentially three concrete files: | ||
49 | * - <read-only path>/<filename> | cf. rdonly_stat_ | ||
50 | * - <scratch (read-write) branch>/<filename> | cf. scratch_stat_ | ||
51 | * - <union volume path>/<filename> | cf. union_stat_ | ||
52 | * | ||
53 | * This class caches stat calls to the underlying files in different branches of | ||
54 | * the union file system and hides some interpretation details. | ||
55 | */ | ||
56 | class SyncItem { | ||
57 | // only SyncUnion can create SyncItems (see SyncUnion::CreateSyncItem). | ||
58 | // SyncUnionTarball can create SyncItemTar and SyncItemDummyDir. | ||
59 | |||
60 | public: | ||
61 | SyncItem(); | ||
62 | virtual ~SyncItem(); | ||
63 | |||
64 | 33 | inline bool IsDirectory() const { return IsType(kItemDir); } | |
65 | ✗ | inline bool WasDirectory() const { return WasType(kItemDir); } | |
66 | 25 | inline bool IsRegularFile() const { return IsType(kItemFile); } | |
67 | ✗ | inline bool WasRegularFile() const { return WasType(kItemFile); } | |
68 | 1 | inline bool IsSymlink() const { return IsType(kItemSymlink); } | |
69 | ✗ | inline bool WasSymlink() const { return WasType(kItemSymlink); } | |
70 | 19 | inline bool IsNew() const { return WasType(kItemNew); } | |
71 | inline bool IsTouched() const { | ||
72 | return (GetRdOnlyFiletype() == GetUnionFiletype()) && | ||
73 | (GetRdOnlyFiletype() == GetScratchFiletype()) && | ||
74 | (GetUnionFiletype() == GetScratchFiletype()); | ||
75 | } | ||
76 | ✗ | inline bool IsCharacterDevice() const { return IsType(kItemCharacterDevice); } | |
77 | ✗ | inline bool IsBlockDevice() const { return IsType(kItemBlockDevice); } | |
78 | ✗ | inline bool IsFifo() const { return IsType(kItemFifo); } | |
79 | ✗ | inline bool IsSocket() const { return IsType(kItemSocket); } | |
80 | ✗ | inline bool IsGraftMarker() const { return IsType(kItemMarker); } | |
81 | ✗ | inline bool IsExternalData() const { return external_data_; } | |
82 | ✗ | inline bool IsDirectIo() const { return direct_io_; } | |
83 | |||
84 | 10 | inline bool IsWhiteout() const { return whiteout_; } | |
85 | ✗ | inline bool IsCatalogMarker() const { return filename_ == ".cvmfscatalog"; } | |
86 | ✗ | inline bool IsOpaqueDirectory() const { return IsDirectory() && opaque_; } | |
87 | |||
88 | ✗ | inline bool IsSpecialFile() const { | |
89 | ✗ | return IsCharacterDevice() || IsBlockDevice() || IsFifo() || IsSocket(); | |
90 | } | ||
91 | ✗ | inline bool WasSpecialFile() const { | |
92 | ✗ | return WasType(kItemCharacterDevice) || | |
93 | ✗ | WasType(kItemBlockDevice) || | |
94 | ✗ | WasType(kItemFifo) || | |
95 | ✗ | WasType(kItemSocket); | |
96 | } | ||
97 | ✗ | inline bool IsBundleSpec() const { | |
98 | ✗ | return filename_ == ".cvmfsbundles"; | |
99 | } | ||
100 | ✗ | inline bool WasBundleSpec() const { | |
101 | ✗ | return filename_ == ".cvmfsbundles"; | |
102 | } | ||
103 | |||
104 | ✗ | inline unsigned int GetRdevMajor() const { | |
105 | ✗ | assert(IsSpecialFile()); | |
106 | ✗ | StatUnion(true); return major(union_stat_.stat.st_rdev); | |
107 | } | ||
108 | |||
109 | ✗ | inline unsigned int GetRdevMinor() const { | |
110 | ✗ | assert(IsSpecialFile()); | |
111 | ✗ | StatUnion(true); return minor(union_stat_.stat.st_rdev); | |
112 | } | ||
113 | |||
114 | ✗ | bool HasCatalogMarker() const { return has_catalog_marker_; } | |
115 | ✗ | bool HasGraftMarker() const { return graft_marker_present_; } | |
116 | ✗ | bool HasCompressionAlgorithm() const { return has_compression_algorithm_; } | |
117 | ✗ | bool IsValidGraft() const { return valid_graft_; } | |
118 | ✗ | bool IsChunkedGraft() const { return graft_chunklist_; } | |
119 | |||
120 | ✗ | inline const FileChunkList* GetGraftChunks() const {return graft_chunklist_;} | |
121 | ✗ | inline shash::Any GetContentHash() const { return content_hash_; } | |
122 | ✗ | inline void SetContentHash(const shash::Any &hash) { content_hash_ = hash; } | |
123 | inline bool HasContentHash() const { return !content_hash_.IsNull(); } | ||
124 | ✗ | void SetExternalData(bool val) {external_data_ = val;} | |
125 | ✗ | void SetDirectIo(bool val) {direct_io_ = val;} | |
126 | |||
127 | ✗ | inline zlib::Algorithms GetCompressionAlgorithm() const { | |
128 | ✗ | return compression_algorithm_; | |
129 | } | ||
130 | ✗ | inline void SetCompressionAlgorithm(const zlib::Algorithms &alg) { | |
131 | ✗ | compression_algorithm_ = alg; | |
132 | ✗ | has_compression_algorithm_ = true; | |
133 | } | ||
134 | |||
135 | /** | ||
136 | * Generates a DirectoryEntry that can be directly stored into a catalog db. | ||
137 | * Note: this sets the inode fields to kInvalidInode as well as the link | ||
138 | * count to 1 if MaskHardlink() has been called before (cf. OverlayFS) | ||
139 | * @return a DirectoryEntry structure to be written into a catalog | ||
140 | */ | ||
141 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent() const = 0; | ||
142 | |||
143 | 62 | inline std::string GetRelativePath() const { | |
144 | 62 | return (relative_parent_path_.empty()) ? | |
145 | 12 | filename_ : | |
146 |
9/21✓ Branch 0 taken 12 times.
✓ Branch 1 taken 50 times.
✓ Branch 3 taken 12 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 50 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 13 taken 50 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 50 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 50 times.
✓ Branch 19 taken 12 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 62 times.
✗ Branch 24 not taken.
✗ Branch 25 not taken.
✗ Branch 27 not taken.
✗ Branch 28 not taken.
|
124 | relative_parent_path_ + (filename_.empty() ? "" : ("/" + filename_)); |
147 | } | ||
148 | |||
149 | std::string GetRdOnlyPath() const; | ||
150 | std::string GetUnionPath() const; | ||
151 | std::string GetScratchPath() const; | ||
152 | |||
153 | void MarkAsWhiteout(const std::string &actual_filename); | ||
154 | void MarkAsOpaqueDirectory(); | ||
155 | |||
156 | /** | ||
157 | * Union file systems (i.e. OverlayFS) might not properly support hardlinks, | ||
158 | * forcing us to ignore them during publishing. A 'masked hardlink' will be | ||
159 | * treated as a normal file (linkcount == 1). Hence, any created hardlinks | ||
160 | * will be broken up into individual files with differing inodes. | ||
161 | */ | ||
162 | ✗ | inline void MaskHardlink() { masked_hardlink_ = true; } | |
163 | ✗ | inline bool HasHardlinks() const { | |
164 | ✗ | return !masked_hardlink_ && GetUnionLinkcount() > 1; | |
165 | } | ||
166 | |||
167 | unsigned int GetRdOnlyLinkcount() const; | ||
168 | uint64_t GetRdOnlyInode() const; | ||
169 | unsigned int GetUnionLinkcount() const; | ||
170 | uint64_t GetUnionInode() const; | ||
171 | uint64_t GetScratchSize() const; | ||
172 | uint64_t GetRdOnlySize() const; | ||
173 | |||
174 | 30 | inline std::string filename() const { return filename_; } | |
175 | ✗ | inline std::string relative_parent_path() const { | |
176 | ✗ | return relative_parent_path_; | |
177 | } | ||
178 | |||
179 | virtual IngestionSource *CreateIngestionSource() const = 0; | ||
180 | virtual void MakePlaceholderDirectory() const = 0; | ||
181 | ✗ | void SetCatalogMarker() { has_catalog_marker_ = true; } | |
182 | |||
183 | ✗ | bool operator==(const SyncItem &other) const { | |
184 | ✗ | return ((relative_parent_path_ == other.relative_parent_path_) && | |
185 | ✗ | (filename_ == other.filename_)); | |
186 | } | ||
187 | |||
188 | protected: | ||
189 | /** | ||
190 | * create a new SyncItem | ||
191 | * Note: SyncItems cannot be created by any using code. SyncUnion will take | ||
192 | * care of their creating through a factory method to make sure they | ||
193 | * are initialised correctly (whiteout, hardlink handling, ...) | ||
194 | * | ||
195 | * @param dirPath the RELATIVE path to the file | ||
196 | * @param filename the name of the file ;-) | ||
197 | * @param entryType well... | ||
198 | */ | ||
199 | SyncItem(const std::string &relative_parent_path, | ||
200 | const std::string &filename, | ||
201 | const SyncUnion *union_engine, | ||
202 | const SyncItemType entry_type); | ||
203 | |||
204 | ✗ | inline platform_stat64 GetUnionStat() const { | |
205 | ✗ | StatUnion(); | |
206 | ✗ | return union_stat_.stat; | |
207 | } | ||
208 | |||
209 | SyncItemType GetRdOnlyFiletype() const; | ||
210 | SyncItemType GetUnionFiletype() const; | ||
211 | |||
212 | virtual SyncItemType GetScratchFiletype() const = 0; | ||
213 | |||
214 | /** | ||
215 | * Checks if the SyncItem _is_ the given file type (file, dir, symlink, ...) | ||
216 | * in the union file system volume. Hence: After the publish operation, the | ||
217 | * file will be this type in CVMFS. | ||
218 | * @param expected_type the file type to be checked against | ||
219 | * @return true if file type matches the expected type | ||
220 | */ | ||
221 | virtual bool IsType(const SyncItemType expected_type) const = 0; | ||
222 | |||
223 | /** | ||
224 | * Checks if the SyncItem _was_ the given file type (file, dir, symlink, ...) | ||
225 | * in CVMFS (or the lower layer of the union file system). Hence: Before the | ||
226 | * current transaction the file _was_ this type in CVMFS. | ||
227 | * @param expected_type the file type to be checked against | ||
228 | * @return true if file type was the expected type in CVMFS | ||
229 | */ | ||
230 | 19 | inline bool WasType(const SyncItemType expected_type) const { | |
231 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 2 times.
|
19 | if (rdonly_type_ == kItemUnknown) { |
232 | 17 | rdonly_type_ = GetRdOnlyFiletype(); | |
233 | } | ||
234 | 19 | return rdonly_type_ == expected_type; | |
235 | } | ||
236 | |||
237 | /** | ||
238 | * Structure to cache stat calls to the different file locations. | ||
239 | */ | ||
240 | struct EntryStat { | ||
241 | 62 | EntryStat() : obtained(false), error_code(0) { | |
242 | 62 | memset(&stat, 0, sizeof(stat)); | |
243 | 62 | } | |
244 | |||
245 | ✗ | inline SyncItemType GetSyncItemType() const { | |
246 | ✗ | assert(obtained); | |
247 | ✗ | if (S_ISREG(stat.st_mode)) return kItemFile; | |
248 | ✗ | if (S_ISLNK(stat.st_mode)) return kItemSymlink; | |
249 | ✗ | if (S_ISDIR(stat.st_mode)) return kItemDir; | |
250 | ✗ | if (S_ISFIFO(stat.st_mode)) return kItemFifo; | |
251 | ✗ | if (S_ISSOCK(stat.st_mode)) return kItemSocket; | |
252 | ✗ | if (S_ISCHR(stat.st_mode)) return kItemCharacterDevice; | |
253 | ✗ | if (S_ISBLK(stat.st_mode)) return kItemBlockDevice; | |
254 | ✗ | return kItemUnknown; | |
255 | } | ||
256 | |||
257 | bool obtained; /**< false at the beginning, true after first stat call */ | ||
258 | int error_code; /**< errno value of the stat call */ | ||
259 | platform_stat64 stat; | ||
260 | }; | ||
261 | |||
262 | static void StatGeneric(const std::string &path, | ||
263 | EntryStat *info, | ||
264 | const bool refresh); | ||
265 | SyncItemType GetGenericFiletype(const EntryStat &stat) const; | ||
266 | void CheckMarkerFiles(); | ||
267 | |||
268 | mutable SyncItemType rdonly_type_; | ||
269 | mutable EntryStat scratch_stat_; | ||
270 | |||
271 | ssize_t graft_size_; | ||
272 | |||
273 | // The hash of regular file's content | ||
274 | shash::Any content_hash_; | ||
275 | |||
276 | mutable SyncItemType scratch_type_; | ||
277 | |||
278 | private: | ||
279 | void CheckCatalogMarker(); | ||
280 | |||
281 | std::string filename_; | ||
282 | |||
283 | std::string GetGraftMarkerPath() const; | ||
284 | void CheckGraft(); | ||
285 | |||
286 | const SyncUnion *union_engine_; /**< this SyncUnion created this object */ | ||
287 | |||
288 | mutable EntryStat rdonly_stat_; | ||
289 | mutable EntryStat union_stat_; | ||
290 | |||
291 | bool whiteout_; /**< SyncUnion marked this as whiteout */ | ||
292 | bool opaque_; /**< SyncUnion marked this as opaque dir*/ | ||
293 | bool masked_hardlink_; /**< SyncUnion masked out the linkcount */ | ||
294 | bool has_catalog_marker_; /**< directory containing .cvmfscatalog */ | ||
295 | bool valid_graft_; /**< checksum and size in graft marker */ | ||
296 | bool graft_marker_present_; /**< .cvmfsgraft-$filename exists */ | ||
297 | |||
298 | bool external_data_; | ||
299 | bool direct_io_; | ||
300 | std::string relative_parent_path_; | ||
301 | |||
302 | /** | ||
303 | * Chunklist from graft. Not initialized by default to save memory. | ||
304 | */ | ||
305 | FileChunkList *graft_chunklist_; | ||
306 | |||
307 | // The compression algorithm for the file | ||
308 | zlib::Algorithms compression_algorithm_; | ||
309 | // The compression algorithm has been set explicitly | ||
310 | bool has_compression_algorithm_; | ||
311 | |||
312 | // Lazy evaluation and caching of results of file stats | ||
313 | 17 | inline void StatRdOnly(const bool refresh = false) const { | |
314 | 17 | StatGeneric(GetRdOnlyPath(), &rdonly_stat_, refresh); | |
315 | 17 | } | |
316 | ✗ | inline void StatUnion(const bool refresh = false) const { | |
317 | ✗ | StatGeneric(GetUnionPath(), &union_stat_, refresh); | |
318 | } | ||
319 | virtual void StatScratch(const bool refresh) const = 0; | ||
320 | }; | ||
321 | |||
322 | typedef std::map<std::string, SharedPtr<SyncItem> > SyncItemList; | ||
323 | |||
324 | class SyncItemNative : public SyncItem { | ||
325 | friend class SyncUnion; | ||
326 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent() const; | ||
327 | virtual IngestionSource *CreateIngestionSource() const; | ||
328 | ✗ | virtual void MakePlaceholderDirectory() const { assert(false); } | |
329 | virtual SyncItemType GetScratchFiletype() const; | ||
330 | virtual bool IsType(const SyncItemType expected_type) const; | ||
331 | ✗ | virtual void StatScratch(const bool refresh) const { | |
332 | ✗ | StatGeneric(GetScratchPath(), &scratch_stat_, refresh); | |
333 | } | ||
334 | |||
335 | protected: | ||
336 | 5 | SyncItemNative(const std::string &relative_parent_path, | |
337 | const std::string &filename, const SyncUnion *union_engine, | ||
338 | const SyncItemType entry_type) | ||
339 | 5 | : SyncItem(relative_parent_path, filename, union_engine, entry_type) { | |
340 |
1/2✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
|
5 | CheckMarkerFiles(); |
341 | 5 | } | |
342 | }; | ||
343 | |||
344 | } // namespace publish | ||
345 | |||
346 | #endif // CVMFS_SYNC_ITEM_H_ | ||
347 |