| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/sync_item.h |
| Date: | 2025-04-20 02:34:28 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 24 | 95 | 25.3% |
| Branches: | 12 | 79 | 15.2% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System | ||
| 3 | */ | ||
| 4 | |||
| 5 | #ifndef CVMFS_SYNC_ITEM_H_ | ||
| 6 | #define CVMFS_SYNC_ITEM_H_ | ||
| 7 | |||
| 8 | #include <sys/types.h> | ||
| 9 | |||
| 10 | #if !defined(__APPLE__) | ||
| 11 | #include <sys/sysmacros.h> | ||
| 12 | #endif // __APPLE__ | ||
| 13 | |||
| 14 | #include <cstring> | ||
| 15 | #include <map> | ||
| 16 | #include <string> | ||
| 17 | |||
| 18 | #include "crypto/hash.h" | ||
| 19 | #include "directory_entry.h" | ||
| 20 | #include "duplex_libarchive.h" | ||
| 21 | #include "file_chunk.h" | ||
| 22 | #include "util/platform.h" | ||
| 23 | #include "util/shared_ptr.h" | ||
| 24 | |||
| 25 | class IngestionSource; | ||
| 26 | |||
| 27 | namespace publish { | ||
| 28 | |||
| 29 | enum SyncItemType { | ||
| 30 | kItemDir, | ||
| 31 | kItemFile, | ||
| 32 | kItemSymlink, | ||
| 33 | kItemCharacterDevice, | ||
| 34 | kItemBlockDevice, | ||
| 35 | kItemFifo, | ||
| 36 | kItemSocket, | ||
| 37 | kItemNew, | ||
| 38 | kItemMarker, | ||
| 39 | kItemUnknown, | ||
| 40 | }; | ||
| 41 | |||
| 42 | class SyncUnion; | ||
| 43 | /** | ||
| 44 | * Every directory entry emitted by the FileSystemTraversal is wrapped in a | ||
| 45 | * SyncItem structure by the factory method SyncUnion::CreateSyncItem(). | ||
| 46 | * | ||
| 47 | * Since we are dealing with a union file system setup, this class represents | ||
| 48 | * potentially three concrete files: | ||
| 49 | * - <read-only path>/<filename> | cf. rdonly_stat_ | ||
| 50 | * - <scratch (read-write) branch>/<filename> | cf. scratch_stat_ | ||
| 51 | * - <union volume path>/<filename> | cf. union_stat_ | ||
| 52 | * | ||
| 53 | * This class caches stat calls to the underlying files in different branches of | ||
| 54 | * the union file system and hides some interpretation details. | ||
| 55 | */ | ||
| 56 | class SyncItem { | ||
| 57 | // only SyncUnion can create SyncItems (see SyncUnion::CreateSyncItem). | ||
| 58 | // SyncUnionTarball can create SyncItemTar and SyncItemDummyDir. | ||
| 59 | |||
| 60 | public: | ||
| 61 | SyncItem(); | ||
| 62 | virtual ~SyncItem(); | ||
| 63 | |||
| 64 | 33 | inline bool IsDirectory() const { return IsType(kItemDir); } | |
| 65 | ✗ | inline bool WasDirectory() const { return WasType(kItemDir); } | |
| 66 | 25 | inline bool IsRegularFile() const { return IsType(kItemFile); } | |
| 67 | ✗ | inline bool WasRegularFile() const { return WasType(kItemFile); } | |
| 68 | 1 | inline bool IsSymlink() const { return IsType(kItemSymlink); } | |
| 69 | ✗ | inline bool WasSymlink() const { return WasType(kItemSymlink); } | |
| 70 | 19 | inline bool IsNew() const { return WasType(kItemNew); } | |
| 71 | inline bool IsTouched() const { | ||
| 72 | return (GetRdOnlyFiletype() == GetUnionFiletype()) && | ||
| 73 | (GetRdOnlyFiletype() == GetScratchFiletype()) && | ||
| 74 | (GetUnionFiletype() == GetScratchFiletype()); | ||
| 75 | } | ||
| 76 | ✗ | inline bool IsCharacterDevice() const { return IsType(kItemCharacterDevice); } | |
| 77 | ✗ | inline bool IsBlockDevice() const { return IsType(kItemBlockDevice); } | |
| 78 | ✗ | inline bool IsFifo() const { return IsType(kItemFifo); } | |
| 79 | ✗ | inline bool IsSocket() const { return IsType(kItemSocket); } | |
| 80 | ✗ | inline bool IsGraftMarker() const { return IsType(kItemMarker); } | |
| 81 | ✗ | inline bool IsExternalData() const { return external_data_; } | |
| 82 | ✗ | inline bool IsDirectIo() const { return direct_io_; } | |
| 83 | |||
| 84 | 10 | inline bool IsWhiteout() const { return whiteout_; } | |
| 85 | ✗ | inline bool IsCatalogMarker() const { return filename_ == ".cvmfscatalog"; } | |
| 86 | ✗ | inline bool IsOpaqueDirectory() const { return IsDirectory() && opaque_; } | |
| 87 | |||
| 88 | ✗ | inline bool IsSpecialFile() const { | |
| 89 | ✗ | return IsCharacterDevice() || IsBlockDevice() || IsFifo() || IsSocket(); | |
| 90 | } | ||
| 91 | ✗ | inline bool WasSpecialFile() const { | |
| 92 | ✗ | return WasType(kItemCharacterDevice) || | |
| 93 | ✗ | WasType(kItemBlockDevice) || | |
| 94 | ✗ | WasType(kItemFifo) || | |
| 95 | ✗ | WasType(kItemSocket); | |
| 96 | } | ||
| 97 | ✗ | inline bool IsBundleSpec() const { | |
| 98 | ✗ | return filename_ == ".cvmfsbundles"; | |
| 99 | } | ||
| 100 | ✗ | inline bool WasBundleSpec() const { | |
| 101 | ✗ | return filename_ == ".cvmfsbundles"; | |
| 102 | } | ||
| 103 | |||
| 104 | ✗ | inline unsigned int GetRdevMajor() const { | |
| 105 | ✗ | assert(IsSpecialFile()); | |
| 106 | ✗ | StatUnion(true); return major(union_stat_.stat.st_rdev); | |
| 107 | } | ||
| 108 | |||
| 109 | ✗ | inline unsigned int GetRdevMinor() const { | |
| 110 | ✗ | assert(IsSpecialFile()); | |
| 111 | ✗ | StatUnion(true); return minor(union_stat_.stat.st_rdev); | |
| 112 | } | ||
| 113 | |||
| 114 | ✗ | bool HasCatalogMarker() const { return has_catalog_marker_; } | |
| 115 | ✗ | bool HasGraftMarker() const { return graft_marker_present_; } | |
| 116 | ✗ | bool HasCompressionAlgorithm() const { return has_compression_algorithm_; } | |
| 117 | ✗ | bool IsValidGraft() const { return valid_graft_; } | |
| 118 | ✗ | bool IsChunkedGraft() const { return graft_chunklist_; } | |
| 119 | |||
| 120 | ✗ | inline const FileChunkList* GetGraftChunks() const {return graft_chunklist_;} | |
| 121 | ✗ | inline shash::Any GetContentHash() const { return content_hash_; } | |
| 122 | ✗ | inline void SetContentHash(const shash::Any &hash) { content_hash_ = hash; } | |
| 123 | inline bool HasContentHash() const { return !content_hash_.IsNull(); } | ||
| 124 | ✗ | void SetExternalData(bool val) {external_data_ = val;} | |
| 125 | ✗ | void SetDirectIo(bool val) {direct_io_ = val;} | |
| 126 | |||
| 127 | ✗ | inline zlib::Algorithms GetCompressionAlgorithm() const { | |
| 128 | ✗ | return compression_algorithm_; | |
| 129 | } | ||
| 130 | ✗ | inline void SetCompressionAlgorithm(const zlib::Algorithms &alg) { | |
| 131 | ✗ | compression_algorithm_ = alg; | |
| 132 | ✗ | has_compression_algorithm_ = true; | |
| 133 | } | ||
| 134 | |||
| 135 | /** | ||
| 136 | * Generates a DirectoryEntry that can be directly stored into a catalog db. | ||
| 137 | * Note: this sets the inode fields to kInvalidInode as well as the link | ||
| 138 | * count to 1 if MaskHardlink() has been called before (cf. OverlayFS) | ||
| 139 | * | ||
| 140 | * If nanosecond timestamps are off, the directory entry will have a | ||
| 141 | * default initialized, negative nanosecond timestamp and as a result | ||
| 142 | * the corresponding field in the catalog table will be NULL. | ||
| 143 | * | ||
| 144 | * @return a DirectoryEntry structure to be written into a catalog | ||
| 145 | */ | ||
| 146 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent( | ||
| 147 | bool enable_mtime_ns) const = 0; | ||
| 148 | |||
| 149 | 62 | inline std::string GetRelativePath() const { | |
| 150 | 62 | return (relative_parent_path_.empty()) ? | |
| 151 | 12 | filename_ : | |
| 152 |
9/21✓ Branch 0 taken 12 times.
✓ Branch 1 taken 50 times.
✓ Branch 3 taken 12 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 50 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 13 taken 50 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 50 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 50 times.
✓ Branch 19 taken 12 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 62 times.
✗ Branch 24 not taken.
✗ Branch 25 not taken.
✗ Branch 27 not taken.
✗ Branch 28 not taken.
|
124 | relative_parent_path_ + (filename_.empty() ? "" : ("/" + filename_)); |
| 153 | } | ||
| 154 | |||
| 155 | std::string GetRdOnlyPath() const; | ||
| 156 | std::string GetUnionPath() const; | ||
| 157 | std::string GetScratchPath() const; | ||
| 158 | |||
| 159 | void MarkAsWhiteout(const std::string &actual_filename); | ||
| 160 | void MarkAsOpaqueDirectory(); | ||
| 161 | |||
| 162 | /** | ||
| 163 | * Union file systems (i.e. OverlayFS) might not properly support hardlinks, | ||
| 164 | * forcing us to ignore them during publishing. A 'masked hardlink' will be | ||
| 165 | * treated as a normal file (linkcount == 1). Hence, any created hardlinks | ||
| 166 | * will be broken up into individual files with differing inodes. | ||
| 167 | */ | ||
| 168 | ✗ | inline void MaskHardlink() { masked_hardlink_ = true; } | |
| 169 | ✗ | inline bool HasHardlinks() const { | |
| 170 | ✗ | return !masked_hardlink_ && GetUnionLinkcount() > 1; | |
| 171 | } | ||
| 172 | |||
| 173 | unsigned int GetRdOnlyLinkcount() const; | ||
| 174 | uint64_t GetRdOnlyInode() const; | ||
| 175 | unsigned int GetUnionLinkcount() const; | ||
| 176 | uint64_t GetUnionInode() const; | ||
| 177 | uint64_t GetScratchSize() const; | ||
| 178 | uint64_t GetRdOnlySize() const; | ||
| 179 | |||
| 180 | 30 | inline std::string filename() const { return filename_; } | |
| 181 | ✗ | inline std::string relative_parent_path() const { | |
| 182 | ✗ | return relative_parent_path_; | |
| 183 | } | ||
| 184 | |||
| 185 | virtual IngestionSource *CreateIngestionSource() const = 0; | ||
| 186 | virtual void MakePlaceholderDirectory() const = 0; | ||
| 187 | ✗ | void SetCatalogMarker() { has_catalog_marker_ = true; } | |
| 188 | |||
| 189 | ✗ | bool operator==(const SyncItem &other) const { | |
| 190 | ✗ | return ((relative_parent_path_ == other.relative_parent_path_) && | |
| 191 | ✗ | (filename_ == other.filename_)); | |
| 192 | } | ||
| 193 | |||
| 194 | protected: | ||
| 195 | /** | ||
| 196 | * create a new SyncItem | ||
| 197 | * Note: SyncItems cannot be created by any using code. SyncUnion will take | ||
| 198 | * care of their creating through a factory method to make sure they | ||
| 199 | * are initialised correctly (whiteout, hardlink handling, ...) | ||
| 200 | * | ||
| 201 | * @param dirPath the RELATIVE path to the file | ||
| 202 | * @param filename the name of the file ;-) | ||
| 203 | * @param entryType well... | ||
| 204 | */ | ||
| 205 | SyncItem(const std::string &relative_parent_path, | ||
| 206 | const std::string &filename, | ||
| 207 | const SyncUnion *union_engine, | ||
| 208 | const SyncItemType entry_type); | ||
| 209 | |||
| 210 | ✗ | inline platform_stat64 GetUnionStat() const { | |
| 211 | ✗ | StatUnion(); | |
| 212 | ✗ | return union_stat_.stat; | |
| 213 | } | ||
| 214 | |||
| 215 | SyncItemType GetRdOnlyFiletype() const; | ||
| 216 | SyncItemType GetUnionFiletype() const; | ||
| 217 | |||
| 218 | virtual SyncItemType GetScratchFiletype() const = 0; | ||
| 219 | |||
| 220 | /** | ||
| 221 | * Checks if the SyncItem _is_ the given file type (file, dir, symlink, ...) | ||
| 222 | * in the union file system volume. Hence: After the publish operation, the | ||
| 223 | * file will be this type in CVMFS. | ||
| 224 | * @param expected_type the file type to be checked against | ||
| 225 | * @return true if file type matches the expected type | ||
| 226 | */ | ||
| 227 | virtual bool IsType(const SyncItemType expected_type) const = 0; | ||
| 228 | |||
| 229 | /** | ||
| 230 | * Checks if the SyncItem _was_ the given file type (file, dir, symlink, ...) | ||
| 231 | * in CVMFS (or the lower layer of the union file system). Hence: Before the | ||
| 232 | * current transaction the file _was_ this type in CVMFS. | ||
| 233 | * @param expected_type the file type to be checked against | ||
| 234 | * @return true if file type was the expected type in CVMFS | ||
| 235 | */ | ||
| 236 | 19 | inline bool WasType(const SyncItemType expected_type) const { | |
| 237 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 2 times.
|
19 | if (rdonly_type_ == kItemUnknown) { |
| 238 | 17 | rdonly_type_ = GetRdOnlyFiletype(); | |
| 239 | } | ||
| 240 | 19 | return rdonly_type_ == expected_type; | |
| 241 | } | ||
| 242 | |||
| 243 | /** | ||
| 244 | * Structure to cache stat calls to the different file locations. | ||
| 245 | */ | ||
| 246 | struct EntryStat { | ||
| 247 | 62 | EntryStat() : obtained(false), error_code(0) { | |
| 248 | 62 | memset(&stat, 0, sizeof(stat)); | |
| 249 | 62 | } | |
| 250 | |||
| 251 | ✗ | inline SyncItemType GetSyncItemType() const { | |
| 252 | ✗ | assert(obtained); | |
| 253 | ✗ | if (S_ISREG(stat.st_mode)) return kItemFile; | |
| 254 | ✗ | if (S_ISLNK(stat.st_mode)) return kItemSymlink; | |
| 255 | ✗ | if (S_ISDIR(stat.st_mode)) return kItemDir; | |
| 256 | ✗ | if (S_ISFIFO(stat.st_mode)) return kItemFifo; | |
| 257 | ✗ | if (S_ISSOCK(stat.st_mode)) return kItemSocket; | |
| 258 | ✗ | if (S_ISCHR(stat.st_mode)) return kItemCharacterDevice; | |
| 259 | ✗ | if (S_ISBLK(stat.st_mode)) return kItemBlockDevice; | |
| 260 | ✗ | return kItemUnknown; | |
| 261 | } | ||
| 262 | |||
| 263 | bool obtained; /**< false at the beginning, true after first stat call */ | ||
| 264 | int error_code; /**< errno value of the stat call */ | ||
| 265 | platform_stat64 stat; | ||
| 266 | }; | ||
| 267 | |||
| 268 | static void StatGeneric(const std::string &path, | ||
| 269 | EntryStat *info, | ||
| 270 | const bool refresh); | ||
| 271 | SyncItemType GetGenericFiletype(const EntryStat &stat) const; | ||
| 272 | void CheckMarkerFiles(); | ||
| 273 | |||
| 274 | mutable SyncItemType rdonly_type_; | ||
| 275 | mutable EntryStat scratch_stat_; | ||
| 276 | |||
| 277 | ssize_t graft_size_; | ||
| 278 | |||
| 279 | // The hash of regular file's content | ||
| 280 | shash::Any content_hash_; | ||
| 281 | |||
| 282 | mutable SyncItemType scratch_type_; | ||
| 283 | |||
| 284 | private: | ||
| 285 | void CheckCatalogMarker(); | ||
| 286 | |||
| 287 | std::string filename_; | ||
| 288 | |||
| 289 | std::string GetGraftMarkerPath() const; | ||
| 290 | void CheckGraft(); | ||
| 291 | |||
| 292 | const SyncUnion *union_engine_; /**< this SyncUnion created this object */ | ||
| 293 | |||
| 294 | mutable EntryStat rdonly_stat_; | ||
| 295 | mutable EntryStat union_stat_; | ||
| 296 | |||
| 297 | bool whiteout_; /**< SyncUnion marked this as whiteout */ | ||
| 298 | bool opaque_; /**< SyncUnion marked this as opaque dir*/ | ||
| 299 | bool masked_hardlink_; /**< SyncUnion masked out the linkcount */ | ||
| 300 | bool has_catalog_marker_; /**< directory containing .cvmfscatalog */ | ||
| 301 | bool valid_graft_; /**< checksum and size in graft marker */ | ||
| 302 | bool graft_marker_present_; /**< .cvmfsgraft-$filename exists */ | ||
| 303 | |||
| 304 | bool external_data_; | ||
| 305 | bool direct_io_; | ||
| 306 | std::string relative_parent_path_; | ||
| 307 | |||
| 308 | /** | ||
| 309 | * Chunklist from graft. Not initialized by default to save memory. | ||
| 310 | */ | ||
| 311 | FileChunkList *graft_chunklist_; | ||
| 312 | |||
| 313 | // The compression algorithm for the file | ||
| 314 | zlib::Algorithms compression_algorithm_; | ||
| 315 | // The compression algorithm has been set explicitly | ||
| 316 | bool has_compression_algorithm_; | ||
| 317 | |||
| 318 | // Lazy evaluation and caching of results of file stats | ||
| 319 | 17 | inline void StatRdOnly(const bool refresh = false) const { | |
| 320 | 17 | StatGeneric(GetRdOnlyPath(), &rdonly_stat_, refresh); | |
| 321 | 17 | } | |
| 322 | ✗ | inline void StatUnion(const bool refresh = false) const { | |
| 323 | ✗ | StatGeneric(GetUnionPath(), &union_stat_, refresh); | |
| 324 | } | ||
| 325 | virtual void StatScratch(const bool refresh) const = 0; | ||
| 326 | }; | ||
| 327 | |||
| 328 | typedef std::map<std::string, SharedPtr<SyncItem> > SyncItemList; | ||
| 329 | |||
| 330 | class SyncItemNative : public SyncItem { | ||
| 331 | friend class SyncUnion; | ||
| 332 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent( | ||
| 333 | bool enable_mtime_ns) const; | ||
| 334 | virtual IngestionSource *CreateIngestionSource() const; | ||
| 335 | ✗ | virtual void MakePlaceholderDirectory() const { assert(false); } | |
| 336 | virtual SyncItemType GetScratchFiletype() const; | ||
| 337 | virtual bool IsType(const SyncItemType expected_type) const; | ||
| 338 | ✗ | virtual void StatScratch(const bool refresh) const { | |
| 339 | ✗ | StatGeneric(GetScratchPath(), &scratch_stat_, refresh); | |
| 340 | } | ||
| 341 | |||
| 342 | protected: | ||
| 343 | 5 | SyncItemNative(const std::string &relative_parent_path, | |
| 344 | const std::string &filename, const SyncUnion *union_engine, | ||
| 345 | const SyncItemType entry_type) | ||
| 346 | 5 | : SyncItem(relative_parent_path, filename, union_engine, entry_type) { | |
| 347 |
1/2✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
|
5 | CheckMarkerFiles(); |
| 348 | 5 | } | |
| 349 | }; | ||
| 350 | |||
| 351 | } // namespace publish | ||
| 352 | |||
| 353 | #endif // CVMFS_SYNC_ITEM_H_ | ||
| 354 |