| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/sync_item.h |
| Date: | 2025-07-06 02:35:01 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 25 | 102 | 24.5% |
| Branches: | 12 | 77 | 15.6% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System | ||
| 3 | */ | ||
| 4 | |||
| 5 | #ifndef CVMFS_SYNC_ITEM_H_ | ||
| 6 | #define CVMFS_SYNC_ITEM_H_ | ||
| 7 | |||
| 8 | #include <sys/types.h> | ||
| 9 | |||
| 10 | #if !defined(__APPLE__) | ||
| 11 | #include <sys/sysmacros.h> | ||
| 12 | #endif // __APPLE__ | ||
| 13 | |||
| 14 | #include <cstring> | ||
| 15 | #include <map> | ||
| 16 | #include <string> | ||
| 17 | |||
| 18 | #include "crypto/hash.h" | ||
| 19 | #include "directory_entry.h" | ||
| 20 | #include "duplex_libarchive.h" | ||
| 21 | #include "file_chunk.h" | ||
| 22 | #include "util/platform.h" | ||
| 23 | #include "util/shared_ptr.h" | ||
| 24 | |||
| 25 | class IngestionSource; | ||
| 26 | |||
| 27 | namespace publish { | ||
| 28 | |||
| 29 | enum SyncItemType { | ||
| 30 | kItemDir, | ||
| 31 | kItemFile, | ||
| 32 | kItemSymlink, | ||
| 33 | kItemCharacterDevice, | ||
| 34 | kItemBlockDevice, | ||
| 35 | kItemFifo, | ||
| 36 | kItemSocket, | ||
| 37 | kItemNew, | ||
| 38 | kItemMarker, | ||
| 39 | kItemUnknown, | ||
| 40 | }; | ||
| 41 | |||
| 42 | class SyncUnion; | ||
| 43 | /** | ||
| 44 | * Every directory entry emitted by the FileSystemTraversal is wrapped in a | ||
| 45 | * SyncItem structure by the factory method SyncUnion::CreateSyncItem(). | ||
| 46 | * | ||
| 47 | * Since we are dealing with a union file system setup, this class represents | ||
| 48 | * potentially three concrete files: | ||
| 49 | * - <read-only path>/<filename> | cf. rdonly_stat_ | ||
| 50 | * - <scratch (read-write) branch>/<filename> | cf. scratch_stat_ | ||
| 51 | * - <union volume path>/<filename> | cf. union_stat_ | ||
| 52 | * | ||
| 53 | * This class caches stat calls to the underlying files in different branches of | ||
| 54 | * the union file system and hides some interpretation details. | ||
| 55 | */ | ||
| 56 | class SyncItem { | ||
| 57 | // only SyncUnion can create SyncItems (see SyncUnion::CreateSyncItem). | ||
| 58 | // SyncUnionTarball can create SyncItemTar and SyncItemDummyDir. | ||
| 59 | |||
| 60 | public: | ||
| 61 | SyncItem(); | ||
| 62 | virtual ~SyncItem(); | ||
| 63 | |||
| 64 | 825 | inline bool IsDirectory() const { return IsType(kItemDir); } | |
| 65 | ✗ | inline bool WasDirectory() const { return WasType(kItemDir); } | |
| 66 | 625 | inline bool IsRegularFile() const { return IsType(kItemFile); } | |
| 67 | ✗ | inline bool WasRegularFile() const { return WasType(kItemFile); } | |
| 68 | 25 | inline bool IsSymlink() const { return IsType(kItemSymlink); } | |
| 69 | ✗ | inline bool WasSymlink() const { return WasType(kItemSymlink); } | |
| 70 | 475 | inline bool IsNew() const { return WasType(kItemNew); } | |
| 71 | inline bool IsTouched() const { | ||
| 72 | return (GetRdOnlyFiletype() == GetUnionFiletype()) | ||
| 73 | && (GetRdOnlyFiletype() == GetScratchFiletype()) | ||
| 74 | && (GetUnionFiletype() == GetScratchFiletype()); | ||
| 75 | } | ||
| 76 | ✗ | inline bool IsCharacterDevice() const { return IsType(kItemCharacterDevice); } | |
| 77 | ✗ | inline bool IsBlockDevice() const { return IsType(kItemBlockDevice); } | |
| 78 | ✗ | inline bool IsFifo() const { return IsType(kItemFifo); } | |
| 79 | ✗ | inline bool IsSocket() const { return IsType(kItemSocket); } | |
| 80 | ✗ | inline bool IsGraftMarker() const { return IsType(kItemMarker); } | |
| 81 | ✗ | inline bool IsExternalData() const { return external_data_; } | |
| 82 | ✗ | inline bool IsDirectIo() const { return direct_io_; } | |
| 83 | |||
| 84 | 250 | inline bool IsWhiteout() const { return whiteout_; } | |
| 85 | ✗ | inline bool IsCatalogMarker() const { return filename_ == ".cvmfscatalog"; } | |
| 86 | ✗ | inline bool IsOpaqueDirectory() const { return IsDirectory() && opaque_; } | |
| 87 | |||
| 88 | ✗ | inline bool IsSpecialFile() const { | |
| 89 | ✗ | return IsCharacterDevice() || IsBlockDevice() || IsFifo() || IsSocket(); | |
| 90 | } | ||
| 91 | ✗ | inline bool WasSpecialFile() const { | |
| 92 | ✗ | return WasType(kItemCharacterDevice) || WasType(kItemBlockDevice) | |
| 93 | ✗ | || WasType(kItemFifo) || WasType(kItemSocket); | |
| 94 | } | ||
| 95 | ✗ | inline bool IsBundleSpec() const { return filename_ == ".cvmfsbundles"; } | |
| 96 | ✗ | inline bool WasBundleSpec() const { return filename_ == ".cvmfsbundles"; } | |
| 97 | |||
| 98 | ✗ | inline unsigned int GetRdevMajor() const { | |
| 99 | ✗ | assert(IsSpecialFile()); | |
| 100 | ✗ | StatUnion(true); | |
| 101 | ✗ | return major(union_stat_.stat.st_rdev); | |
| 102 | } | ||
| 103 | |||
| 104 | ✗ | inline unsigned int GetRdevMinor() const { | |
| 105 | ✗ | assert(IsSpecialFile()); | |
| 106 | ✗ | StatUnion(true); | |
| 107 | ✗ | return minor(union_stat_.stat.st_rdev); | |
| 108 | } | ||
| 109 | |||
| 110 | ✗ | bool HasCatalogMarker() const { return has_catalog_marker_; } | |
| 111 | ✗ | bool HasGraftMarker() const { return graft_marker_present_; } | |
| 112 | ✗ | bool HasCompressionAlgorithm() const { return has_compression_algorithm_; } | |
| 113 | ✗ | bool IsValidGraft() const { return valid_graft_; } | |
| 114 | ✗ | bool IsChunkedGraft() const { return graft_chunklist_; } | |
| 115 | |||
| 116 | ✗ | inline const FileChunkList *GetGraftChunks() const { | |
| 117 | ✗ | return graft_chunklist_; | |
| 118 | } | ||
| 119 | ✗ | inline shash::Any GetContentHash() const { return content_hash_; } | |
| 120 | ✗ | inline void SetContentHash(const shash::Any &hash) { content_hash_ = hash; } | |
| 121 | inline bool HasContentHash() const { return !content_hash_.IsNull(); } | ||
| 122 | ✗ | void SetExternalData(bool val) { external_data_ = val; } | |
| 123 | ✗ | void SetDirectIo(bool val) { direct_io_ = val; } | |
| 124 | |||
| 125 | ✗ | inline zlib::Algorithms GetCompressionAlgorithm() const { | |
| 126 | ✗ | return compression_algorithm_; | |
| 127 | } | ||
| 128 | ✗ | inline void SetCompressionAlgorithm(const zlib::Algorithms &alg) { | |
| 129 | ✗ | compression_algorithm_ = alg; | |
| 130 | ✗ | has_compression_algorithm_ = true; | |
| 131 | } | ||
| 132 | |||
| 133 | /** | ||
| 134 | * Generates a DirectoryEntry that can be directly stored into a catalog db. | ||
| 135 | * Note: this sets the inode fields to kInvalidInode as well as the link | ||
| 136 | * count to 1 if MaskHardlink() has been called before (cf. OverlayFS) | ||
| 137 | * | ||
| 138 | * If nanosecond timestamps are off, the directory entry will have a | ||
| 139 | * default initialized, negative nanosecond timestamp and as a result | ||
| 140 | * the corresponding field in the catalog table will be NULL. | ||
| 141 | * | ||
| 142 | * @return a DirectoryEntry structure to be written into a catalog | ||
| 143 | */ | ||
| 144 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent( | ||
| 145 | bool enable_mtime_ns) const = 0; | ||
| 146 | |||
| 147 | 1550 | inline std::string GetRelativePath() const { | |
| 148 | 1550 | return (relative_parent_path_.empty()) | |
| 149 | 300 | ? filename_ | |
| 150 | 1250 | : relative_parent_path_ | |
| 151 |
9/21✓ Branch 0 taken 300 times.
✓ Branch 1 taken 1250 times.
✓ Branch 3 taken 300 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 1250 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 13 taken 1250 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 1250 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 1250 times.
✓ Branch 19 taken 300 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 1550 times.
✗ Branch 24 not taken.
✗ Branch 25 not taken.
✗ Branch 27 not taken.
✗ Branch 28 not taken.
|
3100 | + (filename_.empty() ? "" : ("/" + filename_)); |
| 152 | } | ||
| 153 | |||
| 154 | std::string GetRdOnlyPath() const; | ||
| 155 | std::string GetUnionPath() const; | ||
| 156 | std::string GetScratchPath() const; | ||
| 157 | |||
| 158 | void MarkAsWhiteout(const std::string &actual_filename); | ||
| 159 | void MarkAsOpaqueDirectory(); | ||
| 160 | |||
| 161 | /** | ||
| 162 | * Union file systems (i.e. OverlayFS) might not properly support hardlinks, | ||
| 163 | * forcing us to ignore them during publishing. A 'masked hardlink' will be | ||
| 164 | * treated as a normal file (linkcount == 1). Hence, any created hardlinks | ||
| 165 | * will be broken up into individual files with differing inodes. | ||
| 166 | */ | ||
| 167 | ✗ | inline void MaskHardlink() { masked_hardlink_ = true; } | |
| 168 | ✗ | inline bool HasHardlinks() const { | |
| 169 | ✗ | return !masked_hardlink_ && GetUnionLinkcount() > 1; | |
| 170 | } | ||
| 171 | |||
| 172 | unsigned int GetRdOnlyLinkcount() const; | ||
| 173 | uint64_t GetRdOnlyInode() const; | ||
| 174 | unsigned int GetUnionLinkcount() const; | ||
| 175 | uint64_t GetUnionInode() const; | ||
| 176 | uint64_t GetScratchSize() const; | ||
| 177 | uint64_t GetRdOnlySize() const; | ||
| 178 | |||
| 179 | 750 | inline std::string filename() const { return filename_; } | |
| 180 | ✗ | inline std::string relative_parent_path() const { | |
| 181 | ✗ | return relative_parent_path_; | |
| 182 | } | ||
| 183 | |||
| 184 | virtual IngestionSource *CreateIngestionSource() const = 0; | ||
| 185 | virtual void MakePlaceholderDirectory() const = 0; | ||
| 186 | ✗ | void SetCatalogMarker() { has_catalog_marker_ = true; } | |
| 187 | |||
| 188 | ✗ | bool operator==(const SyncItem &other) const { | |
| 189 | ✗ | return ((relative_parent_path_ == other.relative_parent_path_) | |
| 190 | ✗ | && (filename_ == other.filename_)); | |
| 191 | } | ||
| 192 | |||
| 193 | protected: | ||
| 194 | /** | ||
| 195 | * create a new SyncItem | ||
| 196 | * Note: SyncItems cannot be created by any using code. SyncUnion will take | ||
| 197 | * care of their creating through a factory method to make sure they | ||
| 198 | * are initialised correctly (whiteout, hardlink handling, ...) | ||
| 199 | * | ||
| 200 | * @param dirPath the RELATIVE path to the file | ||
| 201 | * @param filename the name of the file ;-) | ||
| 202 | * @param entryType well... | ||
| 203 | */ | ||
| 204 | SyncItem(const std::string &relative_parent_path, | ||
| 205 | const std::string &filename, | ||
| 206 | const SyncUnion *union_engine, | ||
| 207 | const SyncItemType entry_type); | ||
| 208 | |||
| 209 | ✗ | inline platform_stat64 GetUnionStat() const { | |
| 210 | ✗ | StatUnion(); | |
| 211 | ✗ | return union_stat_.stat; | |
| 212 | } | ||
| 213 | |||
| 214 | SyncItemType GetRdOnlyFiletype() const; | ||
| 215 | SyncItemType GetUnionFiletype() const; | ||
| 216 | |||
| 217 | virtual SyncItemType GetScratchFiletype() const = 0; | ||
| 218 | |||
| 219 | /** | ||
| 220 | * Checks if the SyncItem _is_ the given file type (file, dir, symlink, ...) | ||
| 221 | * in the union file system volume. Hence: After the publish operation, the | ||
| 222 | * file will be this type in CVMFS. | ||
| 223 | * @param expected_type the file type to be checked against | ||
| 224 | * @return true if file type matches the expected type | ||
| 225 | */ | ||
| 226 | virtual bool IsType(const SyncItemType expected_type) const = 0; | ||
| 227 | |||
| 228 | /** | ||
| 229 | * Checks if the SyncItem _was_ the given file type (file, dir, symlink, ...) | ||
| 230 | * in CVMFS (or the lower layer of the union file system). Hence: Before the | ||
| 231 | * current transaction the file _was_ this type in CVMFS. | ||
| 232 | * @param expected_type the file type to be checked against | ||
| 233 | * @return true if file type was the expected type in CVMFS | ||
| 234 | */ | ||
| 235 | 475 | inline bool WasType(const SyncItemType expected_type) const { | |
| 236 |
2/2✓ Branch 0 taken 425 times.
✓ Branch 1 taken 50 times.
|
475 | if (rdonly_type_ == kItemUnknown) { |
| 237 | 425 | rdonly_type_ = GetRdOnlyFiletype(); | |
| 238 | } | ||
| 239 | 475 | return rdonly_type_ == expected_type; | |
| 240 | } | ||
| 241 | |||
| 242 | /** | ||
| 243 | * Structure to cache stat calls to the different file locations. | ||
| 244 | */ | ||
| 245 | struct EntryStat { | ||
| 246 | 1550 | EntryStat() : obtained(false), error_code(0) { | |
| 247 | 1550 | memset(&stat, 0, sizeof(stat)); | |
| 248 | 1550 | } | |
| 249 | |||
| 250 | ✗ | inline SyncItemType GetSyncItemType() const { | |
| 251 | ✗ | assert(obtained); | |
| 252 | ✗ | if (S_ISREG(stat.st_mode)) | |
| 253 | ✗ | return kItemFile; | |
| 254 | ✗ | if (S_ISLNK(stat.st_mode)) | |
| 255 | ✗ | return kItemSymlink; | |
| 256 | ✗ | if (S_ISDIR(stat.st_mode)) | |
| 257 | ✗ | return kItemDir; | |
| 258 | ✗ | if (S_ISFIFO(stat.st_mode)) | |
| 259 | ✗ | return kItemFifo; | |
| 260 | ✗ | if (S_ISSOCK(stat.st_mode)) | |
| 261 | ✗ | return kItemSocket; | |
| 262 | ✗ | if (S_ISCHR(stat.st_mode)) | |
| 263 | ✗ | return kItemCharacterDevice; | |
| 264 | ✗ | if (S_ISBLK(stat.st_mode)) | |
| 265 | ✗ | return kItemBlockDevice; | |
| 266 | ✗ | return kItemUnknown; | |
| 267 | } | ||
| 268 | |||
| 269 | bool obtained; /**< false at the beginning, true after first stat call */ | ||
| 270 | int error_code; /**< errno value of the stat call */ | ||
| 271 | platform_stat64 stat; | ||
| 272 | }; | ||
| 273 | |||
| 274 | static void StatGeneric(const std::string &path, | ||
| 275 | EntryStat *info, | ||
| 276 | const bool refresh); | ||
| 277 | SyncItemType GetGenericFiletype(const EntryStat &stat) const; | ||
| 278 | void CheckMarkerFiles(); | ||
| 279 | |||
| 280 | mutable SyncItemType rdonly_type_; | ||
| 281 | mutable EntryStat scratch_stat_; | ||
| 282 | |||
| 283 | ssize_t graft_size_; | ||
| 284 | |||
| 285 | // The hash of regular file's content | ||
| 286 | shash::Any content_hash_; | ||
| 287 | |||
| 288 | mutable SyncItemType scratch_type_; | ||
| 289 | |||
| 290 | private: | ||
| 291 | void CheckCatalogMarker(); | ||
| 292 | |||
| 293 | std::string filename_; | ||
| 294 | |||
| 295 | std::string GetGraftMarkerPath() const; | ||
| 296 | void CheckGraft(); | ||
| 297 | |||
| 298 | const SyncUnion *union_engine_; /**< this SyncUnion created this object */ | ||
| 299 | |||
| 300 | mutable EntryStat rdonly_stat_; | ||
| 301 | mutable EntryStat union_stat_; | ||
| 302 | |||
| 303 | bool whiteout_; /**< SyncUnion marked this as whiteout */ | ||
| 304 | bool opaque_; /**< SyncUnion marked this as opaque dir*/ | ||
| 305 | bool masked_hardlink_; /**< SyncUnion masked out the linkcount */ | ||
| 306 | bool has_catalog_marker_; /**< directory containing .cvmfscatalog */ | ||
| 307 | bool valid_graft_; /**< checksum and size in graft marker */ | ||
| 308 | bool graft_marker_present_; /**< .cvmfsgraft-$filename exists */ | ||
| 309 | |||
| 310 | bool external_data_; | ||
| 311 | bool direct_io_; | ||
| 312 | std::string relative_parent_path_; | ||
| 313 | |||
| 314 | /** | ||
| 315 | * Chunklist from graft. Not initialized by default to save memory. | ||
| 316 | */ | ||
| 317 | FileChunkList *graft_chunklist_; | ||
| 318 | |||
| 319 | // The compression algorithm for the file | ||
| 320 | zlib::Algorithms compression_algorithm_; | ||
| 321 | // The compression algorithm has been set explicitly | ||
| 322 | bool has_compression_algorithm_; | ||
| 323 | |||
| 324 | // Lazy evaluation and caching of results of file stats | ||
| 325 | 425 | inline void StatRdOnly(const bool refresh = false) const { | |
| 326 | 425 | StatGeneric(GetRdOnlyPath(), &rdonly_stat_, refresh); | |
| 327 | 425 | } | |
| 328 | ✗ | inline void StatUnion(const bool refresh = false) const { | |
| 329 | ✗ | StatGeneric(GetUnionPath(), &union_stat_, refresh); | |
| 330 | } | ||
| 331 | virtual void StatScratch(const bool refresh) const = 0; | ||
| 332 | }; | ||
| 333 | |||
| 334 | typedef std::map<std::string, SharedPtr<SyncItem> > SyncItemList; | ||
| 335 | |||
| 336 | class SyncItemNative : public SyncItem { | ||
| 337 | friend class SyncUnion; | ||
| 338 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent( | ||
| 339 | bool enable_mtime_ns) const; | ||
| 340 | virtual IngestionSource *CreateIngestionSource() const; | ||
| 341 | ✗ | virtual void MakePlaceholderDirectory() const { assert(false); } | |
| 342 | virtual SyncItemType GetScratchFiletype() const; | ||
| 343 | virtual bool IsType(const SyncItemType expected_type) const; | ||
| 344 | ✗ | virtual void StatScratch(const bool refresh) const { | |
| 345 | ✗ | StatGeneric(GetScratchPath(), &scratch_stat_, refresh); | |
| 346 | } | ||
| 347 | |||
| 348 | protected: | ||
| 349 | 125 | SyncItemNative(const std::string &relative_parent_path, | |
| 350 | const std::string &filename, const SyncUnion *union_engine, | ||
| 351 | const SyncItemType entry_type) | ||
| 352 | 125 | : SyncItem(relative_parent_path, filename, union_engine, entry_type) { | |
| 353 |
1/2✓ Branch 1 taken 125 times.
✗ Branch 2 not taken.
|
125 | CheckMarkerFiles(); |
| 354 | 125 | } | |
| 355 | }; | ||
| 356 | |||
| 357 | } // namespace publish | ||
| 358 | |||
| 359 | #endif // CVMFS_SYNC_ITEM_H_ | ||
| 360 |