| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/sync_item.h |
| Date: | 2026-06-21 02:37:04 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 25 | 104 | 24.0% |
| Branches: | 12 | 85 | 14.1% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System | ||
| 3 | */ | ||
| 4 | |||
| 5 | #ifndef CVMFS_SYNC_ITEM_H_ | ||
| 6 | #define CVMFS_SYNC_ITEM_H_ | ||
| 7 | |||
| 8 | #include <sys/types.h> | ||
| 9 | |||
| 10 | #if !defined(__APPLE__) | ||
| 11 | #include <sys/sysmacros.h> | ||
| 12 | #endif // __APPLE__ | ||
| 13 | |||
| 14 | #include <cstring> | ||
| 15 | #include <map> | ||
| 16 | #include <string> | ||
| 17 | |||
| 18 | #include "crypto/hash.h" | ||
| 19 | #include "directory_entry.h" | ||
| 20 | #include "duplex_libarchive.h" | ||
| 21 | #include "file_chunk.h" | ||
| 22 | #include "util/platform.h" | ||
| 23 | #include "util/shared_ptr.h" | ||
| 24 | #include "util/string.h" | ||
| 25 | |||
| 26 | class IngestionSource; | ||
| 27 | |||
| 28 | namespace publish { | ||
| 29 | |||
| 30 | enum SyncItemType { | ||
| 31 | kItemDir, | ||
| 32 | kItemFile, | ||
| 33 | kItemSymlink, | ||
| 34 | kItemCharacterDevice, | ||
| 35 | kItemBlockDevice, | ||
| 36 | kItemFifo, | ||
| 37 | kItemSocket, | ||
| 38 | kItemNew, | ||
| 39 | kItemMarker, | ||
| 40 | kItemUnknown, | ||
| 41 | }; | ||
| 42 | |||
| 43 | class SyncUnion; | ||
| 44 | /** | ||
| 45 | * Every directory entry emitted by the FileSystemTraversal is wrapped in a | ||
| 46 | * SyncItem structure by the factory method SyncUnion::CreateSyncItem(). | ||
| 47 | * | ||
| 48 | * Since we are dealing with a union file system setup, this class represents | ||
| 49 | * potentially three concrete files: | ||
| 50 | * - <read-only path>/<filename> | cf. rdonly_stat_ | ||
| 51 | * - <scratch (read-write) branch>/<filename> | cf. scratch_stat_ | ||
| 52 | * - <union volume path>/<filename> | cf. union_stat_ | ||
| 53 | * | ||
| 54 | * This class caches stat calls to the underlying files in different branches of | ||
| 55 | * the union file system and hides some interpretation details. | ||
| 56 | */ | ||
| 57 | class SyncItem { | ||
| 58 | // only SyncUnion can create SyncItems (see SyncUnion::CreateSyncItem). | ||
| 59 | // SyncUnionTarball can create SyncItemTar and SyncItemDummyDir. | ||
| 60 | |||
| 61 | public: | ||
| 62 | SyncItem(); | ||
| 63 | virtual ~SyncItem(); | ||
| 64 | |||
| 65 | 574 | inline bool IsDirectory() const { return IsType(kItemDir); } | |
| 66 | ✗ | inline bool WasDirectory() const { return WasType(kItemDir); } | |
| 67 | 406 | inline bool IsRegularFile() const { return IsType(kItemFile); } | |
| 68 | ✗ | inline bool WasRegularFile() const { return WasType(kItemFile); } | |
| 69 | 14 | inline bool IsSymlink() const { return IsType(kItemSymlink); } | |
| 70 | ✗ | inline bool WasSymlink() const { return WasType(kItemSymlink); } | |
| 71 | 266 | inline bool IsNew() const { return WasType(kItemNew); } | |
| 72 | inline bool IsTouched() const { | ||
| 73 | return (GetRdOnlyFiletype() == GetUnionFiletype()) | ||
| 74 | && (GetRdOnlyFiletype() == GetScratchFiletype()) | ||
| 75 | && (GetUnionFiletype() == GetScratchFiletype()); | ||
| 76 | } | ||
| 77 | ✗ | inline bool IsCharacterDevice() const { return IsType(kItemCharacterDevice); } | |
| 78 | ✗ | inline bool IsBlockDevice() const { return IsType(kItemBlockDevice); } | |
| 79 | ✗ | inline bool IsFifo() const { return IsType(kItemFifo); } | |
| 80 | ✗ | inline bool IsSocket() const { return IsType(kItemSocket); } | |
| 81 | ✗ | inline bool IsGraftMarker() const { return IsType(kItemMarker); } | |
| 82 | ✗ | inline bool IsExternalData() const { return external_data_; } | |
| 83 | ✗ | inline bool IsDirectIo() const { return direct_io_; } | |
| 84 | |||
| 85 | 140 | inline bool IsWhiteout() const { return whiteout_; } | |
| 86 | ✗ | inline bool IsCatalogMarker() const { return filename_ == ".cvmfscatalog"; } | |
| 87 | ✗ | inline bool IsOpaqueDirectory() const { return IsDirectory() && opaque_; } | |
| 88 | |||
| 89 | ✗ | inline bool IsSpecialFile() const { | |
| 90 | ✗ | return IsCharacterDevice() || IsBlockDevice() || IsFifo() || IsSocket(); | |
| 91 | } | ||
| 92 | ✗ | inline bool WasSpecialFile() const { | |
| 93 | ✗ | return WasType(kItemCharacterDevice) || WasType(kItemBlockDevice) | |
| 94 | ✗ | || WasType(kItemFifo) || WasType(kItemSocket); | |
| 95 | } | ||
| 96 | ✗ | inline bool IsBundleSpec() const { | |
| 97 | ✗ | return HasPrefix(filename_, ".cvmfsbundle-", false /* ignore_case */); | |
| 98 | } | ||
| 99 | ✗ | inline bool WasBundleSpec() const { | |
| 100 | ✗ | return HasPrefix(filename_, ".cvmfsbundle-", false /* ignore_case */); | |
| 101 | } | ||
| 102 | |||
| 103 | ✗ | inline unsigned int GetRdevMajor() const { | |
| 104 | ✗ | assert(IsSpecialFile()); | |
| 105 | ✗ | StatUnion(true); | |
| 106 | ✗ | return major(union_stat_.stat.st_rdev); | |
| 107 | } | ||
| 108 | |||
| 109 | ✗ | inline unsigned int GetRdevMinor() const { | |
| 110 | ✗ | assert(IsSpecialFile()); | |
| 111 | ✗ | StatUnion(true); | |
| 112 | ✗ | return minor(union_stat_.stat.st_rdev); | |
| 113 | } | ||
| 114 | |||
| 115 | ✗ | bool HasCatalogMarker() const { return has_catalog_marker_; } | |
| 116 | ✗ | bool HasGraftMarker() const { return graft_marker_present_; } | |
| 117 | ✗ | bool HasCompressionAlgorithm() const { return has_compression_algorithm_; } | |
| 118 | ✗ | bool IsValidGraft() const { return valid_graft_; } | |
| 119 | ✗ | bool IsChunkedGraft() const { return graft_chunklist_; } | |
| 120 | |||
| 121 | ✗ | inline const FileChunkList *GetGraftChunks() const { | |
| 122 | ✗ | return graft_chunklist_; | |
| 123 | } | ||
| 124 | ✗ | inline shash::Any GetContentHash() const { return content_hash_; } | |
| 125 | ✗ | inline void SetContentHash(const shash::Any &hash) { content_hash_ = hash; } | |
| 126 | inline bool HasContentHash() const { return !content_hash_.IsNull(); } | ||
| 127 | ✗ | void SetExternalData(bool val) { external_data_ = val; } | |
| 128 | ✗ | void SetDirectIo(bool val) { direct_io_ = val; } | |
| 129 | |||
| 130 | ✗ | inline zlib::Algorithms GetCompressionAlgorithm() const { | |
| 131 | ✗ | return compression_algorithm_; | |
| 132 | } | ||
| 133 | ✗ | inline void SetCompressionAlgorithm(const zlib::Algorithms &alg) { | |
| 134 | ✗ | compression_algorithm_ = alg; | |
| 135 | ✗ | has_compression_algorithm_ = true; | |
| 136 | } | ||
| 137 | |||
| 138 | /** | ||
| 139 | * Generates a DirectoryEntry that can be directly stored into a catalog db. | ||
| 140 | * Note: this sets the inode fields to kInvalidInode as well as the link | ||
| 141 | * count to 1 if MaskHardlink() has been called before (cf. OverlayFS) | ||
| 142 | * | ||
| 143 | * If nanosecond timestamps are off, the directory entry will have a | ||
| 144 | * default initialized, negative nanosecond timestamp and as a result | ||
| 145 | * the corresponding field in the catalog table will be NULL. | ||
| 146 | * | ||
| 147 | * @return a DirectoryEntry structure to be written into a catalog | ||
| 148 | */ | ||
| 149 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent( | ||
| 150 | bool enable_mtime_ns) const = 0; | ||
| 151 | |||
| 152 | 980 | inline std::string GetRelativePath() const { | |
| 153 | 980 | return (relative_parent_path_.empty()) | |
| 154 | 168 | ? filename_ | |
| 155 | 812 | : relative_parent_path_ | |
| 156 |
9/21✓ Branch 0 taken 168 times.
✓ Branch 1 taken 812 times.
✓ Branch 3 taken 168 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 812 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 13 taken 812 times.
✗ Branch 14 not taken.
✓ Branch 16 taken 812 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 812 times.
✓ Branch 19 taken 168 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 980 times.
✗ Branch 24 not taken.
✗ Branch 25 not taken.
✗ Branch 27 not taken.
✗ Branch 28 not taken.
|
1960 | + (filename_.empty() ? "" : ("/" + filename_)); |
| 157 | } | ||
| 158 | |||
| 159 | std::string GetRdOnlyPath() const; | ||
| 160 | std::string GetUnionPath() const; | ||
| 161 | std::string GetScratchPath() const; | ||
| 162 | |||
| 163 | void MarkAsWhiteout(const std::string &actual_filename); | ||
| 164 | void MarkAsOpaqueDirectory(); | ||
| 165 | |||
| 166 | /** | ||
| 167 | * Union file systems (i.e. OverlayFS) might not properly support hardlinks, | ||
| 168 | * forcing us to ignore them during publishing. A 'masked hardlink' will be | ||
| 169 | * treated as a normal file (linkcount == 1). Hence, any created hardlinks | ||
| 170 | * will be broken up into individual files with differing inodes. | ||
| 171 | */ | ||
| 172 | ✗ | inline void MaskHardlink() { masked_hardlink_ = true; } | |
| 173 | ✗ | inline bool HasHardlinks() const { | |
| 174 | ✗ | return !masked_hardlink_ && GetUnionLinkcount() > 1; | |
| 175 | } | ||
| 176 | |||
| 177 | unsigned int GetRdOnlyLinkcount() const; | ||
| 178 | uint64_t GetRdOnlyInode() const; | ||
| 179 | unsigned int GetUnionLinkcount() const; | ||
| 180 | uint64_t GetUnionInode() const; | ||
| 181 | uint64_t GetScratchSize() const; | ||
| 182 | uint64_t GetRdOnlySize() const; | ||
| 183 | |||
| 184 | 588 | inline std::string filename() const { return filename_; } | |
| 185 | ✗ | inline std::string relative_parent_path() const { | |
| 186 | ✗ | return relative_parent_path_; | |
| 187 | } | ||
| 188 | |||
| 189 | virtual IngestionSource *CreateIngestionSource() const = 0; | ||
| 190 | virtual void MakePlaceholderDirectory() const = 0; | ||
| 191 | ✗ | void SetCatalogMarker() { has_catalog_marker_ = true; } | |
| 192 | |||
| 193 | ✗ | bool operator==(const SyncItem &other) const { | |
| 194 | ✗ | return ((relative_parent_path_ == other.relative_parent_path_) | |
| 195 | ✗ | && (filename_ == other.filename_)); | |
| 196 | } | ||
| 197 | |||
| 198 | protected: | ||
| 199 | /** | ||
| 200 | * create a new SyncItem | ||
| 201 | * Note: SyncItems cannot be created by any using code. SyncUnion will take | ||
| 202 | * care of their creating through a factory method to make sure they | ||
| 203 | * are initialised correctly (whiteout, hardlink handling, ...) | ||
| 204 | * | ||
| 205 | * @param dirPath the RELATIVE path to the file | ||
| 206 | * @param filename the name of the file ;-) | ||
| 207 | * @param entryType well... | ||
| 208 | */ | ||
| 209 | SyncItem(const std::string &relative_parent_path, | ||
| 210 | const std::string &filename, | ||
| 211 | const SyncUnion *union_engine, | ||
| 212 | const SyncItemType entry_type); | ||
| 213 | |||
| 214 | ✗ | inline platform_stat64 GetUnionStat() const { | |
| 215 | ✗ | StatUnion(); | |
| 216 | ✗ | return union_stat_.stat; | |
| 217 | } | ||
| 218 | |||
| 219 | SyncItemType GetRdOnlyFiletype() const; | ||
| 220 | SyncItemType GetUnionFiletype() const; | ||
| 221 | |||
| 222 | virtual SyncItemType GetScratchFiletype() const = 0; | ||
| 223 | |||
| 224 | /** | ||
| 225 | * Checks if the SyncItem _is_ the given file type (file, dir, symlink, ...) | ||
| 226 | * in the union file system volume. Hence: After the publish operation, the | ||
| 227 | * file will be this type in CVMFS. | ||
| 228 | * @param expected_type the file type to be checked against | ||
| 229 | * @return true if file type matches the expected type | ||
| 230 | */ | ||
| 231 | virtual bool IsType(const SyncItemType expected_type) const = 0; | ||
| 232 | |||
| 233 | /** | ||
| 234 | * Checks if the SyncItem _was_ the given file type (file, dir, symlink, ...) | ||
| 235 | * in CVMFS (or the lower layer of the union file system). Hence: Before the | ||
| 236 | * current transaction the file _was_ this type in CVMFS. | ||
| 237 | * @param expected_type the file type to be checked against | ||
| 238 | * @return true if file type was the expected type in CVMFS | ||
| 239 | */ | ||
| 240 | 266 | inline bool WasType(const SyncItemType expected_type) const { | |
| 241 |
2/2✓ Branch 0 taken 238 times.
✓ Branch 1 taken 28 times.
|
266 | if (rdonly_type_ == kItemUnknown) { |
| 242 | 238 | rdonly_type_ = GetRdOnlyFiletype(); | |
| 243 | } | ||
| 244 | 266 | return rdonly_type_ == expected_type; | |
| 245 | } | ||
| 246 | |||
| 247 | /** | ||
| 248 | * Structure to cache stat calls to the different file locations. | ||
| 249 | */ | ||
| 250 | struct EntryStat { | ||
| 251 | 1092 | EntryStat() : obtained(false), error_code(0) { | |
| 252 | 1092 | memset(&stat, 0, sizeof(stat)); | |
| 253 | 1092 | } | |
| 254 | |||
| 255 | ✗ | inline SyncItemType GetSyncItemType() const { | |
| 256 | ✗ | assert(obtained); | |
| 257 | ✗ | if (S_ISREG(stat.st_mode)) | |
| 258 | ✗ | return kItemFile; | |
| 259 | ✗ | if (S_ISLNK(stat.st_mode)) | |
| 260 | ✗ | return kItemSymlink; | |
| 261 | ✗ | if (S_ISDIR(stat.st_mode)) | |
| 262 | ✗ | return kItemDir; | |
| 263 | ✗ | if (S_ISFIFO(stat.st_mode)) | |
| 264 | ✗ | return kItemFifo; | |
| 265 | ✗ | if (S_ISSOCK(stat.st_mode)) | |
| 266 | ✗ | return kItemSocket; | |
| 267 | ✗ | if (S_ISCHR(stat.st_mode)) | |
| 268 | ✗ | return kItemCharacterDevice; | |
| 269 | ✗ | if (S_ISBLK(stat.st_mode)) | |
| 270 | ✗ | return kItemBlockDevice; | |
| 271 | ✗ | return kItemUnknown; | |
| 272 | } | ||
| 273 | |||
| 274 | bool obtained; /**< false at the beginning, true after first stat call */ | ||
| 275 | int error_code; /**< errno value of the stat call */ | ||
| 276 | platform_stat64 stat; | ||
| 277 | }; | ||
| 278 | |||
| 279 | static void StatGeneric(const std::string &path, | ||
| 280 | EntryStat *info, | ||
| 281 | const bool refresh); | ||
| 282 | SyncItemType GetGenericFiletype(const EntryStat &stat) const; | ||
| 283 | void CheckMarkerFiles(); | ||
| 284 | |||
| 285 | mutable SyncItemType rdonly_type_; | ||
| 286 | mutable EntryStat scratch_stat_; | ||
| 287 | |||
| 288 | ssize_t graft_size_; | ||
| 289 | |||
| 290 | // The hash of regular file's content | ||
| 291 | shash::Any content_hash_; | ||
| 292 | |||
| 293 | mutable SyncItemType scratch_type_; | ||
| 294 | |||
| 295 | private: | ||
| 296 | void CheckCatalogMarker(); | ||
| 297 | |||
| 298 | std::string filename_; | ||
| 299 | |||
| 300 | std::string GetGraftMarkerPath() const; | ||
| 301 | void CheckGraft(); | ||
| 302 | |||
| 303 | const SyncUnion *union_engine_; /**< this SyncUnion created this object */ | ||
| 304 | |||
| 305 | mutable EntryStat rdonly_stat_; | ||
| 306 | mutable EntryStat union_stat_; | ||
| 307 | |||
| 308 | bool whiteout_; /**< SyncUnion marked this as whiteout */ | ||
| 309 | bool opaque_; /**< SyncUnion marked this as opaque dir*/ | ||
| 310 | bool masked_hardlink_; /**< SyncUnion masked out the linkcount */ | ||
| 311 | bool has_catalog_marker_; /**< directory containing .cvmfscatalog */ | ||
| 312 | bool valid_graft_; /**< checksum and size in graft marker */ | ||
| 313 | bool graft_marker_present_; /**< .cvmfsgraft-$filename exists */ | ||
| 314 | |||
| 315 | bool external_data_; | ||
| 316 | bool direct_io_; | ||
| 317 | std::string relative_parent_path_; | ||
| 318 | |||
| 319 | /** | ||
| 320 | * Chunklist from graft. Not initialized by default to save memory. | ||
| 321 | */ | ||
| 322 | FileChunkList *graft_chunklist_; | ||
| 323 | |||
| 324 | // The compression algorithm for the file | ||
| 325 | zlib::Algorithms compression_algorithm_; | ||
| 326 | // The compression algorithm has been set explicitly | ||
| 327 | bool has_compression_algorithm_; | ||
| 328 | |||
| 329 | // Lazy evaluation and caching of results of file stats | ||
| 330 | 238 | inline void StatRdOnly(const bool refresh = false) const { | |
| 331 | 238 | StatGeneric(GetRdOnlyPath(), &rdonly_stat_, refresh); | |
| 332 | 238 | } | |
| 333 | ✗ | inline void StatUnion(const bool refresh = false) const { | |
| 334 | ✗ | StatGeneric(GetUnionPath(), &union_stat_, refresh); | |
| 335 | } | ||
| 336 | virtual void StatScratch(const bool refresh) const = 0; | ||
| 337 | }; | ||
| 338 | |||
| 339 | typedef std::map<std::string, SharedPtr<SyncItem> > SyncItemList; | ||
| 340 | |||
| 341 | class SyncItemNative : public SyncItem { | ||
| 342 | friend class SyncUnion; | ||
| 343 | virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent( | ||
| 344 | bool enable_mtime_ns) const; | ||
| 345 | virtual IngestionSource *CreateIngestionSource() const; | ||
| 346 | ✗ | virtual void MakePlaceholderDirectory() const { assert(false); } | |
| 347 | virtual SyncItemType GetScratchFiletype() const; | ||
| 348 | virtual bool IsType(const SyncItemType expected_type) const; | ||
| 349 | ✗ | virtual void StatScratch(const bool refresh) const { | |
| 350 | ✗ | StatGeneric(GetScratchPath(), &scratch_stat_, refresh); | |
| 351 | } | ||
| 352 | |||
| 353 | protected: | ||
| 354 | 126 | SyncItemNative(const std::string &relative_parent_path, | |
| 355 | const std::string &filename, const SyncUnion *union_engine, | ||
| 356 | const SyncItemType entry_type) | ||
| 357 | 126 | : SyncItem(relative_parent_path, filename, union_engine, entry_type) { | |
| 358 |
1/2✓ Branch 1 taken 126 times.
✗ Branch 2 not taken.
|
126 | CheckMarkerFiles(); |
| 359 | 126 | } | |
| 360 | }; | ||
| 361 | |||
| 362 | } // namespace publish | ||
| 363 | |||
| 364 | #endif // CVMFS_SYNC_ITEM_H_ | ||
| 365 |