| Directory: | cvmfs/ |
|---|---|
| File: | cvmfs/catalog.h |
| Date: | 2024-04-21 02:33:16 |
| Exec | Total | Coverage | |
|---|---|---|---|
| Lines: | 46 | 51 | 90.2% |
| Branches: | 17 | 40 | 42.5% |
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /** | ||
| 2 | * This file is part of the CernVM File System. | ||
| 3 | */ | ||
| 4 | |||
| 5 | #ifndef CVMFS_CATALOG_H_ | ||
| 6 | #define CVMFS_CATALOG_H_ | ||
| 7 | |||
| 8 | #include <pthread.h> | ||
| 9 | #include <stdint.h> | ||
| 10 | |||
| 11 | #include <cassert> | ||
| 12 | #include <map> | ||
| 13 | #include <string> | ||
| 14 | #include <vector> | ||
| 15 | |||
| 16 | #include "catalog_counters.h" | ||
| 17 | #include "catalog_sql.h" | ||
| 18 | #include "crypto/hash.h" | ||
| 19 | #include "directory_entry.h" | ||
| 20 | #include "file_chunk.h" | ||
| 21 | #include "gtest/gtest_prod.h" | ||
| 22 | #include "shortstring.h" | ||
| 23 | #include "sql.h" | ||
| 24 | #include "uid_map.h" | ||
| 25 | #include "xattr.h" | ||
| 26 | |||
| 27 | namespace swissknife { | ||
| 28 | class CommandMigrate; | ||
| 29 | } | ||
| 30 | |||
| 31 | namespace catalog { | ||
| 32 | |||
| 33 | template <class CatalogT> | ||
| 34 | class AbstractCatalogManager; | ||
| 35 | |||
| 36 | class Catalog; | ||
| 37 | |||
| 38 | class Counters; | ||
| 39 | |||
| 40 | typedef std::vector<Catalog *> CatalogList; | ||
| 41 | typedef IntegerMap<uint64_t> OwnerMap; // used to map uid/gid | ||
| 42 | |||
| 43 | |||
| 44 | /** | ||
| 45 | * Every Catalog gets an InodeRange assigned when attached to | ||
| 46 | * a CatalogManager. Inodes are assigned at runtime out of this InodeRange. | ||
| 47 | * An inode is computed by <row ID of entry> + offset | ||
| 48 | */ | ||
| 49 | struct InodeRange { | ||
| 50 | uint64_t offset; | ||
| 51 | uint64_t size; | ||
| 52 | |||
| 53 | 349 | InodeRange() : offset(0), size(0) { } | |
| 54 | |||
| 55 | inline bool ContainsInode(const inode_t inode) const { | ||
| 56 | return ((inode > offset) && (inode <= size + offset)); | ||
| 57 | } | ||
| 58 | |||
| 59 | 41 | inline void MakeDummy() { offset = 1; } | |
| 60 | |||
| 61 | 2168 | inline bool IsInitialized() const { return offset > 0; } | |
| 62 |
3/4✓ Branch 1 taken 707 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 19 times.
✓ Branch 4 taken 688 times.
|
707 | inline bool IsDummy() const { return IsInitialized() && size == 0; } |
| 63 | }; | ||
| 64 | |||
| 65 | |||
| 66 | /** | ||
| 67 | * Allows to define a class that transforms the inode in order to ensure | ||
| 68 | * that inodes are not reused after reloads (catalog or fuse module). | ||
| 69 | * Currently, annotation is used to set an offset starting at the highest | ||
| 70 | * so far issued inode. The implementation is in the catalog manager. | ||
| 71 | */ | ||
| 72 | class InodeAnnotation { | ||
| 73 | public: | ||
| 74 | 66 | virtual ~InodeAnnotation() { } | |
| 75 | virtual inode_t Annotate(const inode_t raw_inode) = 0; | ||
| 76 | virtual void IncGeneration(const uint64_t by) = 0; | ||
| 77 | virtual inode_t GetGeneration() = 0; | ||
| 78 | virtual bool ValidInode(const uint64_t inode) = 0; | ||
| 79 | virtual inode_t Strip(const inode_t annotated_inode) = 0; | ||
| 80 | }; | ||
| 81 | |||
| 82 | |||
| 83 | /** | ||
| 84 | * This class wraps a catalog database and provides methods | ||
| 85 | * to query for directory entries. | ||
| 86 | * It has a pointer to its parent catalog and its children, thereby creating | ||
| 87 | * a tree structure of nested catalogs. | ||
| 88 | * | ||
| 89 | * Read-only catalog. A sub-class provides read-write access. | ||
| 90 | */ | ||
| 91 | class Catalog : SingleCopy { | ||
| 92 | FRIEND_TEST(T_Catalog, NormalizePath); | ||
| 93 | FRIEND_TEST(T_Catalog, PlantPath); | ||
| 94 | friend class swissknife::CommandMigrate; // for catalog version migration | ||
| 95 | |||
| 96 | public: | ||
| 97 | typedef std::vector<shash::Any> HashVector; | ||
| 98 | |||
| 99 | /** | ||
| 100 | * The default TTL should be shorter than the autofs idle unmount time | ||
| 101 | * which is 5 minutes, because the config repo is accessed on every root | ||
| 102 | * catalog refresh and we want to avoid thrashing that mountpoint. | ||
| 103 | */ | ||
| 104 | static const uint64_t kDefaultTTL = 240; /**< 4 minutes default TTL */ | ||
| 105 | |||
| 106 | /** | ||
| 107 | * Note: is_nested only has an effect if parent == NULL otherwise being | ||
| 108 | * a root catalog is determined by having a parent pointer or not. | ||
| 109 | */ | ||
| 110 | Catalog(const PathString &mountpoint, | ||
| 111 | const shash::Any &catalog_hash, | ||
| 112 | Catalog *parent, | ||
| 113 | const bool is_nested = false); | ||
| 114 | virtual ~Catalog(); | ||
| 115 | |||
| 116 | static Catalog *AttachFreely(const std::string &imaginary_mountpoint, | ||
| 117 | const std::string &file, | ||
| 118 | const shash::Any &catalog_hash, | ||
| 119 | Catalog *parent = NULL, | ||
| 120 | const bool is_nested = false); | ||
| 121 | |||
| 122 | bool OpenDatabase(const std::string &db_path); | ||
| 123 | |||
| 124 | 536 | inline bool LookupPath(const PathString &path, DirectoryEntry *dirent) const { | |
| 125 |
1/2✓ Branch 2 taken 536 times.
✗ Branch 3 not taken.
|
536 | return LookupMd5Path(NormalizePath(path), dirent); |
| 126 | } | ||
| 127 | bool LookupRawSymlink(const PathString &path, LinkString *raw_symlink) const; | ||
| 128 | 1 | bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const { | |
| 129 |
1/2✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
|
1 | return LookupXattrsMd5Path(NormalizePath(path), xattrs); |
| 130 | } | ||
| 131 | |||
| 132 | 83 | inline bool ListingPath(const PathString &path, | |
| 133 | DirectoryEntryList *listing, | ||
| 134 | const bool expand_symlink = true) const | ||
| 135 | { | ||
| 136 |
1/2✓ Branch 2 taken 83 times.
✗ Branch 3 not taken.
|
83 | return ListingMd5Path(NormalizePath(path), listing, expand_symlink); |
| 137 | } | ||
| 138 | 5 | bool ListingPathStat(const PathString &path, | |
| 139 | StatEntryList *listing) const | ||
| 140 | { | ||
| 141 |
1/2✓ Branch 2 taken 5 times.
✗ Branch 3 not taken.
|
5 | return ListingMd5PathStat(NormalizePath(path), listing); |
| 142 | } | ||
| 143 | bool AllChunksBegin(); | ||
| 144 | bool AllChunksNext(shash::Any *hash, zlib::Algorithms *compression_alg); | ||
| 145 | bool AllChunksEnd(); | ||
| 146 | |||
| 147 | 1 | inline bool ListPathChunks(const PathString &path, | |
| 148 | const shash::Algorithms interpret_hashes_as, | ||
| 149 | FileChunkList *chunks) const | ||
| 150 | { | ||
| 151 |
1/2✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
|
1 | return ListMd5PathChunks(NormalizePath(path), interpret_hashes_as, chunks); |
| 152 | } | ||
| 153 | |||
| 154 | CatalogList GetChildren() const; | ||
| 155 | Catalog* FindSubtree(const PathString &path) const; | ||
| 156 | Catalog* FindChild(const PathString &mountpoint) const; | ||
| 157 | void AddChild(Catalog *child); | ||
| 158 | void RemoveChild(Catalog *child); | ||
| 159 | |||
| 160 | const HashVector& GetReferencedObjects() const; | ||
| 161 | void TakeDatabaseFileOwnership(); | ||
| 162 | void DropDatabaseFileOwnership(); | ||
| 163 | 5 | bool OwnsDatabaseFile() const { | |
| 164 |
5/6✓ Branch 0 taken 4 times.
✓ Branch 1 taken 1 times.
✓ Branch 3 taken 2 times.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 3 times.
|
5 | return ((database_ != NULL) && database_->OwnsFile()) || managed_database_; |
| 165 | } | ||
| 166 | |||
| 167 | uint64_t GetTTL() const; | ||
| 168 | bool HasExplicitTTL() const; | ||
| 169 | uint64_t GetRevision() const; | ||
| 170 | bool GetVOMSAuthz(std::string *authz) const; | ||
| 171 | uint64_t GetLastModified() const; | ||
| 172 | uint64_t GetNumEntries() const; | ||
| 173 | uint64_t GetNumChunks() const; | ||
| 174 | shash::Any GetPreviousRevision() const; | ||
| 175 | 42 | const Counters& GetCounters() const { return counters_; } | |
| 176 | std::string PrintMemStatistics() const; | ||
| 177 | |||
| 178 | 709 | inline float schema() const { return database().schema_version(); } | |
| 179 | 2161 | inline PathString mountpoint() const { return mountpoint_; } | |
| 180 | 139 | inline Catalog* parent() const { return parent_; } | |
| 181 | 102 | inline uint64_t max_row_id() const { return max_row_id_; } | |
| 182 | 101 | inline InodeRange inode_range() const { return inode_range_; } | |
| 183 | 142 | inline void set_inode_range(const InodeRange value) { inode_range_ = value; } | |
| 184 | 96 | inline std::string database_path() const { return database_->filename(); } | |
| 185 | 20 | inline PathString root_prefix() const { return root_prefix_; } | |
| 186 | 5 | inline shash::Any hash() const { return catalog_hash_; } | |
| 187 | 58 | inline bool volatile_flag() const { return volatile_flag_; } | |
| 188 | 5 | inline uint64_t revision() const { return GetRevision(); } | |
| 189 | |||
| 190 | 1461 | inline bool IsInitialized() const { | |
| 191 |
2/4✓ Branch 1 taken 1461 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1461 times.
✗ Branch 4 not taken.
|
1461 | return inode_range_.IsInitialized() && initialized_; |
| 192 | } | ||
| 193 | 159 | inline bool IsRoot() const { return is_root_; } | |
| 194 | ✗ | bool IsAutogenerated() const { | |
| 195 | ✗ | DirectoryEntry dirent; | |
| 196 | ✗ | assert(IsInitialized()); | |
| 197 | ✗ | return LookupPath(PathString( | |
| 198 | ✗ | mountpoint_.ToString() + "/.cvmfsautocatalog"), &dirent); | |
| 199 | } | ||
| 200 | 1010 | inline bool HasParent() const { return parent_ != NULL; } | |
| 201 | 2 | inline virtual bool IsWritable() const { return false; } | |
| 202 | |||
| 203 | typedef struct { | ||
| 204 | PathString mountpoint; | ||
| 205 | shash::Any hash; | ||
| 206 | uint64_t size; | ||
| 207 | } NestedCatalog; | ||
| 208 | typedef std::vector<NestedCatalog> NestedCatalogList; | ||
| 209 | const NestedCatalogList& ListNestedCatalogs() const; | ||
| 210 | const NestedCatalogList ListOwnNestedCatalogs() const; | ||
| 211 | bool FindNested(const PathString &mountpoint, | ||
| 212 | shash::Any *hash, uint64_t *size) const; | ||
| 213 | |||
| 214 | void SetInodeAnnotation(InodeAnnotation *new_annotation); | ||
| 215 | inode_t GetMangledInode(const uint64_t row_id, | ||
| 216 | const uint64_t hardlink_group) const; | ||
| 217 | |||
| 218 | void SetOwnerMaps(const OwnerMap *uid_map, const OwnerMap *gid_map); | ||
| 219 | 707 | uint64_t MapUid(const uint64_t uid) const { | |
| 220 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 707 times.
|
707 | if (uid_map_) { return uid_map_->Map(uid); } |
| 221 | 707 | return uid; | |
| 222 | } | ||
| 223 | 707 | uint64_t MapGid(const uint64_t gid) const { | |
| 224 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 707 times.
|
707 | if (gid_map_) { return gid_map_->Map(gid); } |
| 225 | 707 | return gid; | |
| 226 | } | ||
| 227 | |||
| 228 | protected: | ||
| 229 | typedef std::map<uint64_t, inode_t> HardlinkGroupMap; | ||
| 230 | mutable HardlinkGroupMap hardlink_groups_; | ||
| 231 | |||
| 232 | pthread_mutex_t *lock_; | ||
| 233 | |||
| 234 | bool InitStandalone(const std::string &database_file); | ||
| 235 | bool ReadCatalogCounters(); | ||
| 236 | |||
| 237 | /** | ||
| 238 | * Specifies the SQLite open flags. Overwritten by r/w catalog. | ||
| 239 | */ | ||
| 240 | 57 | virtual CatalogDatabase::OpenMode DatabaseOpenMode() const { | |
| 241 | 57 | return CatalogDatabase::kOpenReadOnly; | |
| 242 | } | ||
| 243 | |||
| 244 | virtual void InitPreparedStatements(); | ||
| 245 | void FinalizePreparedStatements(); | ||
| 246 | |||
| 247 | 1 | Counters& GetWritableCounters() { return counters_; } | |
| 248 | |||
| 249 | 1040 | inline const CatalogDatabase &database() const { return *database_; } | |
| 250 | 3832 | inline CatalogDatabase &database() { return *database_; } | |
| 251 | 99 | inline void set_parent(Catalog *catalog) { parent_ = catalog; } | |
| 252 | |||
| 253 | void ResetNestedCatalogCacheUnprotected(); | ||
| 254 | |||
| 255 | bool LookupMd5Path(const shash::Md5 &md5path, DirectoryEntry *dirent) const; | ||
| 256 | |||
| 257 | private: | ||
| 258 | typedef std::map<PathString, Catalog*> NestedCatalogMap; | ||
| 259 | |||
| 260 | /** | ||
| 261 | * The hash of the empty string. Used to identify the root entry of a | ||
| 262 | * repository, which is the child transition point of a bind mountpoint. | ||
| 263 | */ | ||
| 264 | static const shash::Md5 kMd5PathEmpty; | ||
| 265 | |||
| 266 | enum VomsAuthzStatus { | ||
| 267 | kVomsUnknown, // Not yet looked up | ||
| 268 | kVomsNone, // No voms_authz key in properties table | ||
| 269 | kVomsPresent, // voms_authz property available | ||
| 270 | }; | ||
| 271 | |||
| 272 | shash::Md5 NormalizePath(const PathString &path) const; | ||
| 273 | PathString NormalizePath2(const PathString &path) const; | ||
| 274 | PathString PlantPath(const PathString &path) const; | ||
| 275 | |||
| 276 | void FixTransitionPoint(const shash::Md5 &md5path, | ||
| 277 | DirectoryEntry *dirent) const; | ||
| 278 | |||
| 279 | bool LookupXattrsMd5Path(const shash::Md5 &md5path, XattrList *xattrs) const; | ||
| 280 | bool ListMd5PathChunks(const shash::Md5 &md5path, | ||
| 281 | const shash::Algorithms interpret_hashes_as, | ||
| 282 | FileChunkList *chunks) const; | ||
| 283 | bool ListingMd5Path(const shash::Md5 &md5path, | ||
| 284 | DirectoryEntryList *listing, | ||
| 285 | const bool expand_symlink = true) const; | ||
| 286 | bool ListingMd5PathStat(const shash::Md5 &md5path, | ||
| 287 | StatEntryList *listing) const; | ||
| 288 | bool LookupEntry(const shash::Md5 &md5path, const bool expand_symlink, | ||
| 289 | DirectoryEntry *dirent) const; | ||
| 290 | |||
| 291 | CatalogDatabase *database_; | ||
| 292 | |||
| 293 | const shash::Any catalog_hash_; | ||
| 294 | PathString root_prefix_; | ||
| 295 | /** | ||
| 296 | * Normally, catalogs are mounted at their root_prefix_. But for the structure | ||
| 297 | * under /.cvmfs/snapshots/..., that's not the case. | ||
| 298 | */ | ||
| 299 | PathString mountpoint_; | ||
| 300 | /** | ||
| 301 | * True, iff root_prefix_ == mountpoint_ | ||
| 302 | */ | ||
| 303 | bool is_regular_mountpoint_; | ||
| 304 | bool volatile_flag_; | ||
| 305 | /** | ||
| 306 | * For catalogs in a catalog manager: doesn't have a parent catalog | ||
| 307 | */ | ||
| 308 | const bool is_root_; | ||
| 309 | bool managed_database_; | ||
| 310 | |||
| 311 | Catalog *parent_; | ||
| 312 | NestedCatalogMap children_; | ||
| 313 | mutable NestedCatalogList nested_catalog_cache_; | ||
| 314 | mutable bool nested_catalog_cache_dirty_; | ||
| 315 | |||
| 316 | mutable VomsAuthzStatus voms_authz_status_; | ||
| 317 | mutable std::string voms_authz_; | ||
| 318 | |||
| 319 | bool initialized_; | ||
| 320 | InodeRange inode_range_; | ||
| 321 | uint64_t max_row_id_; | ||
| 322 | InodeAnnotation *inode_annotation_; | ||
| 323 | Counters counters_; | ||
| 324 | // Point to the maps in the catalog manager | ||
| 325 | const OwnerMap *uid_map_; | ||
| 326 | const OwnerMap *gid_map_; | ||
| 327 | |||
| 328 | SqlListing *sql_listing_; | ||
| 329 | SqlLookupPathHash *sql_lookup_md5path_; | ||
| 330 | SqlNestedCatalogLookup *sql_lookup_nested_; | ||
| 331 | SqlNestedCatalogListing *sql_list_nested_; | ||
| 332 | SqlOwnNestedCatalogListing *sql_own_list_nested_; | ||
| 333 | SqlAllChunks *sql_all_chunks_; | ||
| 334 | SqlChunksListing *sql_chunks_listing_; | ||
| 335 | SqlLookupXattrs *sql_lookup_xattrs_; | ||
| 336 | |||
| 337 | mutable HashVector referenced_hashes_; | ||
| 338 | }; // class Catalog | ||
| 339 | |||
| 340 | } // namespace catalog | ||
| 341 | |||
| 342 | #endif // CVMFS_CATALOG_H_ | ||
| 343 |