1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System. |
3 |
|
|
*/ |
4 |
|
|
|
5 |
|
|
#ifndef CVMFS_CATALOG_H_ |
6 |
|
|
#define CVMFS_CATALOG_H_ |
7 |
|
|
|
8 |
|
|
#include <pthread.h> |
9 |
|
|
#include <stdint.h> |
10 |
|
|
|
11 |
|
|
#include <cassert> |
12 |
|
|
#include <map> |
13 |
|
|
#include <string> |
14 |
|
|
#include <vector> |
15 |
|
|
|
16 |
|
|
#include "catalog_counters.h" |
17 |
|
|
#include "catalog_sql.h" |
18 |
|
|
#include "directory_entry.h" |
19 |
|
|
#include "file_chunk.h" |
20 |
|
|
#include "gtest/gtest_prod.h" |
21 |
|
|
#include "hash.h" |
22 |
|
|
#include "shortstring.h" |
23 |
|
|
#include "sql.h" |
24 |
|
|
#include "uid_map.h" |
25 |
|
|
#include "xattr.h" |
26 |
|
|
|
27 |
|
|
namespace swissknife { |
28 |
|
|
class CommandMigrate; |
29 |
|
|
} |
30 |
|
|
|
31 |
|
|
namespace catalog { |
32 |
|
|
|
33 |
|
|
template <class CatalogT> |
34 |
|
|
class AbstractCatalogManager; |
35 |
|
|
|
36 |
|
|
class Catalog; |
37 |
|
|
|
38 |
|
|
class Counters; |
39 |
|
|
|
40 |
|
|
typedef std::vector<Catalog *> CatalogList; |
41 |
|
|
typedef IntegerMap<uint64_t> OwnerMap; // used to map uid/gid |
42 |
|
|
|
43 |
|
|
|
44 |
|
|
/** |
45 |
|
|
* Every Catalog gets an InodeRange assigned when attached to |
46 |
|
|
* a CatalogManager. Inodes are assigned at runtime out of this InodeRange. |
47 |
|
|
* An inode is computed by <row ID of entry> + offset |
48 |
|
|
*/ |
49 |
|
|
struct InodeRange { |
50 |
|
|
uint64_t offset; |
51 |
|
|
uint64_t size; |
52 |
|
|
|
53 |
|
261 |
InodeRange() : offset(0), size(0) { } |
54 |
|
|
|
55 |
|
|
inline bool ContainsInode(const inode_t inode) const { |
56 |
|
|
return ((inode > offset) && (inode <= size + offset)); |
57 |
|
|
} |
58 |
|
|
|
59 |
|
28 |
inline void MakeDummy() { offset = 1; } |
60 |
|
|
|
61 |
|
662 |
inline bool IsInitialized() const { return offset > 0; } |
62 |
✓✗✓✓
|
199 |
inline bool IsDummy() const { return IsInitialized() && size == 0; } |
63 |
|
|
}; |
64 |
|
|
|
65 |
|
|
|
66 |
|
|
/** |
67 |
|
|
* Allows to define a class that transforms the inode in order to ensure |
68 |
|
|
* that inodes are not reused after reloads (catalog or fuse module). |
69 |
|
|
* Currently, annotation is used to set an offset starting at the highest |
70 |
|
|
* so far issued inode. The implementation is in the catalog manager. |
71 |
|
|
*/ |
72 |
|
58 |
class InodeAnnotation { |
73 |
|
|
public: |
74 |
✗✓ |
58 |
virtual ~InodeAnnotation() { } |
75 |
|
|
virtual inode_t Annotate(const inode_t raw_inode) = 0; |
76 |
|
|
virtual void IncGeneration(const uint64_t by) = 0; |
77 |
|
|
virtual inode_t GetGeneration() = 0; |
78 |
|
|
virtual bool ValidInode(const uint64_t inode) = 0; |
79 |
|
|
virtual inode_t Strip(const inode_t annotated_inode) = 0; |
80 |
|
|
}; |
81 |
|
|
|
82 |
|
|
|
83 |
|
|
/** |
84 |
|
|
* This class wraps a catalog database and provides methods |
85 |
|
|
* to query for directory entries. |
86 |
|
|
* It has a pointer to its parent catalog and its children, thereby creating |
87 |
|
|
* a tree structure of nested catalogs. |
88 |
|
|
* |
89 |
|
|
* Read-only catalog. A sub-class provides read-write access. |
90 |
|
|
*/ |
91 |
|
|
class Catalog : SingleCopy { |
92 |
|
|
FRIEND_TEST(T_Catalog, NormalizePath); |
93 |
|
|
FRIEND_TEST(T_Catalog, PlantPath); |
94 |
|
|
friend class swissknife::CommandMigrate; // for catalog version migration |
95 |
|
|
|
96 |
|
|
public: |
97 |
|
|
typedef std::vector<shash::Any> HashVector; |
98 |
|
|
|
99 |
|
|
/** |
100 |
|
|
* The default TTL should be shorter than the autofs idle unmount time |
101 |
|
|
* which is 5 minutes, because the config repo is accessed on every root |
102 |
|
|
* catalog refresh and we want to avoid thrashing that mountpoint. |
103 |
|
|
*/ |
104 |
|
|
static const uint64_t kDefaultTTL = 240; /**< 4 minutes default TTL */ |
105 |
|
|
|
106 |
|
|
/** |
107 |
|
|
* Note: is_nested only has an effect if parent == NULL otherwise being |
108 |
|
|
* a root catalog is determined by having a parent pointer or not. |
109 |
|
|
*/ |
110 |
|
|
Catalog(const PathString &mountpoint, |
111 |
|
|
const shash::Any &catalog_hash, |
112 |
|
|
Catalog *parent, |
113 |
|
|
const bool is_nested = false); |
114 |
|
|
virtual ~Catalog(); |
115 |
|
|
|
116 |
|
|
static Catalog *AttachFreely(const std::string &imaginary_mountpoint, |
117 |
|
|
const std::string &file, |
118 |
|
|
const shash::Any &catalog_hash, |
119 |
|
|
Catalog *parent = NULL, |
120 |
|
|
const bool is_nested = false); |
121 |
|
|
|
122 |
|
|
bool OpenDatabase(const std::string &db_path); |
123 |
|
|
|
124 |
|
154 |
inline bool LookupPath(const PathString &path, DirectoryEntry *dirent) const { |
125 |
|
154 |
return LookupMd5Path(NormalizePath(path), dirent); |
126 |
|
|
} |
127 |
|
|
bool LookupRawSymlink(const PathString &path, LinkString *raw_symlink) const; |
128 |
|
1 |
bool LookupXattrsPath(const PathString &path, XattrList *xattrs) const { |
129 |
|
1 |
return LookupXattrsMd5Path(NormalizePath(path), xattrs); |
130 |
|
|
} |
131 |
|
|
|
132 |
|
30 |
inline bool ListingPath(const PathString &path, |
133 |
|
|
DirectoryEntryList *listing, |
134 |
|
|
const bool expand_symlink = true) const |
135 |
|
|
{ |
136 |
|
30 |
return ListingMd5Path(NormalizePath(path), listing, expand_symlink); |
137 |
|
|
} |
138 |
|
4 |
bool ListingPathStat(const PathString &path, |
139 |
|
|
StatEntryList *listing) const |
140 |
|
|
{ |
141 |
|
4 |
return ListingMd5PathStat(NormalizePath(path), listing); |
142 |
|
|
} |
143 |
|
|
bool AllChunksBegin(); |
144 |
|
|
bool AllChunksNext(shash::Any *hash, zlib::Algorithms *compression_alg); |
145 |
|
|
bool AllChunksEnd(); |
146 |
|
|
|
147 |
|
1 |
inline bool ListPathChunks(const PathString &path, |
148 |
|
|
const shash::Algorithms interpret_hashes_as, |
149 |
|
|
FileChunkList *chunks) const |
150 |
|
|
{ |
151 |
|
1 |
return ListMd5PathChunks(NormalizePath(path), interpret_hashes_as, chunks); |
152 |
|
|
} |
153 |
|
|
|
154 |
|
|
CatalogList GetChildren() const; |
155 |
|
|
Catalog* FindSubtree(const PathString &path) const; |
156 |
|
|
Catalog* FindChild(const PathString &mountpoint) const; |
157 |
|
|
void AddChild(Catalog *child); |
158 |
|
|
void RemoveChild(Catalog *child); |
159 |
|
|
|
160 |
|
|
const HashVector& GetReferencedObjects() const; |
161 |
|
|
void TakeDatabaseFileOwnership(); |
162 |
|
|
void DropDatabaseFileOwnership(); |
163 |
|
5 |
bool OwnsDatabaseFile() const { |
164 |
✓✓✓✓ ✗✓ |
5 |
return ((database_ != NULL) && database_->OwnsFile()) || managed_database_; |
165 |
|
|
} |
166 |
|
|
|
167 |
|
|
uint64_t GetTTL() const; |
168 |
|
|
bool HasExplicitTTL() const; |
169 |
|
|
uint64_t GetRevision() const; |
170 |
|
|
bool GetVOMSAuthz(std::string *authz) const; |
171 |
|
|
uint64_t GetLastModified() const; |
172 |
|
|
uint64_t GetNumEntries() const; |
173 |
|
|
uint64_t GetNumChunks() const; |
174 |
|
|
shash::Any GetPreviousRevision() const; |
175 |
|
44 |
const Counters& GetCounters() const { return counters_; } |
176 |
|
|
std::string PrintMemStatistics() const; |
177 |
|
|
|
178 |
|
199 |
inline float schema() const { return database().schema_version(); } |
179 |
|
731 |
inline PathString mountpoint() const { return mountpoint_; } |
180 |
|
42 |
inline Catalog* parent() const { return parent_; } |
181 |
|
70 |
inline uint64_t max_row_id() const { return max_row_id_; } |
182 |
|
69 |
inline InodeRange inode_range() const { return inode_range_; } |
183 |
|
97 |
inline void set_inode_range(const InodeRange value) { inode_range_ = value; } |
184 |
|
32 |
inline std::string database_path() const { return database_->filename(); } |
185 |
|
10 |
inline PathString root_prefix() const { return root_prefix_; } |
186 |
|
2 |
inline shash::Any hash() const { return catalog_hash_; } |
187 |
|
57 |
inline bool volatile_flag() const { return volatile_flag_; } |
188 |
|
5 |
inline uint64_t revision() const { return GetRevision(); } |
189 |
|
|
|
190 |
|
463 |
inline bool IsInitialized() const { |
191 |
✓✗✓✗
|
463 |
return inode_range_.IsInitialized() && initialized_; |
192 |
|
|
} |
193 |
|
86 |
inline bool IsRoot() const { return is_root_; } |
194 |
|
|
bool IsAutogenerated() const { |
195 |
|
|
DirectoryEntry dirent; |
196 |
|
|
assert(IsInitialized()); |
197 |
|
|
return LookupPath(PathString( |
198 |
|
|
mountpoint_.ToString() + "/.cvmfsautocatalog"), &dirent); |
199 |
|
|
} |
200 |
|
388 |
inline bool HasParent() const { return parent_ != NULL; } |
201 |
|
2 |
inline virtual bool IsWritable() const { return false; } |
202 |
|
|
|
203 |
|
|
typedef struct { |
204 |
|
|
PathString mountpoint; |
205 |
|
|
shash::Any hash; |
206 |
|
|
uint64_t size; |
207 |
✗✗✗ |
84 |
} NestedCatalog; |
208 |
|
|
typedef std::vector<NestedCatalog> NestedCatalogList; |
209 |
|
|
const NestedCatalogList& ListNestedCatalogs() const; |
210 |
|
|
const NestedCatalogList ListOwnNestedCatalogs() const; |
211 |
|
|
bool FindNested(const PathString &mountpoint, |
212 |
|
|
shash::Any *hash, uint64_t *size) const; |
213 |
|
|
|
214 |
|
|
void SetInodeAnnotation(InodeAnnotation *new_annotation); |
215 |
|
|
inode_t GetMangledInode(const uint64_t row_id, |
216 |
|
|
const uint64_t hardlink_group) const; |
217 |
|
|
|
218 |
|
|
void SetOwnerMaps(const OwnerMap *uid_map, const OwnerMap *gid_map); |
219 |
|
199 |
uint64_t MapUid(const uint64_t uid) const { |
220 |
✗✓ |
199 |
if (uid_map_) { return uid_map_->Map(uid); } |
221 |
|
199 |
return uid; |
222 |
|
|
} |
223 |
|
199 |
uint64_t MapGid(const uint64_t gid) const { |
224 |
✗✓ |
199 |
if (gid_map_) { return gid_map_->Map(gid); } |
225 |
|
199 |
return gid; |
226 |
|
|
} |
227 |
|
|
|
228 |
|
|
protected: |
229 |
|
|
typedef std::map<uint64_t, inode_t> HardlinkGroupMap; |
230 |
|
|
mutable HardlinkGroupMap hardlink_groups_; |
231 |
|
|
|
232 |
|
|
pthread_mutex_t *lock_; |
233 |
|
|
|
234 |
|
|
bool InitStandalone(const std::string &database_file); |
235 |
|
|
bool ReadCatalogCounters(); |
236 |
|
|
|
237 |
|
|
/** |
238 |
|
|
* Specifies the SQLite open flags. Overwritten by r/w catalog. |
239 |
|
|
*/ |
240 |
|
61 |
virtual CatalogDatabase::OpenMode DatabaseOpenMode() const { |
241 |
|
61 |
return CatalogDatabase::kOpenReadOnly; |
242 |
|
|
} |
243 |
|
|
|
244 |
|
|
virtual void InitPreparedStatements(); |
245 |
|
|
void FinalizePreparedStatements(); |
246 |
|
|
|
247 |
|
|
Counters& GetWritableCounters() { return counters_; } |
248 |
|
|
|
249 |
|
355 |
inline const CatalogDatabase &database() const { return *database_; } |
250 |
|
2118 |
inline CatalogDatabase &database() { return *database_; } |
251 |
|
35 |
inline void set_parent(Catalog *catalog) { parent_ = catalog; } |
252 |
|
|
|
253 |
|
|
void ResetNestedCatalogCacheUnprotected(); |
254 |
|
|
|
255 |
|
|
private: |
256 |
|
|
typedef std::map<PathString, Catalog*> NestedCatalogMap; |
257 |
|
|
|
258 |
|
|
/** |
259 |
|
|
* The hash of the empty string. Used to identify the root entry of a |
260 |
|
|
* repository, which is the child transition point of a bind mountpoint. |
261 |
|
|
*/ |
262 |
|
|
static const shash::Md5 kMd5PathEmpty; |
263 |
|
|
|
264 |
|
|
enum VomsAuthzStatus { |
265 |
|
|
kVomsUnknown, // Not yet looked up |
266 |
|
|
kVomsNone, // No voms_authz key in properties table |
267 |
|
|
kVomsPresent, // voms_authz property available |
268 |
|
|
}; |
269 |
|
|
|
270 |
|
|
shash::Md5 NormalizePath(const PathString &path) const; |
271 |
|
|
PathString NormalizePath2(const PathString &path) const; |
272 |
|
|
PathString PlantPath(const PathString &path) const; |
273 |
|
|
|
274 |
|
|
void FixTransitionPoint(const shash::Md5 &md5path, |
275 |
|
|
DirectoryEntry *dirent) const; |
276 |
|
|
|
277 |
|
|
bool LookupMd5Path(const shash::Md5 &md5path, DirectoryEntry *dirent) const; |
278 |
|
|
bool LookupXattrsMd5Path(const shash::Md5 &md5path, XattrList *xattrs) const; |
279 |
|
|
bool ListMd5PathChunks(const shash::Md5 &md5path, |
280 |
|
|
const shash::Algorithms interpret_hashes_as, |
281 |
|
|
FileChunkList *chunks) const; |
282 |
|
|
bool ListingMd5Path(const shash::Md5 &md5path, |
283 |
|
|
DirectoryEntryList *listing, |
284 |
|
|
const bool expand_symlink = true) const; |
285 |
|
|
bool ListingMd5PathStat(const shash::Md5 &md5path, |
286 |
|
|
StatEntryList *listing) const; |
287 |
|
|
bool LookupEntry(const shash::Md5 &md5path, const bool expand_symlink, |
288 |
|
|
DirectoryEntry *dirent) const; |
289 |
|
|
|
290 |
|
|
CatalogDatabase *database_; |
291 |
|
|
|
292 |
|
|
const shash::Any catalog_hash_; |
293 |
|
|
PathString root_prefix_; |
294 |
|
|
/** |
295 |
|
|
* Normally, catalogs are mounted at their root_prefix_. But for the structure |
296 |
|
|
* under /.cvmfs/snapshots/..., that's not the case. |
297 |
|
|
*/ |
298 |
|
|
PathString mountpoint_; |
299 |
|
|
/** |
300 |
|
|
* True, iff root_prefix_ == mountpoint_ |
301 |
|
|
*/ |
302 |
|
|
bool is_regular_mountpoint_; |
303 |
|
|
bool volatile_flag_; |
304 |
|
|
/** |
305 |
|
|
* For catalogs in a catalog manager: doesn't have a parent catalog |
306 |
|
|
*/ |
307 |
|
|
const bool is_root_; |
308 |
|
|
bool managed_database_; |
309 |
|
|
|
310 |
|
|
Catalog *parent_; |
311 |
|
|
NestedCatalogMap children_; |
312 |
|
|
mutable NestedCatalogList nested_catalog_cache_; |
313 |
|
|
mutable bool nested_catalog_cache_dirty_; |
314 |
|
|
|
315 |
|
|
mutable VomsAuthzStatus voms_authz_status_; |
316 |
|
|
mutable std::string voms_authz_; |
317 |
|
|
|
318 |
|
|
bool initialized_; |
319 |
|
|
InodeRange inode_range_; |
320 |
|
|
uint64_t max_row_id_; |
321 |
|
|
InodeAnnotation *inode_annotation_; |
322 |
|
|
Counters counters_; |
323 |
|
|
// Point to the maps in the catalog manager |
324 |
|
|
const OwnerMap *uid_map_; |
325 |
|
|
const OwnerMap *gid_map_; |
326 |
|
|
|
327 |
|
|
SqlListing *sql_listing_; |
328 |
|
|
SqlLookupPathHash *sql_lookup_md5path_; |
329 |
|
|
SqlNestedCatalogLookup *sql_lookup_nested_; |
330 |
|
|
SqlNestedCatalogListing *sql_list_nested_; |
331 |
|
|
SqlOwnNestedCatalogListing *sql_own_list_nested_; |
332 |
|
|
SqlAllChunks *sql_all_chunks_; |
333 |
|
|
SqlChunksListing *sql_chunks_listing_; |
334 |
|
|
SqlLookupXattrs *sql_lookup_xattrs_; |
335 |
|
|
|
336 |
|
|
mutable HashVector referenced_hashes_; |
337 |
|
|
}; // class Catalog |
338 |
|
|
|
339 |
|
|
} // namespace catalog |
340 |
|
|
|
341 |
|
|
#endif // CVMFS_CATALOG_H_ |