1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System. |
3 |
|
|
* |
4 |
|
|
* A WritableCatalogManager adds write functionality to the catalog |
5 |
|
|
* manager. It is used for the server side of CVMFS |
6 |
|
|
* All nested catalog loading functionality is inherited from |
7 |
|
|
* AbstractCatalogManager. |
8 |
|
|
* |
9 |
|
|
* The WritableCatalogManager is provided with DirectoryEntryBase objects from |
10 |
|
|
* the underlying sync infrastructure (namely SyncMediator) on the server side |
11 |
|
|
* of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con- |
12 |
|
|
* tains only pure file system specific meta data (i.e. mtime, mode, filename). |
13 |
|
|
* WritableCatalogManager is responsible for the addition and management of all |
14 |
|
|
* CVMFS-specific meta data in a full DirectoryEntry, which is then saved into |
15 |
|
|
* the actual Catalog databases. |
16 |
|
|
* |
17 |
|
|
* The inode assignment is based on the fact that the number of entries in a |
18 |
|
|
* catalog do not change (expect on reload). As we do exactly that with the |
19 |
|
|
* WritableCatalogManager here, inode numbers derived from WritableCatalogs |
20 |
|
|
* and the WritableCatalogManager may (and will) be screwed. This is not an |
21 |
|
|
* issue in the current implementation, as they are not used in the synching |
22 |
|
|
* process. Just keep in mind. |
23 |
|
|
* |
24 |
|
|
* The WritableCatalogManager starts with a base repository (given by the |
25 |
|
|
* root hash), and downloads and uncompresses all required catalogs into |
26 |
|
|
* temporary storage. |
27 |
|
|
*/ |
28 |
|
|
|
29 |
|
|
#ifndef CVMFS_CATALOG_MGR_RW_H_ |
30 |
|
|
#define CVMFS_CATALOG_MGR_RW_H_ |
31 |
|
|
|
32 |
|
|
#include <pthread.h> |
33 |
|
|
#include <stdint.h> |
34 |
|
|
|
35 |
|
|
#include <map> |
36 |
|
|
#include <set> |
37 |
|
|
#include <string> |
38 |
|
|
|
39 |
|
|
#include "catalog_mgr_ro.h" |
40 |
|
|
#include "catalog_rw.h" |
41 |
|
|
#include "file_chunk.h" |
42 |
|
|
#include "upload_spooler_result.h" |
43 |
|
|
#include "util_concurrency.h" |
44 |
|
|
#include "xattr.h" |
45 |
|
|
|
46 |
|
|
class XattrList; |
47 |
|
|
namespace upload { |
48 |
|
|
class Spooler; |
49 |
|
|
} |
50 |
|
|
|
51 |
|
|
namespace download { |
52 |
|
|
class DownloadManager; |
53 |
|
|
} |
54 |
|
|
|
55 |
|
|
namespace manifest { |
56 |
|
|
class Manifest; |
57 |
|
|
} |
58 |
|
|
|
59 |
|
|
namespace perf { |
60 |
|
|
class Statistics; |
61 |
|
|
} |
62 |
|
|
|
63 |
|
|
namespace catalog { |
64 |
|
|
template <class CatalogMgrT> |
65 |
|
|
class CatalogBalancer; |
66 |
|
|
} |
67 |
|
|
|
68 |
|
|
namespace catalog { |
69 |
|
|
|
70 |
|
|
class WritableCatalogManager : public SimpleCatalogManager { |
71 |
|
|
friend class CatalogBalancer<WritableCatalogManager>; |
72 |
|
|
// TODO(jblomer): only needed to get Spooler's hash algorithm. Remove me |
73 |
|
|
// after refactoring of the swissknife utility. |
74 |
|
|
friend class VirtualCatalog; |
75 |
|
|
|
76 |
|
|
public: |
77 |
|
|
WritableCatalogManager(const shash::Any &base_hash, |
78 |
|
|
const std::string &stratum0, |
79 |
|
|
const std::string &dir_temp, |
80 |
|
|
upload::Spooler *spooler, |
81 |
|
|
download::DownloadManager *download_manager, |
82 |
|
|
bool enforce_limits, |
83 |
|
|
const unsigned nested_kcatalog_limit, |
84 |
|
|
const unsigned root_kcatalog_limit, |
85 |
|
|
const unsigned file_mbyte_limit, |
86 |
|
|
perf::Statistics *statistics, |
87 |
|
|
bool is_balanceable, |
88 |
|
|
unsigned max_weight, |
89 |
|
|
unsigned min_weight); |
90 |
|
|
~WritableCatalogManager(); |
91 |
|
|
static manifest::Manifest *CreateRepository(const std::string &dir_temp, |
92 |
|
|
const bool volatile_content, |
93 |
|
|
const std::string &voms_authz, |
94 |
|
|
upload::Spooler *spooler); |
95 |
|
|
|
96 |
|
|
// DirectoryEntry handling |
97 |
|
39 |
void AddFile(const DirectoryEntryBase &entry, |
98 |
|
|
const XattrList &xattrs, |
99 |
|
|
const std::string &parent_directory) |
100 |
|
|
{ |
101 |
|
39 |
AddFile(DirectoryEntry(entry), xattrs, parent_directory); |
102 |
|
39 |
} |
103 |
|
|
void AddChunkedFile(const DirectoryEntryBase &entry, |
104 |
|
|
const XattrList &xattrs, |
105 |
|
|
const std::string &parent_directory, |
106 |
|
|
const FileChunkList &file_chunks); |
107 |
|
|
void RemoveFile(const std::string &file_path); |
108 |
|
|
|
109 |
|
|
void AddDirectory(const DirectoryEntryBase &entry, |
110 |
|
|
const std::string &parent_directory); |
111 |
|
|
void TouchDirectory(const DirectoryEntryBase &entry, |
112 |
|
|
const std::string &directory_path); |
113 |
|
|
void RemoveDirectory(const std::string &directory_path); |
114 |
|
|
void Clone(const std::string from, const std::string to); |
115 |
|
|
|
116 |
|
|
// Hardlink group handling |
117 |
|
|
void AddHardlinkGroup(const DirectoryEntryBaseList &entries, |
118 |
|
|
const XattrList &xattrs, |
119 |
|
|
const std::string &parent_directory, |
120 |
|
|
const FileChunkList &file_chunks); |
121 |
|
|
void ShrinkHardlinkGroup(const std::string &remove_path); |
122 |
|
|
|
123 |
|
|
// Nested catalog handling |
124 |
|
|
void CreateNestedCatalog(const std::string &mountpoint); |
125 |
|
|
void RemoveNestedCatalog(const std::string &mountpoint, |
126 |
|
|
const bool merge = true); |
127 |
|
|
bool IsTransitionPoint(const std::string &mountpoint); |
128 |
|
|
WritableCatalog *GetHostingCatalog(const std::string &path); |
129 |
|
|
|
130 |
|
|
inline bool IsBalanceable() const { return is_balanceable_; } |
131 |
|
|
/** |
132 |
|
|
* TODO |
133 |
|
|
*/ |
134 |
|
|
void PrecalculateListings(); |
135 |
|
|
|
136 |
|
|
void SetTTL(const uint64_t new_ttl); |
137 |
|
|
bool SetVOMSAuthz(const std::string &voms_authz); |
138 |
|
|
bool Commit(const bool stop_for_tweaks, |
139 |
|
|
const uint64_t manual_revision, |
140 |
|
|
manifest::Manifest *manifest); |
141 |
|
|
|
142 |
|
|
void Balance() { |
143 |
|
|
if (IsBalanceable()) { |
144 |
|
|
DoBalance(); |
145 |
|
|
} else { |
146 |
|
|
LogCvmfs(kLogCatalog, kLogVerboseMsg, "Not balancing the catalog " |
147 |
|
|
"manager because it is not balanceable"); |
148 |
|
|
} |
149 |
|
|
} |
150 |
|
|
|
151 |
|
|
protected: |
152 |
|
8 |
void EnforceSqliteMemLimit() { } |
153 |
|
|
|
154 |
|
|
Catalog *CreateCatalog(const PathString &mountpoint, |
155 |
|
|
const shash::Any &catalog_hash, |
156 |
|
|
Catalog *parent_catalog); |
157 |
|
|
void ActivateCatalog(Catalog *catalog); |
158 |
|
|
|
159 |
|
|
void AddFile(const DirectoryEntry &entry, |
160 |
|
|
const XattrList &xattrs, |
161 |
|
|
const std::string &parent_directory); |
162 |
|
|
|
163 |
|
|
private: |
164 |
|
|
bool FindCatalog(const std::string &path, |
165 |
|
|
WritableCatalog **result, |
166 |
|
|
DirectoryEntry *dirent = NULL); |
167 |
|
|
void DoBalance(); |
168 |
|
|
void FixWeight(WritableCatalog *catalog); |
169 |
|
|
|
170 |
|
24 |
struct CatalogInfo { |
171 |
|
|
uint64_t ttl; |
172 |
|
|
size_t size; |
173 |
|
|
shash::Any content_hash; |
174 |
|
|
unsigned int revision; |
175 |
|
|
}; |
176 |
|
|
|
177 |
|
|
struct CatalogUploadContext { |
178 |
|
|
Future<CatalogInfo>* root_catalog_info; |
179 |
|
|
bool stop_for_tweaks; |
180 |
|
|
}; |
181 |
|
|
|
182 |
|
|
CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks); |
183 |
|
|
void FinalizeCatalog(WritableCatalog *catalog, |
184 |
|
|
const bool stop_for_tweaks); |
185 |
|
|
void ScheduleCatalogProcessing(WritableCatalog *catalog); |
186 |
|
|
|
187 |
|
8 |
void GetModifiedCatalogLeafs(WritableCatalogList *result) const { |
188 |
|
|
const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(), |
189 |
|
8 |
result); |
190 |
✗✓ |
8 |
assert(dirty); |
191 |
|
8 |
} |
192 |
|
|
bool GetModifiedCatalogLeafsRecursively(Catalog *catalog, |
193 |
|
|
WritableCatalogList *result) const; |
194 |
|
|
|
195 |
|
|
void CatalogUploadCallback(const upload::SpoolerResult &result, |
196 |
|
|
const CatalogUploadContext clg_upload_context); |
197 |
|
|
|
198 |
|
|
private: |
199 |
|
106 |
inline void SyncLock() { pthread_mutex_lock(sync_lock_); } |
200 |
|
106 |
inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); } |
201 |
|
|
|
202 |
|
|
//**************************************************************************** |
203 |
|
|
// Workaround -- Serialized Catalog Committing |
204 |
|
|
void GetModifiedCatalogs(WritableCatalogList *result) const { |
205 |
|
|
const unsigned int number_of_dirty_catalogs = |
206 |
|
|
GetModifiedCatalogsRecursively(GetRootCatalog(), result); |
207 |
|
|
assert(number_of_dirty_catalogs <= result->size()); |
208 |
|
|
} |
209 |
|
|
int GetModifiedCatalogsRecursively(const Catalog *catalog, |
210 |
|
|
WritableCatalogList *result) const; |
211 |
|
|
void CatalogUploadSerializedCallback( |
212 |
|
|
const upload::SpoolerResult &result, |
213 |
|
|
const CatalogUploadContext unused); |
214 |
|
|
CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks); |
215 |
|
|
//**************************************************************************** |
216 |
|
|
|
217 |
|
|
// defined in catalog_mgr_rw.cc |
218 |
|
|
static const std::string kCatalogFilename; |
219 |
|
|
|
220 |
|
|
// private lock of WritableCatalogManager |
221 |
|
|
pthread_mutex_t *sync_lock_; |
222 |
|
|
upload::Spooler *spooler_; |
223 |
|
|
|
224 |
|
|
pthread_mutex_t *catalog_processing_lock_; |
225 |
|
|
std::map<std::string, WritableCatalog*> catalog_processing_map_; |
226 |
|
|
|
227 |
|
|
// TODO(jblomer): catalog limits should become its own struct |
228 |
|
|
bool enforce_limits_; |
229 |
|
|
unsigned nested_kcatalog_limit_; |
230 |
|
|
unsigned root_kcatalog_limit_; |
231 |
|
|
unsigned file_mbyte_limit_; |
232 |
|
|
|
233 |
|
|
/** |
234 |
|
|
* Directories don't have extended attributes at this point. |
235 |
|
|
*/ |
236 |
|
|
XattrList empty_xattrs; |
237 |
|
|
|
238 |
|
|
/** |
239 |
|
|
* It indicates whether this catalog manager supports balancing operations |
240 |
|
|
*/ |
241 |
|
|
const bool is_balanceable_; |
242 |
|
|
|
243 |
|
|
/** |
244 |
|
|
* Defines the maximum weight an autogenerated catalog can have. If after a |
245 |
|
|
* publishing operation the catalog's weight is greater than this threshold it |
246 |
|
|
* will be considered overflowed and will automatically be split in different |
247 |
|
|
* sub-catalogs. |
248 |
|
|
*/ |
249 |
|
|
const unsigned max_weight_; |
250 |
|
|
|
251 |
|
|
/** |
252 |
|
|
* Defines the minimum weight an autogenerated catalog can have. If after a |
253 |
|
|
* publishing operation the catalog's weight is lesser than this threshold it |
254 |
|
|
* will be considered underflowed and will automatically be merged with its |
255 |
|
|
* parent. |
256 |
|
|
* This last operation can provoke an overflow in the parent, though. |
257 |
|
|
*/ |
258 |
|
|
const unsigned min_weight_; |
259 |
|
|
|
260 |
|
|
/** |
261 |
|
|
* Defines the threshold that will be used to balance a catalog that has been |
262 |
|
|
* overflowed. Its value should be lesser than max_weight_ and greater than |
263 |
|
|
* min_weight. By default it is set to max_weight / 2. |
264 |
|
|
*/ |
265 |
|
|
const unsigned balance_weight_; |
266 |
|
|
}; // class WritableCatalogManager |
267 |
|
|
|
268 |
|
|
} // namespace catalog |
269 |
|
|
|
270 |
|
|
#endif // CVMFS_CATALOG_MGR_RW_H_ |