1 |
|
|
/** |
2 |
|
|
* This file is part of the CernVM File System. |
3 |
|
|
*/ |
4 |
|
|
#include "cvmfs_config.h" |
5 |
|
|
#include "catalog_virtual.h" |
6 |
|
|
|
7 |
|
|
#include <algorithm> |
8 |
|
|
#include <cassert> |
9 |
|
|
#include <cstdlib> |
10 |
|
|
|
11 |
|
|
#include "catalog_mgr_rw.h" |
12 |
|
|
#include "compression.h" |
13 |
|
|
#include "history.h" |
14 |
|
|
#include "logging.h" |
15 |
|
|
#include "swissknife_history.h" |
16 |
|
|
#include "swissknife_sync.h" |
17 |
|
|
#include "util/pointer.h" |
18 |
|
|
#include "util/posix.h" |
19 |
|
|
#include "util/string.h" |
20 |
|
|
#include "xattr.h" |
21 |
|
|
|
22 |
|
|
using namespace std; // NOLINT |
23 |
|
|
|
24 |
|
|
namespace catalog { |
25 |
|
|
|
26 |
|
|
const char *VirtualCatalog::kVirtualPath = ".cvmfs"; |
27 |
|
|
const char *VirtualCatalog::kSnapshotDirectory = "snapshots"; |
28 |
|
|
const unsigned VirtualCatalog::kActionNone = 0x00; |
29 |
|
|
const unsigned VirtualCatalog::kActionGenerateSnapshots = 0x01; |
30 |
|
|
const unsigned VirtualCatalog::kActionRemove = 0x02; |
31 |
|
|
|
32 |
|
|
|
33 |
|
|
void VirtualCatalog::CreateBaseDirectory() { |
34 |
|
|
// Add /.cvmfs as a nested catalog |
35 |
|
|
DirectoryEntryBase entry_dir; |
36 |
|
|
entry_dir.name_ = NameString(string(kVirtualPath)); |
37 |
|
|
entry_dir.mode_ = S_IFDIR | |
38 |
|
|
S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; |
39 |
|
|
entry_dir.uid_ = 0; |
40 |
|
|
entry_dir.gid_ = 0; |
41 |
|
|
entry_dir.size_ = 97; |
42 |
|
|
entry_dir.mtime_ = time(NULL); |
43 |
|
|
catalog_mgr_->AddDirectory(entry_dir, ""); |
44 |
|
|
WritableCatalog *parent_catalog = |
45 |
|
|
catalog_mgr_->GetHostingCatalog(kVirtualPath); |
46 |
|
|
catalog_mgr_->CreateNestedCatalog(kVirtualPath); |
47 |
|
|
WritableCatalog *virtual_catalog = |
48 |
|
|
catalog_mgr_->GetHostingCatalog(kVirtualPath); |
49 |
|
|
assert(parent_catalog != virtual_catalog); |
50 |
|
|
|
51 |
|
|
// Set hidden flag in parent catalog |
52 |
|
|
DirectoryEntry entry_parent; |
53 |
|
|
bool retval = parent_catalog->LookupPath( |
54 |
|
|
PathString("/" + string(kVirtualPath)), &entry_parent); |
55 |
|
|
assert(retval); |
56 |
|
|
entry_parent.set_is_hidden(true); |
57 |
|
|
parent_catalog->UpdateEntry(entry_parent, "/" + string(kVirtualPath)); |
58 |
|
|
|
59 |
|
|
// Set hidden flag in nested catalog |
60 |
|
|
DirectoryEntry entry_virtual; |
61 |
|
|
retval = virtual_catalog->LookupPath(PathString("/" + string(kVirtualPath)), |
62 |
|
|
&entry_virtual); |
63 |
|
|
assert(retval); |
64 |
|
|
entry_virtual.set_is_hidden(true); |
65 |
|
|
virtual_catalog->UpdateEntry(entry_virtual, "/" + string(kVirtualPath)); |
66 |
|
|
} |
67 |
|
|
|
68 |
|
|
|
69 |
|
|
void VirtualCatalog::CreateNestedCatalogMarker() { |
70 |
|
|
DirectoryEntryBase entry_marker; |
71 |
|
|
// Note that another entity needs to ensure that the object of an empty |
72 |
|
|
// file is in the repository! It is currently done by the sync_mediator. |
73 |
|
|
shash::Algorithms algorithm = catalog_mgr_->spooler_->GetHashAlgorithm(); |
74 |
|
|
shash::Any file_hash(algorithm); |
75 |
|
|
void *empty_compressed; |
76 |
|
|
uint64_t sz_empty_compressed; |
77 |
|
|
bool retval = zlib::CompressMem2Mem( |
78 |
|
|
NULL, 0, &empty_compressed, &sz_empty_compressed); |
79 |
|
|
assert(retval); |
80 |
|
|
shash::HashMem(static_cast<unsigned char *>(empty_compressed), |
81 |
|
|
sz_empty_compressed, &file_hash); |
82 |
|
|
free(empty_compressed); |
83 |
|
|
entry_marker.name_ = NameString(".cvmfscatalog"); |
84 |
|
|
entry_marker.mode_ = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH; |
85 |
|
|
entry_marker.checksum_ = file_hash; |
86 |
|
|
entry_marker.mtime_ = time(NULL); |
87 |
|
|
entry_marker.uid_ = 0; |
88 |
|
|
entry_marker.gid_ = 0; |
89 |
|
|
XattrList xattrs; |
90 |
|
|
catalog_mgr_->AddFile(entry_marker, xattrs, kVirtualPath); |
91 |
|
|
} |
92 |
|
|
|
93 |
|
|
|
94 |
|
|
void VirtualCatalog::CreateSnapshotDirectory() { |
95 |
|
|
DirectoryEntryBase entry_dir; |
96 |
|
|
entry_dir.name_ = NameString(string(kSnapshotDirectory)); |
97 |
|
|
entry_dir.mode_ = S_IFDIR | |
98 |
|
|
S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; |
99 |
|
|
entry_dir.uid_ = 0; |
100 |
|
|
entry_dir.gid_ = 0; |
101 |
|
|
entry_dir.size_ = 97; |
102 |
|
|
entry_dir.mtime_ = time(NULL); |
103 |
|
|
catalog_mgr_->AddDirectory(entry_dir, kVirtualPath); |
104 |
|
|
} |
105 |
|
|
|
106 |
|
|
|
107 |
|
|
/** |
108 |
|
|
* Checks for the top-level /.cvmfs directory and creates it as a nested catalog |
109 |
|
|
* if necessary. |
110 |
|
|
*/ |
111 |
|
|
void VirtualCatalog::EnsurePresence() { |
112 |
|
|
DirectoryEntry e; |
113 |
|
|
bool retval = catalog_mgr_->LookupPath("/" + string(kVirtualPath), |
114 |
|
|
kLookupSole, &e); |
115 |
|
|
if (!retval) { |
116 |
|
|
LogCvmfs(kLogCatalog, kLogDebug, "creating new virtual catalog"); |
117 |
|
|
CreateBaseDirectory(); |
118 |
|
|
CreateNestedCatalogMarker(); |
119 |
|
|
CreateSnapshotDirectory(); |
120 |
|
|
} |
121 |
|
|
assert(catalog_mgr_->IsTransitionPoint(kVirtualPath)); |
122 |
|
|
} |
123 |
|
|
|
124 |
|
|
|
125 |
|
|
void VirtualCatalog::Generate(int actions) { |
126 |
|
|
if (actions & kActionGenerateSnapshots) { |
127 |
|
|
GenerateSnapshots(); |
128 |
|
|
} |
129 |
|
|
if (actions & kActionRemove) { |
130 |
|
|
Remove(); |
131 |
|
|
} |
132 |
|
|
} |
133 |
|
|
|
134 |
|
|
|
135 |
|
|
void VirtualCatalog::GenerateSnapshots() { |
136 |
|
|
LogCvmfs(kLogCvmfs, kLogStdout, "Creating virtual snapshots"); |
137 |
|
|
EnsurePresence(); |
138 |
|
|
|
139 |
|
|
vector<TagId> tags_history; |
140 |
|
|
vector<TagId> tags_catalog; |
141 |
|
|
GetSortedTagsFromHistory(&tags_history); |
142 |
|
|
GetSortedTagsFromCatalog(&tags_catalog); |
143 |
|
|
// Add artifical end markers to both lists |
144 |
|
|
string tag_name_end = ""; |
145 |
|
|
if (!tags_history.empty()) |
146 |
|
|
tag_name_end = std::max(tag_name_end, tags_history.rbegin()->name); |
147 |
|
|
if (!tags_catalog.empty()) |
148 |
|
|
tag_name_end = std::max(tag_name_end, tags_catalog.rbegin()->name); |
149 |
|
|
tag_name_end += "X"; |
150 |
|
|
tags_history.push_back(TagId(tag_name_end, shash::Any())); |
151 |
|
|
tags_catalog.push_back(TagId(tag_name_end, shash::Any())); |
152 |
|
|
|
153 |
|
|
// Walk through both sorted lists concurrently and determine change set |
154 |
|
|
unsigned i_history = 0, i_catalog = 0; |
155 |
|
|
unsigned last_history = tags_history.size() - 1; |
156 |
|
|
unsigned last_catalog = tags_catalog.size() - 1; |
157 |
|
|
while ((i_history < last_history) || (i_catalog < last_catalog)) { |
158 |
|
|
TagId t_history = tags_history[i_history]; |
159 |
|
|
TagId t_catalog = tags_catalog[i_catalog]; |
160 |
|
|
|
161 |
|
|
// Both the same, nothing to do |
162 |
|
|
if (t_history == t_catalog) { |
163 |
|
|
i_history++; |
164 |
|
|
i_catalog++; |
165 |
|
|
continue; |
166 |
|
|
} |
167 |
|
|
|
168 |
|
|
// Same tag name for different hash, re-insert |
169 |
|
|
if (t_history.name == t_catalog.name) { |
170 |
|
|
RemoveSnapshot(t_catalog); |
171 |
|
|
InsertSnapshot(t_history); |
172 |
|
|
i_history++; |
173 |
|
|
i_catalog++; |
174 |
|
|
continue; |
175 |
|
|
} |
176 |
|
|
|
177 |
|
|
// New tag that's missing |
178 |
|
|
if (t_history.name < t_catalog.name) { |
179 |
|
|
InsertSnapshot(t_history); |
180 |
|
|
i_history++; |
181 |
|
|
continue; |
182 |
|
|
} |
183 |
|
|
|
184 |
|
|
// A tag was removed but it is still present in the catalog |
185 |
|
|
assert(t_history.name > t_catalog.name); |
186 |
|
|
RemoveSnapshot(t_catalog); |
187 |
|
|
i_catalog++; |
188 |
|
|
} |
189 |
|
|
} |
190 |
|
|
|
191 |
|
|
|
192 |
|
7 |
bool VirtualCatalog::ParseActions( |
193 |
|
|
const string &action_desc, |
194 |
|
|
unsigned *actions) |
195 |
|
|
{ |
196 |
|
7 |
*actions = kActionNone; |
197 |
✓✓ |
7 |
if (action_desc.empty()) |
198 |
|
1 |
return true; |
199 |
|
|
|
200 |
|
6 |
vector<string> action_tokens = SplitString(action_desc, ','); |
201 |
✓✓ |
14 |
for (unsigned i = 0; i < action_tokens.size(); ++i) { |
202 |
✓✓ |
10 |
if (action_tokens[i] == "snapshots") { |
203 |
|
4 |
*actions |= kActionGenerateSnapshots; |
204 |
✓✓ |
6 |
} else if (action_tokens[i] == "remove") { |
205 |
|
4 |
*actions |= kActionRemove; |
206 |
|
|
} else { |
207 |
|
2 |
return false; |
208 |
|
|
} |
209 |
|
|
} |
210 |
|
4 |
return true; |
211 |
|
|
} |
212 |
|
|
|
213 |
|
|
|
214 |
|
|
void VirtualCatalog::GetSortedTagsFromHistory(vector<TagId> *tags) { |
215 |
|
|
UniquePtr<history::History> history( |
216 |
|
|
assistant_.GetHistory(swissknife::Assistant::kOpenReadOnly)); |
217 |
|
|
vector<history::History::Tag> tags_history; |
218 |
|
|
bool retval = history->List(&tags_history); |
219 |
|
|
assert(retval); |
220 |
|
|
for (unsigned i = 0, l = tags_history.size(); i < l; ++i) { |
221 |
|
|
if ((tags_history[i].name == swissknife::CommandTag::kHeadTag) || |
222 |
|
|
(tags_history[i].name == swissknife::CommandTag::kPreviousHeadTag)) |
223 |
|
|
{ |
224 |
|
|
continue; |
225 |
|
|
} |
226 |
|
|
tags->push_back(TagId(tags_history[i].name, tags_history[i].root_hash)); |
227 |
|
|
} |
228 |
|
|
std::sort(tags->begin(), tags->end()); |
229 |
|
|
} |
230 |
|
|
|
231 |
|
|
|
232 |
|
|
void VirtualCatalog::GetSortedTagsFromCatalog(vector<TagId> *tags) { |
233 |
|
|
WritableCatalog *virtual_catalog = |
234 |
|
|
catalog_mgr_->GetHostingCatalog(kVirtualPath); |
235 |
|
|
assert(virtual_catalog != NULL); |
236 |
|
|
Catalog::NestedCatalogList nested_catalogs = |
237 |
|
|
virtual_catalog->ListNestedCatalogs(); |
238 |
|
|
for (unsigned i = 0, l = nested_catalogs.size(); i < l; ++i) { |
239 |
|
|
tags->push_back(TagId(GetFileName(nested_catalogs[i].mountpoint).ToString(), |
240 |
|
|
nested_catalogs[i].hash)); |
241 |
|
|
} |
242 |
|
|
std::sort(tags->begin(), tags->end()); |
243 |
|
|
} |
244 |
|
|
|
245 |
|
|
|
246 |
|
|
void VirtualCatalog::InsertSnapshot(TagId tag) { |
247 |
|
|
LogCvmfs(kLogCatalog, kLogDebug, "add snapshot %s (%s) to virtual catalog", |
248 |
|
|
tag.name.c_str(), tag.hash.ToString().c_str()); |
249 |
|
|
UniquePtr<Catalog> catalog(assistant_.GetCatalog(tag.hash, |
250 |
|
|
swissknife::Assistant::kOpenReadOnly)); |
251 |
|
|
assert(catalog.IsValid()); |
252 |
|
|
assert(catalog->root_prefix().IsEmpty()); |
253 |
|
|
DirectoryEntry entry_root; |
254 |
|
|
bool retval = catalog->LookupPath(PathString(""), &entry_root); |
255 |
|
|
assert(retval); |
256 |
|
|
|
257 |
|
|
// Add directory entry |
258 |
|
|
DirectoryEntryBase entry_dir = entry_root; |
259 |
|
|
entry_dir.name_ = NameString(tag.name); |
260 |
|
|
catalog_mgr_->AddDirectory( |
261 |
|
|
entry_dir, string(kVirtualPath) + "/" + string(kSnapshotDirectory)); |
262 |
|
|
|
263 |
|
|
// Set "bind mount" flag |
264 |
|
|
WritableCatalog *virtual_catalog = |
265 |
|
|
catalog_mgr_->GetHostingCatalog(kVirtualPath); |
266 |
|
|
assert(virtual_catalog != NULL); |
267 |
|
|
string mountpoint = |
268 |
|
|
"/" + string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + |
269 |
|
|
tag.name; |
270 |
|
|
DirectoryEntry entry_bind_mountpoint(entry_dir); |
271 |
|
|
entry_bind_mountpoint.set_is_bind_mountpoint(true); |
272 |
|
|
virtual_catalog->UpdateEntry(entry_bind_mountpoint, mountpoint); |
273 |
|
|
|
274 |
|
|
// Register nested catalog |
275 |
|
|
uint64_t catalog_size = GetFileSize(catalog->database_path()); |
276 |
|
|
assert(catalog_size > 0); |
277 |
|
|
virtual_catalog->InsertBindMountpoint(mountpoint, tag.hash, catalog_size); |
278 |
|
|
} |
279 |
|
|
|
280 |
|
|
|
281 |
|
|
void VirtualCatalog::Remove() { |
282 |
|
|
LogCvmfs(kLogCvmfs, kLogStdout, "Removing .cvmfs virtual catalog"); |
283 |
|
|
|
284 |
|
|
// Safety check, make sure we don't remove the entire repository |
285 |
|
|
WritableCatalog *virtual_catalog = |
286 |
|
|
catalog_mgr_->GetHostingCatalog(kVirtualPath); |
287 |
|
|
assert(!virtual_catalog->IsRoot()); |
288 |
|
|
DirectoryEntry entry_virtual; |
289 |
|
|
bool retval = catalog_mgr_->LookupPath( |
290 |
|
|
PathString("/" + string(kVirtualPath)), kLookupSole, &entry_virtual); |
291 |
|
|
assert(retval); |
292 |
|
|
assert(entry_virtual.IsHidden()); |
293 |
|
|
|
294 |
|
|
RemoveRecursively(kVirtualPath); |
295 |
|
|
catalog_mgr_->RemoveNestedCatalog(kVirtualPath); |
296 |
|
|
catalog_mgr_->RemoveDirectory(kVirtualPath); |
297 |
|
|
} |
298 |
|
|
|
299 |
|
|
|
300 |
|
|
void VirtualCatalog::RemoveRecursively(const string &directory) { |
301 |
|
|
DirectoryEntryList listing; |
302 |
|
|
bool retval = catalog_mgr_->Listing(PathString("/" + directory), &listing); |
303 |
|
|
assert(retval); |
304 |
|
|
for (unsigned i = 0; i < listing.size(); ++i) { |
305 |
|
|
string this_path = directory + "/" + listing[i].name().ToString(); |
306 |
|
|
if (listing[i].IsDirectory()) { |
307 |
|
|
if (!listing[i].IsBindMountpoint()) |
308 |
|
|
RemoveRecursively(this_path); |
309 |
|
|
catalog_mgr_->RemoveDirectory(this_path); |
310 |
|
|
} else if (listing[i].IsRegular()) { |
311 |
|
|
assert(listing[i].name().ToString() == ".cvmfscatalog"); |
312 |
|
|
catalog_mgr_->RemoveFile(this_path); |
313 |
|
|
} else { |
314 |
|
|
abort(); |
315 |
|
|
} |
316 |
|
|
} |
317 |
|
|
} |
318 |
|
|
|
319 |
|
|
|
320 |
|
|
void VirtualCatalog::RemoveSnapshot(TagId tag) { |
321 |
|
|
LogCvmfs(kLogCatalog, kLogDebug, |
322 |
|
|
"remove snapshot %s (%s) from virtual catalog", |
323 |
|
|
tag.name.c_str(), tag.hash.ToString().c_str()); |
324 |
|
|
string tag_dir = |
325 |
|
|
string(kVirtualPath) + "/" + string(kSnapshotDirectory) + "/" + tag.name; |
326 |
|
|
catalog_mgr_->RemoveDirectory(tag_dir); |
327 |
|
|
|
328 |
|
|
WritableCatalog *virtual_catalog = |
329 |
|
|
catalog_mgr_->GetHostingCatalog(kVirtualPath); |
330 |
|
|
assert(virtual_catalog != NULL); |
331 |
|
|
virtual_catalog->RemoveBindMountpoint("/" + tag_dir); |
332 |
|
|
} |
333 |
|
|
|
334 |
|
|
|
335 |
|
|
VirtualCatalog::VirtualCatalog( |
336 |
|
|
manifest::Manifest *m, |
337 |
|
|
download::DownloadManager *d, |
338 |
|
|
catalog::WritableCatalogManager *c, |
339 |
|
|
SyncParameters *p) |
340 |
|
|
: catalog_mgr_(c) |
341 |
|
|
, assistant_(d, m, p->stratum0, p->dir_temp) |
342 |
|
|
{ } |
343 |
|
|
|
344 |
|
|
} // namespace catalog |