GCC Code Coverage Report
Directory: cvmfs/ Exec Total Coverage
File: cvmfs/catalog_mgr_rw.h Lines: 11 19 57.9 %
Date: 2019-02-03 02:48:13 Branches: 1 6 16.7 %

Line Branch Exec Source
1
/**
2
 * This file is part of the CernVM File System.
3
 *
4
 * A WritableCatalogManager adds write functionality to the catalog
5
 * manager.  It is used for the server side of CVMFS
6
 * All nested catalog loading functionality is inherited from
7
 * AbstractCatalogManager.
8
 *
9
 * The WritableCatalogManager is provided with DirectoryEntryBase objects from
10
 * the underlying sync infrastructure (namely SyncMediator) on the server side
11
 * of CVMFS. In contrast to a full DirectoryEntry object DirectoryEntryBase con-
12
 * tains only pure file system specific meta data (i.e. mtime, mode, filename).
13
 * WritableCatalogManager is responsible for the addition and management of all
14
 * CVMFS-specific meta data in a full DirectoryEntry, which is then saved into
15
 * the actual Catalog databases.
16
 *
17
 * The inode assignment is based on the fact that the number of entries in a
18
 * catalog do not change (expect on reload). As we do exactly that with the
19
 * WritableCatalogManager here, inode numbers derived from WritableCatalogs
20
 * and the WritableCatalogManager may (and will) be screwed.  This is not an
21
 * issue in the current implementation, as they are not used in the synching
22
 * process.  Just keep in mind.
23
 *
24
 * The WritableCatalogManager starts with a base repository (given by the
25
 * root hash), and downloads and uncompresses all required catalogs into
26
 * temporary storage.
27
 */
28
29
#ifndef CVMFS_CATALOG_MGR_RW_H_
30
#define CVMFS_CATALOG_MGR_RW_H_
31
32
#include <pthread.h>
33
#include <stdint.h>
34
35
#include <map>
36
#include <set>
37
#include <string>
38
39
#include "catalog_mgr_ro.h"
40
#include "catalog_rw.h"
41
#include "file_chunk.h"
42
#include "upload_spooler_result.h"
43
#include "util_concurrency.h"
44
#include "xattr.h"
45
46
class XattrList;
47
namespace upload {
48
class Spooler;
49
}
50
51
namespace download {
52
class DownloadManager;
53
}
54
55
namespace manifest {
56
class Manifest;
57
}
58
59
namespace perf {
60
class Statistics;
61
}
62
63
namespace catalog {
64
template <class CatalogMgrT>
65
class CatalogBalancer;
66
}
67
68
namespace catalog {
69
70
class WritableCatalogManager : public SimpleCatalogManager {
71
  friend class CatalogBalancer<WritableCatalogManager>;
72
  // TODO(jblomer): only needed to get Spooler's hash algorithm.  Remove me
73
  // after refactoring of the swissknife utility.
74
  friend class VirtualCatalog;
75
76
 public:
77
  WritableCatalogManager(const shash::Any  &base_hash,
78
                         const std::string &stratum0,
79
                         const std::string &dir_temp,
80
                         upload::Spooler   *spooler,
81
                         download::DownloadManager *download_manager,
82
                         bool enforce_limits,
83
                         const unsigned nested_kcatalog_limit,
84
                         const unsigned root_kcatalog_limit,
85
                         const unsigned file_mbyte_limit,
86
                         perf::Statistics *statistics,
87
                         bool is_balanceable,
88
                         unsigned max_weight,
89
                         unsigned min_weight);
90
  ~WritableCatalogManager();
91
  static manifest::Manifest *CreateRepository(const std::string &dir_temp,
92
                                              const bool volatile_content,
93
                                              const std::string &voms_authz,
94
                                              upload::Spooler   *spooler);
95
96
  // DirectoryEntry handling
97
39
  void AddFile(const DirectoryEntryBase &entry,
98
               const XattrList &xattrs,
99
               const std::string &parent_directory)
100
  {
101
39
    AddFile(DirectoryEntry(entry), xattrs, parent_directory);
102
39
  }
103
  void AddChunkedFile(const DirectoryEntryBase &entry,
104
                      const XattrList &xattrs,
105
                      const std::string &parent_directory,
106
                      const FileChunkList &file_chunks);
107
  void RemoveFile(const std::string &file_path);
108
109
  void AddDirectory(const DirectoryEntryBase &entry,
110
                    const std::string &parent_directory);
111
  void TouchDirectory(const DirectoryEntryBase &entry,
112
                      const std::string &directory_path);
113
  void RemoveDirectory(const std::string &directory_path);
114
  void Clone(const std::string from, const std::string to);
115
116
  // Hardlink group handling
117
  void AddHardlinkGroup(const DirectoryEntryBaseList &entries,
118
                        const XattrList &xattrs,
119
                        const std::string &parent_directory,
120
                        const FileChunkList &file_chunks);
121
  void ShrinkHardlinkGroup(const std::string &remove_path);
122
123
  // Nested catalog handling
124
  void CreateNestedCatalog(const std::string &mountpoint);
125
  void RemoveNestedCatalog(const std::string &mountpoint,
126
                           const bool merge = true);
127
  bool IsTransitionPoint(const std::string &mountpoint);
128
  WritableCatalog *GetHostingCatalog(const std::string &path);
129
130
  inline bool IsBalanceable() const { return is_balanceable_; }
131
  /**
132
   * TODO
133
   */
134
  void PrecalculateListings();
135
136
  void SetTTL(const uint64_t new_ttl);
137
  bool SetVOMSAuthz(const std::string &voms_authz);
138
  bool Commit(const bool           stop_for_tweaks,
139
              const uint64_t       manual_revision,
140
              manifest::Manifest  *manifest);
141
142
  void Balance() {
143
      if (IsBalanceable()) {
144
          DoBalance();
145
      } else {
146
          LogCvmfs(kLogCatalog, kLogVerboseMsg, "Not balancing the catalog "
147
                  "manager because it is not balanceable");
148
      }
149
  }
150
151
 protected:
152
8
  void EnforceSqliteMemLimit() { }
153
154
  Catalog *CreateCatalog(const PathString &mountpoint,
155
                         const shash::Any &catalog_hash,
156
                         Catalog *parent_catalog);
157
  void ActivateCatalog(Catalog *catalog);
158
159
  void AddFile(const DirectoryEntry  &entry,
160
               const XattrList       &xattrs,
161
               const std::string     &parent_directory);
162
163
 private:
164
  bool FindCatalog(const std::string  &path,
165
                   WritableCatalog   **result,
166
                   DirectoryEntry     *dirent = NULL);
167
  void DoBalance();
168
  void FixWeight(WritableCatalog *catalog);
169
170
24
  struct CatalogInfo {
171
    uint64_t     ttl;
172
    size_t       size;
173
    shash::Any   content_hash;
174
    unsigned int revision;
175
  };
176
177
  struct CatalogUploadContext {
178
    Future<CatalogInfo>* root_catalog_info;
179
    bool                 stop_for_tweaks;
180
  };
181
182
  CatalogInfo SnapshotCatalogs(const bool stop_for_tweaks);
183
  void FinalizeCatalog(WritableCatalog *catalog,
184
                       const bool stop_for_tweaks);
185
  void ScheduleCatalogProcessing(WritableCatalog *catalog);
186
187
8
  void GetModifiedCatalogLeafs(WritableCatalogList *result) const {
188
    const bool dirty = GetModifiedCatalogLeafsRecursively(GetRootCatalog(),
189
8
                                                          result);
190
8
    assert(dirty);
191
8
  }
192
  bool GetModifiedCatalogLeafsRecursively(Catalog             *catalog,
193
                                          WritableCatalogList *result) const;
194
195
  void CatalogUploadCallback(const upload::SpoolerResult &result,
196
                             const CatalogUploadContext   clg_upload_context);
197
198
 private:
199
106
  inline void SyncLock() { pthread_mutex_lock(sync_lock_); }
200
106
  inline void SyncUnlock() { pthread_mutex_unlock(sync_lock_); }
201
202
  //****************************************************************************
203
  // Workaround -- Serialized Catalog Committing
204
  void GetModifiedCatalogs(WritableCatalogList *result) const {
205
    const unsigned int number_of_dirty_catalogs =
206
      GetModifiedCatalogsRecursively(GetRootCatalog(), result);
207
    assert(number_of_dirty_catalogs <= result->size());
208
  }
209
  int GetModifiedCatalogsRecursively(const Catalog *catalog,
210
                                     WritableCatalogList *result) const;
211
  void CatalogUploadSerializedCallback(
212
    const upload::SpoolerResult &result,
213
    const CatalogUploadContext unused);
214
  CatalogInfo SnapshotCatalogsSerialized(const bool stop_for_tweaks);
215
  //****************************************************************************
216
217
  // defined in catalog_mgr_rw.cc
218
  static const std::string kCatalogFilename;
219
220
  // private lock of WritableCatalogManager
221
  pthread_mutex_t *sync_lock_;
222
  upload::Spooler *spooler_;
223
224
  pthread_mutex_t                         *catalog_processing_lock_;
225
  std::map<std::string, WritableCatalog*>  catalog_processing_map_;
226
227
  // TODO(jblomer): catalog limits should become its own struct
228
  bool enforce_limits_;
229
  unsigned nested_kcatalog_limit_;
230
  unsigned root_kcatalog_limit_;
231
  unsigned file_mbyte_limit_;
232
233
  /**
234
   * Directories don't have extended attributes at this point.
235
   */
236
  XattrList empty_xattrs;
237
238
  /**
239
   * It indicates whether this catalog manager supports balancing operations
240
   */
241
  const bool is_balanceable_;
242
243
  /**
244
   * Defines the maximum weight an autogenerated catalog can have. If after a
245
   * publishing operation the catalog's weight is greater than this threshold it
246
   * will be considered overflowed and will automatically be split in different
247
   * sub-catalogs.
248
   */
249
  const unsigned max_weight_;
250
251
  /**
252
   * Defines the minimum weight an autogenerated catalog can have. If after a
253
   * publishing operation the catalog's weight is lesser than this threshold it
254
   * will be considered underflowed and will automatically be merged with its
255
   * parent.
256
   * This last operation can provoke an overflow in the parent, though.
257
   */
258
  const unsigned min_weight_;
259
260
  /**
261
   * Defines the threshold that will be used to balance a catalog that has been
262
   * overflowed. Its value should be lesser than max_weight_ and greater than
263
   * min_weight. By default it is set to max_weight / 2.
264
   */
265
  const unsigned balance_weight_;
266
};  // class WritableCatalogManager
267
268
}  // namespace catalog
269
270
#endif  // CVMFS_CATALOG_MGR_RW_H_