CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
preload.cc
Go to the documentation of this file.
1 
5 #include "cvmfs_config.h"
6 
7 #include <stdio.h>
8 #include <sys/stat.h>
9 #include <unistd.h>
10 
11 #include <string>
12 
13 #include "compression.h"
14 #include "download.h"
15 #include "logging.h"
16 #include "signature.h"
17 #include "statistics.h"
18 #include "swissknife.h"
19 #include "swissknife_pull.h"
20 #include "util/posix.h"
21 #include "util/string.h"
22 #include "uuid.h"
23 
24 
25 using namespace std; // NOLINT
26 
27 const char *kVersion = VERSION;
28 const int kDefaultPreloaderTimeout = 10;
30 
31 namespace swissknife {
35 void Usage() {
36  LogCvmfs(kLogCvmfs, kLogStderr, "Version: %s\n\n"
37  "Usage:\n"
38  "cvmfs_preload -u <Stratum 0 URL>\n"
39  " -r <alien cache directory>\n"
40  " [-d <path to dirtab file>]\n"
41  " [-k <public key>]\n"
42  " [-m <fully qualified repository name>]\n"
43  " [-n <num of parallel download threads>]\n"
44  " [-t <timeout in seconds (default %d)>]\n"
45  " [-a <number of retries (default %d)>]\n"
46  " [-x <directory for temporary files>]\n\n",
48 }
49 } // namespace swissknife
50 
51 const char gCernIt1PublicKey[] =
52  "-----BEGIN PUBLIC KEY-----\n"
53  "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAo8uKvscgW7FNxzb65Uhm\n"
54  "yr8jPJiyrl2kVzb/hhgdfN14C0tCbfFoE6ciuZFg+9ytLeiL9pzM96gSC+atIFl4\n"
55  "7wTgtAFO1W4PtDQBwA/IG2bnwvNrzk19ob0JYhjZlS9tYKeh7TKCub55+vMwcEbP\n"
56  "urzo3WSNCzJngiGMh1vM5iSlGLpCdSGzdwxLGwc1VjRM7q3KAd7M7TJCynKqXZPX\n"
57  "R2xiD6I/p4xv39AnwphCFSmDh0MWE1WeeNHIiiveikvvN+l8d/ZNASIDhKNCsz6o\n"
58  "aFDsGXvjGy7dg43YzjSSYSFGUnONtl5Fe6y4bQZj1LEPbeInW334MAbMwYF4LKma\n"
59  "yQIDAQAB\n"
60  "-----END PUBLIC KEY-----\n";
61 
62 const char gCernIt4PublicKey[] =
63  "-----BEGIN PUBLIC KEY-----\n"
64  "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzlAraXimfJP5ie0KtDAE\n"
65  "rNUU5d9bzst+kqfhnb0U0OUtmCIbsueaDlbMmTdRSHMr+T0jI8i9CZxJwtxDqll+\n"
66  "UuB3Li2hYBhk0tYTy29JJYvofVULvrw1kMSLKyTWnV30/MHjYxhLHoZWfdepTjVg\n"
67  "lM0rP58K10wR3Z/AaaikOcy4z6P/MHs9ES1jdZqEBQEmmzKw5nf7pfU2QuVWJrKP\n"
68  "wZ9XeYDzipVbMc1zaLEK0slE+bm2ge/Myvuj/rpYKT+6qzbasQg62abGFuOrjgKI\n"
69  "X4/BVnilkhUfH6ssRKw4yehlKG1M5KJje2+y+iVvLbfoaw3g1Sjrf4p3Gq+ul7AC\n"
70  "PwIDAQAB\n"
71  "-----END PUBLIC KEY-----\n";
72 
73 const char gCernIt5PublicKey[] =
74  "-----BEGIN PUBLIC KEY-----\n"
75  "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqFzLLZAg2xmHJLbbq0+N\n"
76  "eYtjRDghUK5mYhARndnC3skFVowDTiqJVc9dIDX5zuxQ9HyC0iKM1HbvN64IH/Uf\n"
77  "qoXLyZLiXbFwpg6BtEJxwhijdZCiCC5PC//Bb7zSFIVZvWjndujr6ejaY6kx3+jI\n"
78  "sU1HSJ66pqorj+D1fbZCziLcWbS1GzceZ7aTYYPUdGZF1CgjBK5uKrEAoBsPgjWo\n"
79  "+YOEkjskY7swfhUzkCe0YyMyAaS0gsWgYrY2ebrpauFFqKxveKWjDVBTGcwDhiBX\n"
80  "60inUgD6CJXhUpvGHfU8V7Bv6l7dmyzhq/Bk2kRC92TIvxfaHRmS7nuknUY0hW6t\n"
81  "2QIDAQAB\n"
82  "-----END PUBLIC KEY-----\n";
83 
84 
85 static char CheckParameters(const string &params,
87  for (unsigned i = 0; i < params.length(); ++i) {
88  char param = params[i];
89  if (args->find(param) == args->end()) {
90  return param;
91  }
92  }
93  return '\0';
94 }
95 
96 static bool HasDirtabChanged(const string &dirtab_src, const string &dirtab_dst)
97 {
98  bool retval;
99  shash::Any hash_src(shash::kMd5);
100  shash::Any hash_dst(shash::kMd5);
101  retval = shash::HashFile(dirtab_src, &hash_src);
102  if (!retval)
103  return true;
104  retval = shash::HashFile(dirtab_dst, &hash_dst);
105  if (!retval)
106  return true;
107  return hash_src != hash_dst;
108 }
109 
110 
111 int main(int argc, char *argv[]) {
112  int retval;
113 
114  // load some default arguments
116  args['n'].Reset(new string("4"));
117 
118  string option_string = "u:r:k:m:x:d:n:t:a:vh";
119  int c;
120  while ((c = getopt(argc, argv, option_string.c_str())) != -1) {
121  if ((c == 'v') || (c == 'h')) {
123  return 0;
124  }
125  args[c].Reset(new string(optarg));
126  }
127 
128  // check all mandatory parameters are included
129  string necessary_params = "ur";
130  char result;
131  if ((result = CheckParameters(necessary_params, &args)) != '\0') {
132  printf("Argument not included but necessary: -%c\n\n", result);
134  return 2;
135  }
136 
137  if (args.find('m') == args.end()) {
138  string fqrn = GetFileName(*args['u']);
139  LogCvmfs(kLogCvmfs, kLogStdout, "CernVM-FS: guessing fqrn from URL: %s",
140  fqrn.c_str());
141  args['m'].Reset(new string(fqrn));
142  }
143 
144  if (args.find('x') == args.end())
145  args['x'].Reset(new string(*args['r'] + "/txn"));
146 
147  const string cache_directory = *args['r'];
148  const string fqrn = *args['m'];
149  const string dirtab =
150  (args.find('d') == args.end()) ? "/dev/null" : *args['d'];
151  const string dirtab_in_cache = cache_directory + "/dirtab." + fqrn;
152 
153  // Default network parameters: 5 seconds timeout, 2 retries
154  if (args.find('t') == args.end())
155  args['t'].Reset(new string(StringifyInt(kDefaultPreloaderTimeout)));
156  if (args.find('a') == args.end())
157  args['a'].Reset(new string(StringifyInt(kDefaultPreloaderRetries)));
158 
159  // first create the alien cache
160  const string& alien_cache_dir = *(args['r']);
161  retval = MkdirDeep(alien_cache_dir, 0770);
162  if (!retval) {
163  LogCvmfs(kLogCvmfs, kLogStderr, "Failed to create %s",
164  alien_cache_dir.c_str());
165  return 1;
166  }
167  retval = MakeCacheDirectories(alien_cache_dir, 0770);
168  if (!retval) {
169  LogCvmfs(kLogCvmfs, kLogStderr, "Failed to create cache skeleton");
170  return 1;
171  }
172 
173  // if there is no specified public key file we dump the cern.ch public key in
174  // the temporary directory
175  string cern_pk_base_path = *args['x'];
176  string cern_pk_it1_path = cern_pk_base_path + "/cern-it1.cern.ch.pub";
177  string cern_pk_it4_path = cern_pk_base_path + "/cern-it4.cern.ch.pub";
178  string cern_pk_it5_path = cern_pk_base_path + "/cern-it5.cern.ch.pub";
179  bool keys_created = false;
180  if (args.find('k') == args.end()) {
181  keys_created = true;
183  reinterpret_cast<const unsigned char*>(gCernIt1PublicKey),
184  sizeof(gCernIt1PublicKey), cern_pk_it1_path));
186  reinterpret_cast<const unsigned char*>(gCernIt4PublicKey),
187  sizeof(gCernIt4PublicKey), cern_pk_it4_path));
189  reinterpret_cast<const unsigned char*>(gCernIt5PublicKey),
190  sizeof(gCernIt5PublicKey), cern_pk_it5_path));
191  char path_separator = ':';
192  args['k'].Reset(new string(cern_pk_it1_path + path_separator +
193  cern_pk_it4_path + path_separator +
194  cern_pk_it5_path));
195  }
196 
197  // now launch swissknife_pull
201 
202  // load the command
203  if (HasDirtabChanged(dirtab, dirtab_in_cache)) {
204  LogCvmfs(kLogCvmfs, kLogStdout, "CernVM-FS: new dirtab, forced run");
205  args['z'].Reset(); // look into existing catalogs, too
206  }
207  args['c'].Reset();
208  retval = swissknife::CommandPull().Main(args);
209 
210  // Copy dirtab file
211  if (retval == 0) {
212  CopyPath2Path(dirtab, dirtab_in_cache);
213  }
214 
215  // Create cache uuid if not present
216  cvmfs::Uuid *uuid = cvmfs::Uuid::Create(cache_directory + "/uuid");
217  if (uuid == NULL) {
218  LogCvmfs(kLogCvmfs, kLogStderr, "Warning: failed to create %s/uuid",
219  cache_directory.c_str());
220  }
221  delete uuid;
222 
223  if (keys_created) {
224  unlink(cern_pk_it1_path.c_str());
225  unlink(cern_pk_it4_path.c_str());
226  unlink(cern_pk_it5_path.c_str());
227  }
228 
229  return retval;
230 }
bool MakeCacheDirectories(const std::string &path, const mode_t mode)
Definition: posix.cc:907
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
bool HashFile(const std::string &filename, Any *any_digest)
Definition: hash.cc:342
std::string GetFileName(const std::string &path)
Definition: posix.cc:162
int Main(const ArgumentList &args)
assert((mem||(size==0))&&"Out Of Memory")
bool CopyPath2Path(const string &src, const string &dest)
Definition: compression.cc:63
const char gCernIt1PublicKey[]
Definition: preload.cc:51
int main()
Definition: helper_allow.cc:16
signature::SignatureManager * g_signature_manager
Definition: preload.cc:33
bool CopyMem2Path(const unsigned char *buffer, const unsigned buffer_size, const string &path)
Definition: compression.cc:94
perf::Statistics * g_statistics
Definition: preload.cc:34
static char CheckParameters(const string &params, swissknife::ArgumentList *args)
Definition: preload.cc:85
const int kDefaultPreloaderTimeout
Definition: preload.cc:28
bool MkdirDeep(const std::string &path, const mode_t mode, bool verify_writable)
Definition: posix.cc:871
void Usage()
string StringifyInt(const int64_t value)
Definition: string.cc:78
const char gCernIt5PublicKey[]
Definition: preload.cc:73
static Uuid * Create(const std::string &store_path)
Definition: uuid.cc:23
static bool HasDirtabChanged(const string &dirtab_src, const string &dirtab_dst)
Definition: preload.cc:96
std::map< char, SharedPtr< std::string > > ArgumentList
Definition: swissknife.h:72
download::DownloadManager * g_download_manager
Definition: preload.cc:32
const int kDefaultPreloaderRetries
Definition: preload.cc:29
const char * kVersion
Definition: preload.cc:27
void Usage()
Definition: preload.cc:35
const char gCernIt4PublicKey[]
Definition: preload.cc:62