CernVM-FS  2.10.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
compression.cc
Go to the documentation of this file.
1 
11 #include "cvmfs_config.h"
12 #include "compression.h"
13 
14 #include <alloca.h>
15 #include <stdlib.h>
16 #include <sys/stat.h>
17 
18 #include <algorithm>
19 #include <cassert>
20 #include <cstring>
21 
22 #include "hash.h"
23 #include "logging.h"
24 #include "platform.h"
25 #include "smalloc.h"
26 #include "util/exception.h"
27 #include "util/posix.h"
28 
29 using namespace std; // NOLINT
30 
31 
32 static bool CopyFile2File(FILE *fsrc, FILE *fdest) {
33  unsigned char buf[1024];
34  rewind(fsrc);
35  rewind(fdest);
36 
37  size_t have;
38  do {
39  have = fread(buf, 1, 1024, fsrc);
40  if (fwrite(buf, 1, have, fdest) != have)
41  return false;
42  } while (have == 1024);
43  return true;
44 }
45 
46 bool CopyPath2File(const std::string &src, FILE *fdest) {
47  int retval = -1;
48  platform_stat64 info;
49 
50  FILE *fsrc = fopen(src.c_str(), "r");
51  if (!fsrc) goto file_copy_final;
52 
53  if (!CopyFile2File(fsrc, fdest)) goto file_copy_final;
54  retval = platform_fstat(fileno(fsrc), &info);
55  retval |= fchmod(fileno(fdest), info.st_mode);
56 
57  file_copy_final:
58  if (fsrc) fclose(fsrc);
59  return retval == 0;
60 }
61 
62 
63 bool CopyPath2Path(const string &src, const string &dest) {
64  FILE *fsrc = NULL;
65  FILE *fdest = NULL;
66  int retval = -1;
67  platform_stat64 info;
68 
69  fsrc = fopen(src.c_str(), "r");
70  if (!fsrc) goto file_copy_final;
71 
72  fdest = fopen(dest.c_str(), "w");
73  if (!fdest) goto file_copy_final;
74 
75  if (!CopyFile2File(fsrc, fdest)) goto file_copy_final;
76  retval = platform_fstat(fileno(fsrc), &info);
77  retval |= fchmod(fileno(fdest), info.st_mode);
78 
79  file_copy_final:
80  if (fsrc) fclose(fsrc);
81  if (fdest) fclose(fdest);
82  return retval == 0;
83 }
84 
85 
86 bool CopyMem2File(const unsigned char *buffer, const unsigned buffer_size,
87  FILE *fdest)
88 {
89  int written = fwrite(buffer, 1, buffer_size, fdest);
90  return (written >=0) && (unsigned(written) == buffer_size);
91 }
92 
93 
94 bool CopyMem2Path(const unsigned char *buffer, const unsigned buffer_size,
95  const string &path)
96 {
97  int fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, kDefaultFileMode);
98  if (fd < 0)
99  return false;
100 
101  int written = write(fd, buffer, buffer_size);
102  close(fd);
103 
104  return (written >=0) && (unsigned(written) == buffer_size);
105 }
106 
107 
108 bool CopyPath2Mem(const string &path,
109  unsigned char **buffer, unsigned *buffer_size)
110 {
111  const int fd = open(path.c_str(), O_RDONLY);
112  if (fd < 0)
113  return false;
114 
115  *buffer_size = 512;
116  *buffer = reinterpret_cast<unsigned char *>(smalloc(*buffer_size));
117  unsigned total_bytes = 0;
118  while (true) {
119  int num_bytes = read(fd, *buffer + total_bytes, *buffer_size - total_bytes);
120  if (num_bytes == 0)
121  break;
122  if (num_bytes < 0) {
123  close(fd);
124  free(*buffer);
125  *buffer_size = 0;
126  return false;
127  }
128  total_bytes += num_bytes;
129  if (total_bytes >= *buffer_size) {
130  *buffer_size *= 2;
131  *buffer =
132  reinterpret_cast<unsigned char *>(srealloc(*buffer, *buffer_size));
133  }
134  }
135 
136  close(fd);
137  *buffer_size = total_bytes;
138  return true;
139 }
140 
141 namespace zlib {
142 
143 const unsigned kBufferSize = 32768;
144 
148 Algorithms ParseCompressionAlgorithm(const std::string &algorithm_option) {
149  if ((algorithm_option == "default") || (algorithm_option == "zlib"))
150  return kZlibDefault;
151  if (algorithm_option == "none")
152  return kNoCompression;
153  PANIC(kLogStderr, "unknown compression algorithms: %s",
154  algorithm_option.c_str());
155 }
156 
157 
158 std::string AlgorithmName(const zlib::Algorithms alg) {
159  switch (alg) {
160  case kZlibDefault:
161  return "zlib";
162  break;
163  case kNoCompression:
164  return "none";
165  break;
166  // Purposely did not add a 'default' statement here: this will
167  // cause the compiler to generate a warning if a new algorithm
168  // is added but this function is not updated.
169  }
170  return "unknown";
171 }
172 
173 
174 void CompressInit(z_stream *strm) {
175  strm->zalloc = Z_NULL;
176  strm->zfree = Z_NULL;
177  strm->opaque = Z_NULL;
178  strm->next_in = Z_NULL;
179  strm->avail_in = 0;
180  int retval = deflateInit(strm, Z_DEFAULT_COMPRESSION);
181  assert(retval == 0);
182 }
183 
184 
185 void DecompressInit(z_stream *strm) {
186  strm->zalloc = Z_NULL;
187  strm->zfree = Z_NULL;
188  strm->opaque = Z_NULL;
189  strm->avail_in = 0;
190  strm->next_in = Z_NULL;
191  int retval = inflateInit(strm);
192  assert(retval == 0);
193 }
194 
195 
196 void CompressFini(z_stream *strm) {
197  (void)deflateEnd(strm);
198 }
199 
200 
201 void DecompressFini(z_stream *strm) {
202  (void)inflateEnd(strm);
203 }
204 
205 
207  const void *buf,
208  const int64_t size,
209  const bool eof,
210  z_stream *strm,
211  shash::ContextPtr *hash_context)
212 {
213  unsigned char out[kZChunk];
214  int z_ret;
215 
216  strm->avail_in = size;
217  strm->next_in = static_cast<unsigned char *>(const_cast<void *>(buf));
218  // Run deflate() on input until output buffer not full, finish
219  // compression if all of source has been read in
220  do {
221  strm->avail_out = kZChunk;
222  strm->next_out = out;
223  z_ret = deflate(strm, eof ? Z_FINISH : Z_NO_FLUSH); // no bad return value
224  if (z_ret == Z_STREAM_ERROR)
225  return kStreamDataError;
226  size_t have = kZChunk - strm->avail_out;
227  shash::Update(out, have, *hash_context);
228  } while (strm->avail_out == 0);
229 
230  return (z_ret == Z_STREAM_END ? kStreamEnd : kStreamContinue);
231 }
232 
233 
235  const void *buf,
236  const int64_t size,
237  z_stream *strm,
238  cvmfs::Sink *sink)
239 {
240  unsigned char out[kZChunk];
241  int z_ret;
242  int64_t pos = 0;
243 
244  do {
245  strm->avail_in = (kZChunk > (size-pos)) ? size-pos : kZChunk;
246  strm->next_in = ((unsigned char *)buf)+pos;
247 
248  // Run inflate() on input until output buffer not full
249  do {
250  strm->avail_out = kZChunk;
251  strm->next_out = out;
252  z_ret = inflate(strm, Z_NO_FLUSH);
253  switch (z_ret) {
254  case Z_NEED_DICT:
255  z_ret = Z_DATA_ERROR; // and fall through
256  case Z_STREAM_ERROR:
257  case Z_DATA_ERROR:
258  return kStreamDataError;
259  case Z_MEM_ERROR:
260  return kStreamIOError;
261  }
262  size_t have = kZChunk - strm->avail_out;
263  int64_t written = sink->Write(out, have);
264  if ((written < 0) || (static_cast<uint64_t>(written) != have))
265  return kStreamIOError;
266  } while (strm->avail_out == 0);
267 
268  pos += kZChunk;
269  } while (pos < size);
270 
271  return (z_ret == Z_STREAM_END ? kStreamEnd : kStreamContinue);
272 }
273 
274 
276  const void *buf,
277  const int64_t size,
278  z_stream *strm,
279  FILE *f)
280 {
281  unsigned char out[kZChunk];
282  int z_ret;
283  int64_t pos = 0;
284 
285  do {
286  strm->avail_in = (kZChunk > (size-pos)) ? size-pos : kZChunk;
287  strm->next_in = ((unsigned char *)buf)+pos;
288 
289  // Run inflate() on input until output buffer not full
290  do {
291  strm->avail_out = kZChunk;
292  strm->next_out = out;
293  z_ret = inflate(strm, Z_NO_FLUSH);
294  switch (z_ret) {
295  case Z_NEED_DICT:
296  z_ret = Z_DATA_ERROR; // and fall through
297  case Z_STREAM_ERROR:
298  case Z_DATA_ERROR:
299  return kStreamDataError;
300  case Z_MEM_ERROR:
301  return kStreamIOError;
302  }
303  size_t have = kZChunk - strm->avail_out;
304  if (fwrite(out, 1, have, f) != have || ferror(f)) {
305  LogCvmfs(kLogCompress, kLogDebug, "Inflate to file failed with %s "
306  "(errno=%d)", strerror(errno), errno);
307  return kStreamIOError;
308  }
309  } while (strm->avail_out == 0);
310 
311  pos += kZChunk;
312  } while (pos < size);
313 
314  return (z_ret == Z_STREAM_END ? kStreamEnd : kStreamContinue);
315 }
316 
317 
318 bool CompressPath2Path(const string &src, const string &dest) {
319  FILE *fsrc = fopen(src.c_str(), "r");
320  if (!fsrc) {
321  LogCvmfs(kLogCompress, kLogDebug, "open %s as compression source failed",
322  src.c_str());
323  return false;
324  }
325 
326  FILE *fdest = fopen(dest.c_str(), "w");
327  if (!fdest) {
328  LogCvmfs(kLogCompress, kLogDebug, "open %s as compression destination "
329  "failed with errno=%d", dest.c_str(), errno);
330  fclose(fsrc);
331  return false;
332  }
333 
334  LogCvmfs(kLogCompress, kLogDebug, "opened %s and %s for compression",
335  src.c_str(), dest.c_str());
336  const bool result = CompressFile2File(fsrc, fdest);
337 
338  fclose(fsrc);
339  fclose(fdest);
340  return result;
341 }
342 
343 
344 bool CompressPath2Path(const string &src, const string &dest,
345  shash::Any *compressed_hash)
346 {
347  FILE *fsrc = fopen(src.c_str(), "r");
348  if (!fsrc) {
349  LogCvmfs(kLogCompress, kLogDebug, "open %s as compression source failed",
350  src.c_str());
351  return false;
352  }
353 
354  FILE *fdest = fopen(dest.c_str(), "w");
355  if (!fdest) {
356  LogCvmfs(kLogCompress, kLogDebug, "open %s as compression destination "
357  "failed with errno=%d", dest.c_str(), errno);
358  fclose(fsrc);
359  return false;
360  }
361 
362  LogCvmfs(kLogCompress, kLogDebug, "opened %s and %s for compression",
363  src.c_str(), dest.c_str());
364  bool result = false;
365  if (!CompressFile2File(fsrc, fdest, compressed_hash))
366  goto compress_path2path_final;
367  platform_stat64 info;
368  if (platform_fstat(fileno(fsrc), &info) != 0) goto compress_path2path_final;
369  // TODO(jakob): open in the right mode from the beginning
370  if (fchmod(fileno(fdest), info.st_mode) != 0) goto compress_path2path_final;
371 
372  result = true;
373 
374  compress_path2path_final:
375  fclose(fsrc);
376  fclose(fdest);
377  return result;
378 }
379 
380 
381 bool DecompressPath2Path(const string &src, const string &dest) {
382  FILE *fsrc = NULL;
383  FILE *fdest = NULL;
384  int result = false;
385 
386  fsrc = fopen(src.c_str(), "r");
387  if (!fsrc) goto decompress_path2path_final;
388 
389  fdest = fopen(dest.c_str(), "w");
390  if (!fdest) goto decompress_path2path_final;
391 
392  result = DecompressFile2File(fsrc, fdest);
393 
394  decompress_path2path_final:
395  if (fsrc) fclose(fsrc);
396  if (fdest) fclose(fdest);
397  return result;
398 }
399 
400 
401 bool CompressFile2Null(FILE *fsrc, shash::Any *compressed_hash) {
402  int z_ret = 0;
403  int flush = 0;
404  bool result = -1;
405  unsigned have;
406  z_stream strm;
407  unsigned char in[kZChunk];
408  unsigned char out[kZChunk];
409  shash::ContextPtr hash_context(compressed_hash->algorithm);
410 
411  CompressInit(&strm);
412  hash_context.buffer = alloca(hash_context.size);
413  shash::Init(hash_context);
414 
415  // Compress until end of file
416  do {
417  strm.avail_in = fread(in, 1, kZChunk, fsrc);
418  if (ferror(fsrc)) goto compress_file2null_final;
419 
420  flush = feof(fsrc) ? Z_FINISH : Z_NO_FLUSH;
421  strm.next_in = in;
422 
423  // Run deflate() on input until output buffer not full, finish
424  // compression if all of source has been read in
425  do {
426  strm.avail_out = kZChunk;
427  strm.next_out = out;
428  z_ret = deflate(&strm, flush); // no bad return value
429  if (z_ret == Z_STREAM_ERROR)
430  goto compress_file2null_final; // state not clobbered
431  have = kZChunk - strm.avail_out;
432  shash::Update(out, have, hash_context);
433  } while (strm.avail_out == 0);
434 
435  // Done when last data in file processed
436  } while (flush != Z_FINISH);
437 
438  // stream will be complete
439  if (z_ret != Z_STREAM_END) goto compress_file2null_final;
440 
441  shash::Final(hash_context, compressed_hash);
442  result = true;
443 
444  // Clean up and return
445  compress_file2null_final:
446  CompressFini(&strm);
447  LogCvmfs(kLogCompress, kLogDebug, "file compression finished with result %d",
448  result);
449  return result;
450 }
451 
452 
453 bool CompressFd2Null(int fd_src, shash::Any *compressed_hash,
454  uint64_t *processed_bytes) {
455  int z_ret = 0;
456  int flush = 0;
457  bool result = false;
458  unsigned have;
459  z_stream strm;
460  unsigned char in[kZChunk];
461  unsigned char out[kZChunk];
462  off_t cksum_bytes = 0;
463  shash::ContextPtr hash_context(compressed_hash->algorithm);
464 
465  CompressInit(&strm);
466  hash_context.buffer = alloca(hash_context.size);
467  shash::Init(hash_context);
468 
469  // Compress until end of file
470  do {
471  ssize_t bytes_read = read(fd_src, in, kZChunk);
472  if (bytes_read < 0) {
473  if (errno == EINTR) {continue;}
474  goto compress_fd2null_final;
475  }
476  cksum_bytes += bytes_read;
477  strm.avail_in = bytes_read;
478 
479  flush = (static_cast<size_t>(bytes_read) < kZChunk) ? Z_FINISH : Z_NO_FLUSH;
480  strm.next_in = in;
481 
482  // Run deflate() on input until output buffer not full, finish
483  // compression if all of source has been read in
484  do {
485  strm.avail_out = kZChunk;
486  strm.next_out = out;
487  z_ret = deflate(&strm, flush); // no bad return value
488  if (z_ret == Z_STREAM_ERROR)
489  goto compress_fd2null_final; // state not clobbered
490  have = kZChunk - strm.avail_out;
491  shash::Update(out, have, hash_context);
492  } while (strm.avail_out == 0);
493 
494  // Done when last data in file processed
495  } while (flush != Z_FINISH);
496 
497  // stream will be complete
498  if (z_ret != Z_STREAM_END) goto compress_fd2null_final;
499 
500  shash::Final(hash_context, compressed_hash);
501  if (processed_bytes) {
502  *processed_bytes = cksum_bytes;
503  }
504  result = true;
505 
506  // Clean up and return
507  compress_fd2null_final:
508  CompressFini(&strm);
509  LogCvmfs(kLogCompress, kLogDebug, "file compression finished with result %d",
510  result);
511  return result;
512 }
513 
514 
515 bool CompressPath2Null(const string &src, shash::Any *compressed_hash) {
516  FILE *fsrc = fopen(src.c_str(), "r");
517  if (fsrc == NULL)
518  return false;
519  bool retval = CompressFile2Null(fsrc, compressed_hash);
520  fclose(fsrc);
521  return retval;
522 }
523 
524 
525 bool CompressFile2File(FILE *fsrc, FILE *fdest) {
526  int z_ret = 0;
527  int flush = 0;
528  bool result = false;
529  unsigned have;
530  z_stream strm;
531  unsigned char in[kZChunk];
532  unsigned char out[kZChunk];
533 
534  CompressInit(&strm);
535 
536  // Compress until end of file
537  do {
538  strm.avail_in = fread(in, 1, kZChunk, fsrc);
539  if (ferror(fsrc)) goto compress_file2file_final;
540 
541  flush = feof(fsrc) ? Z_FINISH : Z_NO_FLUSH;
542  strm.next_in = in;
543 
544  // Run deflate() on input until output buffer not full, finish
545  // compression if all of source has been read in
546  do {
547  strm.avail_out = kZChunk;
548  strm.next_out = out;
549  z_ret = deflate(&strm, flush); // no bad return value
550  if (z_ret == Z_STREAM_ERROR)
551  goto compress_file2file_final; // state not clobbered
552  have = kZChunk - strm.avail_out;
553  if (fwrite(out, 1, have, fdest) != have || ferror(fdest))
554  goto compress_file2file_final;
555  } while (strm.avail_out == 0);
556 
557  // Done when last data in file processed
558  } while (flush != Z_FINISH);
559 
560  // stream will be complete
561  if (z_ret != Z_STREAM_END) goto compress_file2file_final;
562 
563  result = true;
564 
565  // Clean up and return
566  compress_file2file_final:
567  CompressFini(&strm);
568  LogCvmfs(kLogCompress, kLogDebug, "file compression finished with result %d",
569  result);
570  return result;
571 }
572 
573 bool CompressPath2File(const string &src, FILE *fdest,
574  shash::Any *compressed_hash)
575 {
576  FILE *fsrc = fopen(src.c_str(), "r");
577  if (!fsrc)
578  return false;
579 
580  bool retval = CompressFile2File(fsrc, fdest, compressed_hash);
581  fclose(fsrc);
582  return retval;
583 }
584 
585 
586 bool CompressFile2File(FILE *fsrc, FILE *fdest, shash::Any *compressed_hash) {
587  int z_ret = 0;
588  int flush = 0;
589  bool result = false;
590  unsigned have;
591  z_stream strm;
592  unsigned char in[kZChunk];
593  unsigned char out[kZChunk];
594  shash::ContextPtr hash_context(compressed_hash->algorithm);
595 
596  CompressInit(&strm);
597  hash_context.buffer = alloca(hash_context.size);
598  shash::Init(hash_context);
599 
600  // Compress until end of file
601  do {
602  strm.avail_in = fread(in, 1, kZChunk, fsrc);
603  if (ferror(fsrc)) goto compress_file2file_hashed_final;
604 
605  flush = feof(fsrc) ? Z_FINISH : Z_NO_FLUSH;
606  strm.next_in = in;
607 
608  // Run deflate() on input until output buffer not full, finish
609  // compression if all of source has been read in
610  do {
611  strm.avail_out = kZChunk;
612  strm.next_out = out;
613  z_ret = deflate(&strm, flush); // no bad return value
614  if (z_ret == Z_STREAM_ERROR)
615  goto compress_file2file_hashed_final; // state not clobbered
616  have = kZChunk - strm.avail_out;
617  if (fwrite(out, 1, have, fdest) != have || ferror(fdest))
618  goto compress_file2file_hashed_final;
619  shash::Update(out, have, hash_context);
620  } while (strm.avail_out == 0);
621 
622  // Done when last data in file processed
623  } while (flush != Z_FINISH);
624 
625  // Stream will be complete
626  if (z_ret != Z_STREAM_END) goto compress_file2file_hashed_final;
627 
628  shash::Final(hash_context, compressed_hash);
629  result = true;
630 
631  // Clean up and return
632  compress_file2file_hashed_final:
633  CompressFini(&strm);
634  LogCvmfs(kLogCompress, kLogDebug, "file compression finished with result %d",
635  result);
636  return result;
637 }
638 
639 
640 bool DecompressFile2File(FILE *fsrc, FILE *fdest) {
641  bool result = false;
642  StreamStates stream_state = kStreamIOError;
643  z_stream strm;
644  size_t have;
645  unsigned char buf[kBufferSize];
646 
647  DecompressInit(&strm);
648 
649  while ((have = fread(buf, 1, kBufferSize, fsrc)) > 0) {
650  stream_state = DecompressZStream2File(buf, have, &strm, fdest);
651  if ((stream_state == kStreamDataError) || (stream_state == kStreamIOError))
652  goto decompress_file2file_final;
653  }
654  LogCvmfs(kLogCompress, kLogDebug, "end of decompression, state=%d, error=%d",
655  stream_state, ferror(fsrc));
656  if ((stream_state != kStreamEnd) || ferror(fsrc))
657  goto decompress_file2file_final;
658 
659  result = true;
660 
661  decompress_file2file_final:
662  DecompressFini(&strm);
663  return result;
664 }
665 
666 
667 bool DecompressPath2File(const string &src, FILE *fdest) {
668  FILE *fsrc = fopen(src.c_str(), "r");
669  if (!fsrc)
670  return false;
671 
672  bool retval = DecompressFile2File(fsrc, fdest);
673  fclose(fsrc);
674  return retval;
675 }
676 
677 
678 bool CompressMem2File(const unsigned char *buf, const size_t size,
679  FILE *fdest, shash::Any *compressed_hash) {
680  int z_ret = 0;
681  int flush = 0;
682  bool result = false;
683  unsigned have;
684  z_stream strm;
685  size_t offset = 0;
686  size_t used = 0;
687  unsigned char out[kZChunk];
688  shash::ContextPtr hash_context(compressed_hash->algorithm);
689 
690  CompressInit(&strm);
691  hash_context.buffer = alloca(hash_context.size);
692  shash::Init(hash_context);
693 
694  // Compress the given memory buffer
695  do {
696  used = min((size_t)kZChunk, size - offset);
697  strm.avail_in = used;
698 
699  flush = (strm.avail_in < kZChunk) ? Z_FINISH : Z_NO_FLUSH;
700  strm.next_in = const_cast<unsigned char*>(buf + offset);
701 
702  // Run deflate() on input until output buffer not full, finish
703  // compression if all of source has been read in
704  do {
705  strm.avail_out = kZChunk;
706  strm.next_out = out;
707  z_ret = deflate(&strm, flush); // no bad return value
708  if (z_ret == Z_STREAM_ERROR)
709  goto compress_file2file_hashed_final; // state not clobbered
710  have = kZChunk - strm.avail_out;
711  if (fwrite(out, 1, have, fdest) != have || ferror(fdest))
712  goto compress_file2file_hashed_final;
713  shash::Update(out, have, hash_context);
714  } while (strm.avail_out == 0);
715 
716  offset += used;
717 
718  // Done when last data in file processed
719  } while (flush != Z_FINISH);
720 
721  // Stream will be complete
722  if (z_ret != Z_STREAM_END) goto compress_file2file_hashed_final;
723 
724  shash::Final(hash_context, compressed_hash);
725  result = true;
726 
727  // Clean up and return
728  compress_file2file_hashed_final:
729  CompressFini(&strm);
730  LogCvmfs(kLogCompress, kLogDebug, "file compression finished with result %d",
731  result);
732  return result;
733 }
734 
735 
739 bool CompressMem2Mem(const void *buf, const int64_t size,
740  void **out_buf, uint64_t *out_size)
741 {
742  unsigned char out[kZChunk];
743  int z_ret;
744  int flush;
745  z_stream strm;
746  int64_t pos = 0;
747  uint64_t alloc_size = kZChunk;
748 
749  CompressInit(&strm);
750  *out_buf = smalloc(alloc_size);
751  *out_size = 0;
752 
753  do {
754  strm.avail_in = (kZChunk > (size-pos)) ? size-pos : kZChunk;
755  flush = (pos + kZChunk) >= size ? Z_FINISH : Z_NO_FLUSH;
756  strm.next_in = ((unsigned char *)buf) + pos;
757 
758  // Run deflate() on input until output buffer not full
759  do {
760  strm.avail_out = kZChunk;
761  strm.next_out = out;
762  z_ret = deflate(&strm, flush);
763  if (z_ret == Z_STREAM_ERROR) {
764  CompressFini(&strm);
765  free(*out_buf);
766  *out_buf = NULL;
767  *out_size = 0;
768  return false;
769  }
770  size_t have = kZChunk - strm.avail_out;
771  if (*out_size+have > alloc_size) {
772  alloc_size *= 2;
773  *out_buf = srealloc(*out_buf, alloc_size);
774  }
775  memcpy(static_cast<unsigned char *>(*out_buf) + *out_size, out, have);
776  *out_size += have;
777  } while (strm.avail_out == 0);
778 
779  pos += kZChunk;
780  } while (flush != Z_FINISH);
781 
782  CompressFini(&strm);
783  if (z_ret != Z_STREAM_END) {
784  free(*out_buf);
785  *out_buf = NULL;
786  *out_size = 0;
787  return false;
788  } else {
789  return true;
790  }
791 }
792 
793 
797 bool DecompressMem2Mem(const void *buf, const int64_t size,
798  void **out_buf, uint64_t *out_size)
799 {
800  unsigned char out[kZChunk];
801  int z_ret;
802  z_stream strm;
803  int64_t pos = 0;
804  uint64_t alloc_size = kZChunk;
805 
806  DecompressInit(&strm);
807  *out_buf = smalloc(alloc_size);
808  *out_size = 0;
809 
810  do {
811  strm.avail_in = (kZChunk > (size-pos)) ? size-pos : kZChunk;
812  strm.next_in = ((unsigned char *)buf)+pos;
813 
814  // Run inflate() on input until output buffer not full
815  do {
816  strm.avail_out = kZChunk;
817  strm.next_out = out;
818  z_ret = inflate(&strm, Z_NO_FLUSH);
819  switch (z_ret) {
820  case Z_NEED_DICT:
821  z_ret = Z_DATA_ERROR; // and fall through
822  case Z_STREAM_ERROR:
823  case Z_DATA_ERROR:
824  case Z_MEM_ERROR:
825  DecompressFini(&strm);
826  free(*out_buf);
827  *out_buf = NULL;
828  *out_size = 0;
829  return false;
830  }
831  size_t have = kZChunk - strm.avail_out;
832  if (*out_size+have > alloc_size) {
833  alloc_size *= 2;
834  *out_buf = srealloc(*out_buf, alloc_size);
835  }
836  memcpy(static_cast<unsigned char *>(*out_buf) + *out_size, out, have);
837  *out_size += have;
838  } while (strm.avail_out == 0);
839 
840  pos += kZChunk;
841  } while (pos < size);
842 
843  DecompressFini(&strm);
844  if (z_ret != Z_STREAM_END) {
845  free(*out_buf);
846  *out_buf = NULL;
847  *out_size = 0;
848  return false;
849  }
850 
851  return true;
852 }
853 
854 
855 //------------------------------------------------------------------------------
856 
857 
858 void Compressor::RegisterPlugins() {
859  RegisterPlugin<ZlibCompressor>();
860  RegisterPlugin<EchoCompressor>();
861 }
862 
863 
864 //------------------------------------------------------------------------------
865 
866 
867 bool ZlibCompressor::WillHandle(const zlib::Algorithms &alg) {
868  return alg == kZlibDefault;
869 }
870 
871 
872 ZlibCompressor::ZlibCompressor(const Algorithms &alg)
873  : Compressor(alg)
874 {
875  stream_.zalloc = Z_NULL;
876  stream_.zfree = Z_NULL;
877  stream_.opaque = Z_NULL;
878  stream_.next_in = Z_NULL;
879  stream_.avail_in = 0;
880  const int zlib_retval = deflateInit(&stream_, Z_DEFAULT_COMPRESSION);
881  assert(zlib_retval == 0);
882 }
883 
884 
887  assert(stream_.avail_in == 0);
888  // Delete the other stream
889  int retcode = deflateEnd(&other->stream_);
890  assert(retcode == Z_OK);
891  retcode = deflateCopy(const_cast<z_streamp>(&other->stream_), &stream_);
892  assert(retcode == Z_OK);
893  return other;
894 }
895 
897  const bool flush,
898  unsigned char **inbuf, size_t *inbufsize,
899  unsigned char **outbuf, size_t *outbufsize)
900 {
901  // Adding compression
902  stream_.avail_in = *inbufsize;
903  stream_.next_in = *inbuf;
904  const int flush_int = (flush) ? Z_FINISH : Z_NO_FLUSH;
905  int retcode = 0;
906 
907  // TODO(jblomer) Figure out what exactly behaves differently with zlib 1.2.10
908  // if ((*inbufsize == 0) && !flush)
909  // return true;
910 
911  stream_.avail_out = *outbufsize;
912  stream_.next_out = *outbuf;
913 
914  // Deflate in zlib!
915  retcode = deflate(&stream_, flush_int);
916  assert(retcode == Z_OK || retcode == Z_STREAM_END);
917 
918  *outbufsize -= stream_.avail_out;
919  *inbuf = stream_.next_in;
920  *inbufsize = stream_.avail_in;
921 
922  return (flush_int == Z_NO_FLUSH && retcode == Z_OK && stream_.avail_in == 0)
923  || (flush_int == Z_FINISH && retcode == Z_STREAM_END);
924 }
925 
926 
928  int retcode = deflateEnd(&stream_);
929  assert(retcode == Z_OK);
930 }
931 
932 
933 size_t ZlibCompressor::DeflateBound(const size_t bytes) {
934  // Call zlib's deflate bound
935  return deflateBound(&stream_, bytes);
936 }
937 
938 
939 //------------------------------------------------------------------------------
940 
941 
943  Compressor(alg)
944 {
945 }
946 
947 
949  return alg == kNoCompression;
950 }
951 
952 
955 }
956 
957 
959  const bool flush,
960  unsigned char **inbuf, size_t *inbufsize,
961  unsigned char **outbuf, size_t *outbufsize)
962 {
963  size_t bytes_to_copy = min(*outbufsize, *inbufsize);
964  memcpy(*outbuf, *inbuf, bytes_to_copy);
965  const bool done = (bytes_to_copy == *inbufsize);
966 
967  // Update the return variables
968  *inbuf += bytes_to_copy;
969  *outbufsize = bytes_to_copy;
970  *inbufsize -= bytes_to_copy;
971 
972  return done;
973 }
974 
975 
976 size_t EchoCompressor::DeflateBound(const size_t bytes) {
977  // zero bytes as an upper bound is no good because some callers want to
978  // allocate buffers according to this value
979  return (bytes == 0) ? 1 : bytes;
980 }
981 
982 } // namespace zlib
bool CompressPath2Null(const string &src, shash::Any *compressed_hash)
Definition: compression.cc:515
#define LogCvmfs(source, mask,...)
Definition: logging.h:20
Algorithms ParseCompressionAlgorithm(const std::string &algorithm_option)
Definition: compression.cc:148
struct stat64 platform_stat64
StreamStates DecompressZStream2Sink(const void *buf, const int64_t size, z_stream *strm, cvmfs::Sink *sink)
Definition: compression.cc:234
const int kDefaultFileMode
Definition: posix.h:31
const unsigned kBufferSize
Definition: compression.cc:143
Compressor * Clone()
Definition: compression.cc:953
#define PANIC(...)
Definition: exception.h:26
bool CompressPath2File(const string &src, FILE *fdest, shash::Any *compressed_hash)
Definition: compression.cc:573
void Init(ContextPtr context)
Definition: hash.cc:164
EchoCompressor(const Algorithms &alg)
Definition: compression.cc:942
bool CopyMem2File(const unsigned char *buffer, const unsigned buffer_size, FILE *fdest)
Definition: compression.cc:86
static bool WillHandle(const zlib::Algorithms &alg)
Definition: compression.cc:948
void DecompressInit(z_stream *strm)
Definition: compression.cc:185
bool DecompressMem2Mem(const void *buf, const int64_t size, void **out_buf, uint64_t *out_size)
Definition: compression.cc:797
StreamStates DecompressZStream2File(const void *buf, const int64_t size, z_stream *strm, FILE *f)
Definition: compression.cc:275
std::string AlgorithmName(const zlib::Algorithms alg)
Definition: compression.cc:158
bool Deflate(const bool flush, unsigned char **inbuf, size_t *inbufsize, unsigned char **outbuf, size_t *outbufsize)
Definition: compression.cc:896
assert((mem||(size==0))&&"Out Of Memory")
StreamStates
Definition: compression.h:36
Algorithms algorithm
Definition: hash.h:124
Compressor * Clone()
Definition: compression.cc:885
void DecompressFini(z_stream *strm)
Definition: compression.cc:201
bool CopyPath2Path(const string &src, const string &dest)
Definition: compression.cc:63
bool CopyPath2Mem(const string &path, unsigned char **buffer, unsigned *buffer_size)
Definition: compression.cc:108
const unsigned kZChunk
Definition: compression.h:34
bool CopyPath2File(const std::string &src, FILE *fdest)
Definition: compression.cc:46
StreamStates CompressZStream2Null(const void *buf, const int64_t size, const bool eof, z_stream *strm, shash::ContextPtr *hash_context)
Definition: compression.cc:206
bool DecompressFile2File(FILE *fsrc, FILE *fdest)
Definition: compression.cc:640
bool CompressFile2File(FILE *fsrc, FILE *fdest)
Definition: compression.cc:525
size_t DeflateBound(const size_t bytes)
Definition: compression.cc:933
bool CopyMem2Path(const unsigned char *buffer, const unsigned buffer_size, const string &path)
Definition: compression.cc:94
Algorithms
Definition: compression.h:44
void Update(const unsigned char *buffer, const unsigned buffer_length, ContextPtr context)
Definition: hash.cc:190
void Final(ContextPtr context, Any *any_digest)
Definition: hash.cc:221
bool CompressFd2Null(int fd_src, shash::Any *compressed_hash, uint64_t *processed_bytes)
Definition: compression.cc:453
void CompressFini(z_stream *strm)
Definition: compression.cc:196
bool DecompressPath2File(const string &src, FILE *fdest)
Definition: compression.cc:667
void * buffer
Definition: hash.h:500
void CompressInit(z_stream *strm)
Definition: compression.cc:174
bool Deflate(const bool flush, unsigned char **inbuf, size_t *inbufsize, unsigned char **outbuf, size_t *outbufsize)
Definition: compression.cc:958
bool CompressFile2Null(FILE *fsrc, shash::Any *compressed_hash)
Definition: compression.cc:401
static bool CopyFile2File(FILE *fsrc, FILE *fdest)
Definition: compression.cc:32
bool CompressMem2File(const unsigned char *buf, const size_t size, FILE *fdest, shash::Any *compressed_hash)
Definition: compression.cc:678
bool DecompressPath2Path(const string &src, const string &dest)
Definition: compression.cc:381
bool CompressMem2Mem(const void *buf, const int64_t size, void **out_buf, uint64_t *out_size)
Definition: compression.cc:739
unsigned size
Definition: hash.h:501
int platform_fstat(int filedes, platform_stat64 *buf)
static void size_t size
Definition: smalloc.h:47
ZlibCompressor(const Algorithms &alg)
Definition: compression.cc:872
size_t DeflateBound(const size_t bytes)
Definition: compression.cc:976
bool CompressPath2Path(const string &src, const string &dest)
Definition: compression.cc:318
virtual int64_t Write(const void *buf, uint64_t sz)=0