sync_item.h

Go to the documentation of this file.

#ifndef CVMFS_SYNC_ITEM_H_
#define CVMFS_SYNC_ITEM_H_

#include <sys/types.h>

#if !defined(__APPLE__)
#include <sys/sysmacros.h>
#endif  // __APPLE__

#include <cstring>
#include <map>
#include <string>

#include "crypto/hash.h"
#include "directory_entry.h"
#include "duplex_libarchive.h"
#include "file_chunk.h"
#include "util/platform.h"
#include "util/shared_ptr.h"

class IngestionSource;

namespace publish {

enum SyncItemType {
  kItemDir,
  kItemFile,
  kItemSymlink,
  kItemCharacterDevice,
  kItemBlockDevice,
  kItemFifo,
  kItemSocket,
  kItemNew,
  kItemMarker,
  kItemUnknown,
};

class SyncUnion;
class SyncItem {
  // only SyncUnion can create SyncItems (see SyncUnion::CreateSyncItem).
  // SyncUnionTarball can create SyncItemTar and SyncItemDummyDir.

 public:
  SyncItem();
  virtual ~SyncItem();

  inline bool IsDirectory() const { return IsType(kItemDir); }
  inline bool WasDirectory() const { return WasType(kItemDir); }
  inline bool IsRegularFile() const { return IsType(kItemFile); }
  inline bool WasRegularFile() const { return WasType(kItemFile); }
  inline bool IsSymlink() const { return IsType(kItemSymlink); }
  inline bool WasSymlink() const { return WasType(kItemSymlink); }
  inline bool IsNew() const { return WasType(kItemNew); }
  inline bool IsTouched() const {
    return (GetRdOnlyFiletype() == GetUnionFiletype())
           && (GetRdOnlyFiletype() == GetScratchFiletype())
           && (GetUnionFiletype() == GetScratchFiletype());
  }
  inline bool IsCharacterDevice() const { return IsType(kItemCharacterDevice); }
  inline bool IsBlockDevice() const { return IsType(kItemBlockDevice); }
  inline bool IsFifo() const { return IsType(kItemFifo); }
  inline bool IsSocket() const { return IsType(kItemSocket); }
  inline bool IsGraftMarker() const { return IsType(kItemMarker); }
  inline bool IsExternalData() const { return external_data_; }
  inline bool IsDirectIo() const { return direct_io_; }

  inline bool IsWhiteout() const { return whiteout_; }
  inline bool IsCatalogMarker() const { return filename_ == ".cvmfscatalog"; }
  inline bool IsOpaqueDirectory() const { return IsDirectory() && opaque_; }

  inline bool IsSpecialFile() const {
    return IsCharacterDevice() || IsBlockDevice() || IsFifo() || IsSocket();
  }
  inline bool WasSpecialFile() const {
    return WasType(kItemCharacterDevice) || WasType(kItemBlockDevice)
           || WasType(kItemFifo) || WasType(kItemSocket);
  }
  inline bool IsBundleSpec() const { return filename_ == ".cvmfsbundles"; }
  inline bool WasBundleSpec() const { return filename_ == ".cvmfsbundles"; }

  inline unsigned int GetRdevMajor() const {
    assert(IsSpecialFile());
    StatUnion(true);
    return major(union_stat_.stat.st_rdev);
  }

  inline unsigned int GetRdevMinor() const {
    assert(IsSpecialFile());
    StatUnion(true);
    return minor(union_stat_.stat.st_rdev);
  }

  bool HasCatalogMarker() const { return has_catalog_marker_; }
  bool HasGraftMarker() const { return graft_marker_present_; }
  bool HasCompressionAlgorithm() const { return has_compression_algorithm_; }
  bool IsValidGraft() const { return valid_graft_; }
  bool IsChunkedGraft() const { return graft_chunklist_; }

  inline const FileChunkList *GetGraftChunks() const {
    return graft_chunklist_;
  }
  inline shash::Any GetContentHash() const { return content_hash_; }
  inline void SetContentHash(const shash::Any &hash) { content_hash_ = hash; }
  inline bool HasContentHash() const { return !content_hash_.IsNull(); }
  void SetExternalData(bool val) { external_data_ = val; }
  void SetDirectIo(bool val) { direct_io_ = val; }

  inline zlib::Algorithms GetCompressionAlgorithm() const {
    return compression_algorithm_;
  }
  inline void SetCompressionAlgorithm(const zlib::Algorithms &alg) {
    compression_algorithm_ = alg;
    has_compression_algorithm_ = true;
  }

  virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent(
      bool enable_mtime_ns) const = 0;

  inline std::string GetRelativePath() const {
    return (relative_parent_path_.empty())
               ? filename_
               : relative_parent_path_
                     + (filename_.empty() ? "" : ("/" + filename_));
  }

  std::string GetRdOnlyPath() const;
  std::string GetUnionPath() const;
  std::string GetScratchPath() const;

  void MarkAsWhiteout(const std::string &actual_filename);
  void MarkAsOpaqueDirectory();

  inline void MaskHardlink() { masked_hardlink_ = true; }
  inline bool HasHardlinks() const {
    return !masked_hardlink_ && GetUnionLinkcount() > 1;
  }

  unsigned int GetRdOnlyLinkcount() const;
  uint64_t GetRdOnlyInode() const;
  unsigned int GetUnionLinkcount() const;
  uint64_t GetUnionInode() const;
  uint64_t GetScratchSize() const;
  uint64_t GetRdOnlySize() const;

  inline std::string filename() const { return filename_; }
  inline std::string relative_parent_path() const {
    return relative_parent_path_;
  }

  virtual IngestionSource *CreateIngestionSource() const = 0;
  virtual void MakePlaceholderDirectory() const = 0;
  void SetCatalogMarker() { has_catalog_marker_ = true; }

  bool operator==(const SyncItem &other) const {
    return ((relative_parent_path_ == other.relative_parent_path_)
            && (filename_ == other.filename_));
  }

 protected:
  SyncItem(const std::string &relative_parent_path,
           const std::string &filename,
           const SyncUnion *union_engine,
           const SyncItemType entry_type);

  inline platform_stat64 GetUnionStat() const {
    StatUnion();
    return union_stat_.stat;
  }

  SyncItemType GetRdOnlyFiletype() const;
  SyncItemType GetUnionFiletype() const;

  virtual SyncItemType GetScratchFiletype() const = 0;

  virtual bool IsType(const SyncItemType expected_type) const = 0;

  inline bool WasType(const SyncItemType expected_type) const {
    if (rdonly_type_ == kItemUnknown) {
      rdonly_type_ = GetRdOnlyFiletype();
    }
    return rdonly_type_ == expected_type;
  }

  struct EntryStat {
    EntryStat() : obtained(false), error_code(0) {
      memset(&stat, 0, sizeof(stat));
    }

    inline SyncItemType GetSyncItemType() const {
      assert(obtained);
      if (S_ISREG(stat.st_mode))
        return kItemFile;
      if (S_ISLNK(stat.st_mode))
        return kItemSymlink;
      if (S_ISDIR(stat.st_mode))
        return kItemDir;
      if (S_ISFIFO(stat.st_mode))
        return kItemFifo;
      if (S_ISSOCK(stat.st_mode))
        return kItemSocket;
      if (S_ISCHR(stat.st_mode))
        return kItemCharacterDevice;
      if (S_ISBLK(stat.st_mode))
        return kItemBlockDevice;
      return kItemUnknown;
    }

    bool obtained;  
    int error_code; 
    platform_stat64 stat;
  };

  static void StatGeneric(const std::string &path,
                          EntryStat *info,
                          const bool refresh);
  SyncItemType GetGenericFiletype(const EntryStat &stat) const;
  void CheckMarkerFiles();

  mutable SyncItemType rdonly_type_;
  mutable EntryStat scratch_stat_;

  ssize_t graft_size_;

  // The hash of regular file's content
  shash::Any content_hash_;

  mutable SyncItemType scratch_type_;

 private:
  void CheckCatalogMarker();

  std::string filename_;

  std::string GetGraftMarkerPath() const;
  void CheckGraft();

  const SyncUnion *union_engine_; 
  mutable EntryStat rdonly_stat_;
  mutable EntryStat union_stat_;

  bool whiteout_;             
  bool opaque_;               
  bool masked_hardlink_;      
  bool has_catalog_marker_;   
  bool valid_graft_;          
  bool graft_marker_present_; 
  bool external_data_;
  bool direct_io_;
  std::string relative_parent_path_;

  FileChunkList *graft_chunklist_;

  // The compression algorithm for the file
  zlib::Algorithms compression_algorithm_;
  // The compression algorithm has been set explicitly
  bool has_compression_algorithm_;

  // Lazy evaluation and caching of results of file stats
  inline void StatRdOnly(const bool refresh = false) const {
    StatGeneric(GetRdOnlyPath(), &rdonly_stat_, refresh);
  }
  inline void StatUnion(const bool refresh = false) const {
    StatGeneric(GetUnionPath(), &union_stat_, refresh);
  }
  virtual void StatScratch(const bool refresh) const = 0;
};

typedef std::map<std::string, SharedPtr<SyncItem> > SyncItemList;

class SyncItemNative : public SyncItem {
  friend class SyncUnion;
  virtual catalog::DirectoryEntryBase CreateBasicCatalogDirent(
      bool enable_mtime_ns) const;
  virtual IngestionSource *CreateIngestionSource() const;
  virtual void MakePlaceholderDirectory() const { assert(false); }
  virtual SyncItemType GetScratchFiletype() const;
  virtual bool IsType(const SyncItemType expected_type) const;
  virtual void StatScratch(const bool refresh) const {
    StatGeneric(GetScratchPath(), &scratch_stat_, refresh);
  }

 protected:
  SyncItemNative(const std::string &relative_parent_path,
                 const std::string &filename, const SyncUnion *union_engine,
                 const SyncItemType entry_type)
      : SyncItem(relative_parent_path, filename, union_engine, entry_type) {
    CheckMarkerFiles();
  }
};

}  // namespace publish

#endif  // CVMFS_SYNC_ITEM_H_