CernVM-FS  2.9.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
docker_injector.py
Go to the documentation of this file.
1 #
2 # This file is part of the CernVM File System.
3 #
4 
5 from datetime import datetime, timezone
6 from dxf import DXF, hash_file, hash_bytes
7 from dxf.exceptions import DXFUnauthorizedError
8 import json
9 import subprocess
10 import tarfile
11 import tempfile
12 from requests.exceptions import HTTPError
13 import os
14 import zlib
15 
16 def exec_bash(cmd):
17  process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
18  output, error = process.communicate()
19  return (output, error)
20 
22  """
23  Class which represents a "fat" OCI image configuration manifest
24  """
25  def __init__(self, manif):
26  self.manif = json.loads(manif)
27 
28  def init_cvmfs_layer(self, tar_digest, gz_digest):
29  """
30  Method which initializes the cvmfs injection capability by adding an empty /cvmfs layer
31  to the image's fat manifest
32  """
33  if self.manif["rootfs"]["type"] != "layers":
34  raise ValueError("Cannot inject in rootfs of type " + self.manif["rootfs"]["type"])
35  self.manif["rootfs"]["diff_ids"].append(tar_digest)
36 
37  # Write history
38  local_time = datetime.now(timezone.utc).astimezone()
39  self.manif["history"].append({
40  "created":local_time.isoformat(),
41  "created_by":"/bin/sh -c #(nop) ADD file:"+tar_digest+" in / ",
42  "author":"cvmfs_shrinkwrap",
43  "comment": "This change was executed through the CVMFS Shrinkwrap Docker Injector"
44  })
45 
46  # Setup labels
47  if "Labels" not in self.manif["config"]:
48  self.manif["config"]["Labels"] = {}
49  self.manif["config"]["Labels"]["cvmfs_injection_tar"] = tar_digest
50  self.manif["config"]["Labels"]["cvmfs_injection_gz"] = gz_digest
51 
52  if "container_config" in self.manif:
53  if "Labels" not in self.manif["config"]:
54  self.manif["container_config"]["Labels"] = {}
55  self.manif["container_config"]["Labels"]["cvmfs_injection_tar"] = tar_digest
56  self.manif["container_config"]["Labels"]["cvmfs_injection_gz"] = gz_digest
57 
58  def inject(self, tar_digest, gz_digest):
59  """
60  Injects a new version of the layer by replacing the corresponding digests
61  """
62  if not self.is_cvmfs_prepared():
63  raise ValueError("Cannot inject in unprepated image")
64  old_tar_digest = self.manif["config"]["Labels"]["cvmfs_injection_tar"]
65 
66  self.manif["container_config"]["Labels"]["cvmfs_injection_tar"] = tar_digest
67  self.manif["container_config"]["Labels"]["cvmfs_injection_gz"] = gz_digest
68  self.manif["config"]["Labels"]["cvmfs_injection_tar"] = tar_digest
69  self.manif["config"]["Labels"]["cvmfs_injection_gz"] = gz_digest
70 
71  found = False
72  for i in range(len(self.manif["rootfs"]["diff_ids"])):
73  if self.manif["rootfs"]["diff_ids"][i] == old_tar_digest:
74  self.manif["rootfs"]["diff_ids"][i] = tar_digest
75  found = True
76  break
77  if not found:
78  raise ValueError("Image did not contain old cvmfs injection!")
79 
80  local_time = datetime.now(timezone.utc).astimezone()
81  self.manif["history"].append({
82  "created":local_time.isoformat(),
83  "created_by":"/bin/sh -c #(nop) UPDATE file: from "+old_tar_digest+" to "+tar_digest+" in / ",
84  "author":"cvmfs_shrinkwrap",
85  "comment": "This change was executed through the CVMFS Shrinkwrap Docker Injector",
86  "empty_layer":True
87  })
88 
89  def is_cvmfs_prepared(self):
90  """
91  Checks whether image is prepared for cvmfs injection
92  """
93  return "cvmfs_injection_gz" in self.manif["config"]["Labels"]\
94  and "cvmfs_injection_tar" in self.manif["config"]["Labels"]
95 
96  def get_gz_digest(self):
97  """
98  Retrieves the GZ digest necessary for layer downloading
99  """
100  return self.manif["config"]["Labels"]["cvmfs_injection_gz"]
101 
102  def as_JSON(self):
103  """
104  Retrieve JSON version of OCI manifest (for upload)
105  """
106  res = json.dumps(self.manif)
107  return res
108 
110  """
111  Class which represents the "slim" image manifest used by the OCI distribution spec
112  """
113  def __init__(self, manif):
114  self.manif = json.loads(manif)
116  """
117  Method for retrieving the digest (content address) of the manifest.
118  """
119  return self.manif['config']['digest']
120 
121  def init_cvmfs_layer(self, layer_digest, layer_size, manifest_digest, manifest_size):
122  """
123  Method which initializes the cvmfs injection capability by adding an empty /cvmfs layer
124  to the image's slim manifest
125  """
126  self.manif["layers"].append({
127  'mediaType':'application/vnd.docker.image.rootfs.diff.tar.gzip',
128  'size':layer_size,
129  'digest':layer_digest
130  })
131  self.manif["config"]["size"] = manifest_size
132  self.manif["config"]["digest"] = manifest_digest
133  def inject(self ,old, new, layer_size, manifest_digest, manifest_size):
134  """
135  Injects a new version of the layer by replacing the corresponding digest
136  """
137  for i in range(len(self.manif["layers"])):
138  if self.manif["layers"][i]["digest"] == old:
139  self.manif["layers"][i]["digest"] = new
140  self.manif["layers"][i]["size"] = layer_size
141  self.manif["config"]["size"] = manifest_size
142  self.manif["config"]["digest"] = manifest_digest
143  def as_JSON(self):
144  res = json.dumps(self.manif)
145  return res
146 
148  """
149  The main class of the Docker injector which injects new versions of a layer into
150  OCI images retrieved from an OCI compliant distribution API
151  """
152  def __init__(self, host, repo, alias, user, pw):
153  """
154  Initializes the injector by downloading both the slim and the fat image manifest
155  """
156  def auth(dxf, response):
157  dxf.authenticate(user, pw, response=response)
158  self.dxfObject = DXF(host, repo, tlsverify=True, auth=auth)
159  self.image_manifest = self._get_manifest(alias)
161 
162  def setup(self, push_alias):
163  """
164  Sets an image up for layer injection
165  """
166  tar_digest, gz_digest = self._build_init_tar()
167  layer_size = self.dxfObject.blob_size(gz_digest)
168  self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest)
169  fat_man_json = self.fat_manifest.as_JSON()
170  manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
171  self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
172  manifest_size = self.dxfObject.blob_size(manifest_digest)
173  self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size)
174 
175  image_man_json = self.image_manifest.as_JSON()
176  self.dxfObject.set_manifest(push_alias, image_man_json)
177 
178  def unpack(self, dest_dir):
179  """
180  Unpacks the current version of a layer into the dest_dir directory in order to update it
181  """
182  if not self.fat_manifest.is_cvmfs_prepared():
183  os.makedirs(dest_dir+"/cvmfs", exist_ok=True)
184  return
185 
186  gz_digest = self.fat_manifest.get_gz_digest()
187  # Write out tar file
188  decompress_object = zlib.decompressobj(16+zlib.MAX_WBITS)
189  try:
190  chunk_it = self.dxfObject.pull_blob(gz_digest)
191  except HTTPError as e:
192  if e.response.status_code == 404:
193  print("ERROR: The hash of the CVMFS layer must have changed.")
194  print("This is a known issue. Please do not reupload images to other repositories after CVMFS injection!")
195  else:
196  raise e
197  with tempfile.TemporaryFile() as tmp_file:
198  for chunk in chunk_it:
199  tmp_file.write(decompress_object.decompress(chunk))
200  tmp_file.write(decompress_object.flush())
201  tmp_file.seek(0)
202  tar = tarfile.TarFile(fileobj=tmp_file)
203  tar.extractall(dest_dir)
204  tar.close()
205 
206  def update(self, src_dir, push_alias):
207  """
208  Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
209  The new layer version is stored under the tag push_alias
210  """
211  if not self.fat_manifest.is_cvmfs_prepared():
212  print("Preparing image for CVMFS injection...")
213  self.setup(push_alias)
214  with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
215  print("Bundling file into tar...")
216  _, error = exec_bash("tar --xattrs -C "+src_dir+" -cvf "+tmp_file.name+" .")
217  if error:
218  raise RuntimeError("Failed to tar with error " + str(error))
219  tar_digest = hash_file(tmp_file.name)
220  print("Bundling tar into gz...")
221  gz_dest = tmp_file.name+".gz"
222  _, error = exec_bash("gzip "+tmp_file.name)
223  if error:
224  raise RuntimeError("Failed to tar with error " + str(error))
225  print("Uploading...")
226  gz_digest = self.dxfObject.push_blob(gz_dest)
227  os.unlink(gz_dest)
228  print("Refreshing manifests...")
229  old_gz_digest = self.fat_manifest.get_gz_digest()
230  layer_size = self.dxfObject.blob_size(gz_digest)
231  self.fat_manifest.inject(tar_digest, gz_digest)
232  fat_man_json = self.fat_manifest.as_JSON()
233  manifest_digest = hash_bytes(bytes(fat_man_json, 'utf-8'))
234  self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
235  manifest_size = self.dxfObject.blob_size(manifest_digest)
236 
237  self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size)
238 
239  image_man_json = self.image_manifest.as_JSON()
240  self.dxfObject.set_manifest(push_alias, image_man_json)
241 
242 
243  def _get_manifest(self, alias):
244  return ImageManifest(self.dxfObject.get_manifest(alias))
245 
246  def _get_fat_manifest(self, image_manifest):
247  fat_manifest = ""
248  (readIter, _) = self.dxfObject.pull_blob(self.image_manifest.get_fat_manif_digest(), size=True, chunk_size=4096)
249  for chunk in readIter:
250  fat_manifest += str(chunk)[2:-1]
251  fat_manifest = fat_manifest.replace("\\\\","\\")
252  return FatManifest(fat_manifest)
253 
254  def _build_init_tar(self):
255  """
256  Builds an empty /cvmfs tar and uploads it to the registry
257 
258  :rtype: tuple
259  :returns: Tuple containing the tar digest and gz digest
260  """
261  ident = self.image_manifest.get_fat_manif_digest()[5:15]
262  tmp_name = "/tmp/injector-"+ident
263  os.makedirs(tmp_name+"/cvmfs", exist_ok=True)
264  tar_dest = "/tmp/"+ident+".tar"
265  _, error = exec_bash("tar --xattrs -C "+tmp_name+" -cvf "+tar_dest+" .")
266  if error:
267  print("Failed to tar with error " + str(error))
268  return
269  tar_digest = hash_file(tar_dest)
270  _, error = exec_bash("gzip -n "+tar_dest)
271  if error:
272  print("Failed to tar with error " + str(error))
273  return
274  gz_dest = tar_dest+".gz"
275  gzip_digest = self.dxfObject.push_blob(tar_dest+".gz")
276 
277  # Cleanup
278  os.rmdir(tmp_name+"/cvmfs")
279  os.rmdir(tmp_name)
280  os.unlink(gz_dest)
281  return (tar_digest, gzip_digest)