5 from datetime
import datetime, timezone
6 from dxf
import DXF, hash_file, hash_bytes
7 from dxf.exceptions
import DXFUnauthorizedError
12 from requests.exceptions
import HTTPError
17 process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
18 output, error = process.communicate()
19 return (output, error)
23 Class which represents a "fat" OCI image configuration manifest
30 Method which initializes the cvmfs injection capability by adding an empty /cvmfs layer
31 to the image's fat manifest
33 if self.
manif[
"rootfs"][
"type"] !=
"layers":
34 raise ValueError(
"Cannot inject in rootfs of type " + self.
manif[
"rootfs"][
"type"])
35 self.
manif[
"rootfs"][
"diff_ids"].append(tar_digest)
38 local_time = datetime.now(timezone.utc).astimezone()
39 self.
manif[
"history"].append({
40 "created":local_time.isoformat(),
41 "created_by":
"/bin/sh -c #(nop) ADD file:"+tar_digest+
" in / ",
42 "author":
"cvmfs_shrinkwrap",
43 "comment":
"This change was executed through the CVMFS Shrinkwrap Docker Injector"
47 if "Labels" not in self.
manif[
"config"]:
48 self.
manif[
"config"][
"Labels"] = {}
49 self.
manif[
"config"][
"Labels"][
"cvmfs_injection_tar"] = tar_digest
50 self.
manif[
"config"][
"Labels"][
"cvmfs_injection_gz"] = gz_digest
52 if "container_config" in self.
manif:
53 if "Labels" not in self.
manif[
"config"]:
54 self.
manif[
"container_config"][
"Labels"] = {}
55 self.
manif[
"container_config"][
"Labels"][
"cvmfs_injection_tar"] = tar_digest
56 self.
manif[
"container_config"][
"Labels"][
"cvmfs_injection_gz"] = gz_digest
58 def inject(self, tar_digest, gz_digest):
60 Injects a new version of the layer by replacing the corresponding digests
63 raise ValueError(
"Cannot inject in unprepated image")
64 old_tar_digest = self.
manif[
"config"][
"Labels"][
"cvmfs_injection_tar"]
66 self.
manif[
"container_config"][
"Labels"][
"cvmfs_injection_tar"] = tar_digest
67 self.
manif[
"container_config"][
"Labels"][
"cvmfs_injection_gz"] = gz_digest
68 self.
manif[
"config"][
"Labels"][
"cvmfs_injection_tar"] = tar_digest
69 self.
manif[
"config"][
"Labels"][
"cvmfs_injection_gz"] = gz_digest
72 for i
in range(len(self.
manif[
"rootfs"][
"diff_ids"])):
73 if self.
manif[
"rootfs"][
"diff_ids"][i] == old_tar_digest:
74 self.
manif[
"rootfs"][
"diff_ids"][i] = tar_digest
78 raise ValueError(
"Image did not contain old cvmfs injection!")
80 local_time = datetime.now(timezone.utc).astimezone()
81 self.
manif[
"history"].append({
82 "created":local_time.isoformat(),
83 "created_by":
"/bin/sh -c #(nop) UPDATE file: from "+old_tar_digest+
" to "+tar_digest+
" in / ",
84 "author":
"cvmfs_shrinkwrap",
85 "comment":
"This change was executed through the CVMFS Shrinkwrap Docker Injector",
91 Checks whether image is prepared for cvmfs injection
93 return "cvmfs_injection_gz" in self.
manif[
"config"][
"Labels"]\
94 and "cvmfs_injection_tar" in self.
manif[
"config"][
"Labels"]
98 Retrieves the GZ digest necessary for layer downloading
100 return self.
manif[
"config"][
"Labels"][
"cvmfs_injection_gz"]
104 Retrieve JSON version of OCI manifest (for upload)
106 res = json.dumps(self.
manif)
111 Class which represents the "slim" image manifest used by the OCI distribution spec
117 Method for retrieving the digest (content address) of the manifest.
119 return self.
manif[
'config'][
'digest']
123 Method which initializes the cvmfs injection capability by adding an empty /cvmfs layer
124 to the image's slim manifest
126 self.
manif[
"layers"].append({
127 'mediaType':
'application/vnd.docker.image.rootfs.diff.tar.gzip',
129 'digest':layer_digest
131 self.
manif[
"config"][
"size"] = manifest_size
132 self.
manif[
"config"][
"digest"] = manifest_digest
133 def inject(self ,old, new, layer_size, manifest_digest, manifest_size):
135 Injects a new version of the layer by replacing the corresponding digest
137 for i
in range(len(self.
manif[
"layers"])):
138 if self.
manif[
"layers"][i][
"digest"] == old:
139 self.
manif[
"layers"][i][
"digest"] = new
140 self.
manif[
"layers"][i][
"size"] = layer_size
141 self.
manif[
"config"][
"size"] = manifest_size
142 self.
manif[
"config"][
"digest"] = manifest_digest
144 res = json.dumps(self.
manif)
149 The main class of the Docker injector which injects new versions of a layer into
150 OCI images retrieved from an OCI compliant distribution API
154 Initializes the injector by downloading both the slim and the fat image manifest
156 def auth(dxf, response):
157 dxf.authenticate(user, pw, response=response)
158 self.
dxfObject = DXF(host, repo, tlsverify=
True, auth=auth)
164 Sets an image up for layer injection
167 layer_size = self.dxfObject.blob_size(gz_digest)
168 self.fat_manifest.init_cvmfs_layer(tar_digest, gz_digest)
169 fat_man_json = self.fat_manifest.as_JSON()
170 manifest_digest = hash_bytes(bytes(fat_man_json,
'utf-8'))
171 self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
172 manifest_size = self.dxfObject.blob_size(manifest_digest)
173 self.image_manifest.init_cvmfs_layer(gz_digest, layer_size, manifest_digest, manifest_size)
175 image_man_json = self.image_manifest.as_JSON()
176 self.dxfObject.set_manifest(push_alias, image_man_json)
180 Unpacks the current version of a layer into the dest_dir directory in order to update it
182 if not self.fat_manifest.is_cvmfs_prepared():
183 os.makedirs(dest_dir+
"/cvmfs", exist_ok=
True)
186 gz_digest = self.fat_manifest.get_gz_digest()
188 decompress_object = zlib.decompressobj(16+zlib.MAX_WBITS)
190 chunk_it = self.dxfObject.pull_blob(gz_digest)
191 except HTTPError
as e:
192 if e.response.status_code == 404:
193 print(
"ERROR: The hash of the CVMFS layer must have changed.")
194 print(
"This is a known issue. Please do not reupload images to other repositories after CVMFS injection!")
197 with tempfile.TemporaryFile()
as tmp_file:
198 for chunk
in chunk_it:
199 tmp_file.write(decompress_object.decompress(chunk))
200 tmp_file.write(decompress_object.flush())
202 tar = tarfile.TarFile(fileobj=tmp_file)
203 tar.extractall(dest_dir)
208 Packs and uploads the contents of src_dir as a layer and injects the layer into the image.
209 The new layer version is stored under the tag push_alias
211 if not self.fat_manifest.is_cvmfs_prepared():
212 print(
"Preparing image for CVMFS injection...")
213 self.
setup(push_alias)
214 with tempfile.NamedTemporaryFile(delete=
False)
as tmp_file:
215 print(
"Bundling file into tar...")
216 _, error =
exec_bash(
"tar --xattrs -C "+src_dir+
" -cvf "+tmp_file.name+
" .")
218 raise RuntimeError(
"Failed to tar with error " + str(error))
219 tar_digest = hash_file(tmp_file.name)
220 print(
"Bundling tar into gz...")
221 gz_dest = tmp_file.name+
".gz"
222 _, error =
exec_bash(
"gzip "+tmp_file.name)
224 raise RuntimeError(
"Failed to tar with error " + str(error))
225 print(
"Uploading...")
226 gz_digest = self.dxfObject.push_blob(gz_dest)
228 print(
"Refreshing manifests...")
229 old_gz_digest = self.fat_manifest.get_gz_digest()
230 layer_size = self.dxfObject.blob_size(gz_digest)
231 self.fat_manifest.inject(tar_digest, gz_digest)
232 fat_man_json = self.fat_manifest.as_JSON()
233 manifest_digest = hash_bytes(bytes(fat_man_json,
'utf-8'))
234 self.dxfObject.push_blob(data=fat_man_json, digest=manifest_digest)
235 manifest_size = self.dxfObject.blob_size(manifest_digest)
237 self.image_manifest.inject(old_gz_digest, gz_digest, layer_size, manifest_digest, manifest_size)
239 image_man_json = self.image_manifest.as_JSON()
240 self.dxfObject.set_manifest(push_alias, image_man_json)
248 (readIter, _) = self.dxfObject.pull_blob(self.image_manifest.get_fat_manif_digest(), size=
True, chunk_size=4096)
249 for chunk
in readIter:
250 fat_manifest += str(chunk)[2:-1]
251 fat_manifest = fat_manifest.replace(
"\\\\",
"\\")
256 Builds an empty /cvmfs tar and uploads it to the registry
259 :returns: Tuple containing the tar digest and gz digest
261 ident = self.image_manifest.get_fat_manif_digest()[5:15]
262 tmp_name =
"/tmp/injector-"+ident
263 os.makedirs(tmp_name+
"/cvmfs", exist_ok=
True)
264 tar_dest =
"/tmp/"+ident+
".tar"
265 _, error =
exec_bash(
"tar --xattrs -C "+tmp_name+
" -cvf "+tar_dest+
" .")
267 print(
"Failed to tar with error " + str(error))
269 tar_digest = hash_file(tar_dest)
270 _, error =
exec_bash(
"gzip -n "+tar_dest)
272 print(
"Failed to tar with error " + str(error))
274 gz_dest = tar_dest+
".gz"
275 gzip_digest = self.dxfObject.push_blob(tar_dest+
".gz")
278 os.rmdir(tmp_name+
"/cvmfs")
281 return (tar_digest, gzip_digest)