Last active 1746658351 Unlisted

dch's Avatar dch revised this gist 1746658351. Go to revision

1 file changed, 3 insertions, 4 deletions

thermite.sh.diff

@@ -2,15 +2,14 @@ Index: thermite.sh
2 2 ===================================================================
3 3 --- thermite.sh (revision 2811)
4 4 +++ thermite.sh (working copy)
5 - @@ -679,6 +679,34 @@
5 + @@ -679,6 +679,33 @@
6 6 return 0
7 7 }
8 8
9 9 +# Merge OCI images into a single image for subsequent upload.
10 10 +merge_oci_images() {
11 11 + source_config || return 0
12 - + _build="${rev}-${arch}-${kernel}-${type}"
13 - + info "Merging OCI images for build: ${_build}"
12 + + info "Merging OCI images into ${OCI_DESTDIR}"
14 13 +
15 14 + # define paths for aarch64 and amd64 container images to merge
16 15 + # /releng/15-aarch64-GENERIC-snap/R/ociimages/container-image-$type.txz
@@ -37,7 +36,7 @@ Index: thermite.sh
37 36 main() {
38 37 releasesrc="main"
39 38 export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
40 - @@ -719,6 +747,10 @@
39 + @@ -719,6 +746,10 @@
41 40 runall ${parallel}build_release
42 41 wait
43 42 fi

dch's Avatar dch revised this gist 1746658271. Go to revision

No changes

dch's Avatar dch revised this gist 1746656444. Go to revision

2 files changed, 50 insertions, 86 deletions

thermite.sh (file deleted)

@@ -1,86 +0,0 @@
1 -
2 - # Merge OCI images into a single image for subsequent upload.
3 - merge_oci_images() {
4 - source_config || return 0
5 - info "Merging OCI images"
6 -
7 - # define paths for aarch64 and amd64 container images to merge
8 - # /releng/15-aarch64-GENERIC-snap/R/ociimages/container-image-$type.txz
9 - # /releng/15-amd64-GENERIC-snap/R/ociimages/container-image-$type.txz
10 - aarch64_path="/${zfs_mount}/${heads}-aarch64-GENERIC-${type}/R/ociimages/container-image"
11 - amd64_path="/${zfs_mount}/${heads}-amd64-GENERIC-${type}/R/ociimages/container-image"
12 -
13 - # define destination for merged OCI images, probably should be a dataset
14 - test -d ${OCI_DESTDIR} || mkdir -p ${OCI_DESTDIR}
15 -
16 - # podmanic FreeBSD-14.3-PRERELEASE-arm64-aarch64-container-image-$t.txz \
17 - # FreeBSD-14.3-PRERELEASE-amd64-container-image-$t.txz \
18 - # freebsd-$t-14.3-prerelease.oci
19 - for t in ${oci_images}; do
20 - aarch64="${aarch64_path}-${t}.txz"
21 - amd64="${amd64_path}-${t}.txz"
22 - oci="${OCI_DESTDIR}/freebsd-${t}-${heads}"
23 - podmanic $aarch64 $amd64 $oci >> ${logdir}/${_build}.log 2>&1
24 - sha256 ${oci}.oci > ${oci}.oci.sha256 >> ${logdir}/${_build}.log 2>&1
25 - done
26 - return 0
27 - }
28 -
29 - main() {
30 - releasesrc="main"
31 - export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
32 -
33 - while getopts "bc:du" opt; do
34 - case ${opt} in
35 - b)
36 - DOBUILD=YES
37 - ;;
38 - c)
39 - CONF=${OPTARG}
40 - [ -e ${CONF} ] && . $(realpath ${CONF})
41 - ;;
42 - d)
43 - debug=1
44 - ;;
45 - u)
46 - DOUPLOAD=YES
47 - ;;
48 - \?)
49 - usage
50 - ;;
51 - esac
52 - done
53 - shift $(($OPTIND - 1))
54 - [ -z ${CONF} ] && usage
55 - if [ "$DOBUILD" ]; then
56 - check_zfs_mounts
57 - use_zfs=1
58 - check_use_zfs
59 - runall prebuild_setup
60 - runall truncate_logs
61 - runall zfs_mount_src
62 - runall build_chroots
63 - runall install_chroots
64 - runall zfs_clone_chroots
65 - zfs_finish_bootstrap
66 - runall ${parallel}build_release
67 - wait
68 - fi
69 - if [ "$OCI_MERGE" ]; then
70 - merge_oci_images
71 - # TODO bug_colin_to_sign_them
72 - fi
73 - if [ "$DOUPLOAD" ]; then
74 - runall upload_mount_devfs
75 - runall upload_ec2_ami
76 - runall upload_gce_image
77 - runall upload_vagrant_image
78 - runall upload_azure_image
79 - runall upload_oracle_image
80 - wait
81 - runall upload_umount_devfs
82 - fi
83 - send_completed_email
84 - }
85 -
86 - main "$@"

thermite.sh.diff(file created)

@@ -0,0 +1,50 @@
1 + Index: thermite.sh
2 + ===================================================================
3 + --- thermite.sh (revision 2811)
4 + +++ thermite.sh (working copy)
5 + @@ -679,6 +679,34 @@
6 + return 0
7 + }
8 +
9 + +# Merge OCI images into a single image for subsequent upload.
10 + +merge_oci_images() {
11 + + source_config || return 0
12 + + _build="${rev}-${arch}-${kernel}-${type}"
13 + + info "Merging OCI images for build: ${_build}"
14 + +
15 + + # define paths for aarch64 and amd64 container images to merge
16 + + # /releng/15-aarch64-GENERIC-snap/R/ociimages/container-image-$type.txz
17 + + # /releng/15-amd64-GENERIC-snap/R/ociimages/container-image-$type.txz
18 + + aarch64_path="/${zfs_mount}/${heads}-aarch64-GENERIC-${type}/R/ociimages/container-image"
19 + + amd64_path="/${zfs_mount}/${heads}-amd64-GENERIC-${type}/R/ociimages/container-image"
20 + +
21 + + # define destination for merged OCI images, probably should be a dataset
22 + + test -d ${OCI_DESTDIR} || mkdir -p ${OCI_DESTDIR}
23 + +
24 + + # podmanic FreeBSD-14.3-PRERELEASE-arm64-aarch64-container-image-$t.txz \
25 + + # FreeBSD-14.3-PRERELEASE-amd64-container-image-$t.txz \
26 + + # freebsd-$t-14.3-prerelease.oci
27 + + for t in ${oci_images}; do
28 + + aarch64="${aarch64_path}-${t}.txz"
29 + + amd64="${amd64_path}-${t}.txz"
30 + + oci="${OCI_DESTDIR}/freebsd-${t}-${heads}"
31 + + podmanic $aarch64 $amd64 $oci >> ${logdir}/${_build}.log 2>&1
32 + + sha256 ${oci}.oci > ${oci}.oci.sha256 >> ${logdir}/${_build}.log 2>&1
33 + + done
34 + + return 0
35 + +}
36 + +
37 + main() {
38 + releasesrc="main"
39 + export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
40 + @@ -719,6 +747,10 @@
41 + runall ${parallel}build_release
42 + wait
43 + fi
44 + + if [ "$OCI_MERGE" ]; then
45 + + merge_oci_images
46 + + # TODO bug_colin_to_sign_them
47 + + fi
48 + if [ "$DOUPLOAD" ]; then
49 + runall upload_mount_devfs
50 + runall upload_ec2_ami

dch's Avatar dch revised this gist 1746656372. Go to revision

3 files changed, 295 insertions

dch.conf(file created)

@@ -0,0 +1,45 @@
1 + #!/bin/sh
2 + #
3 + # $FreeBSD: tools/git-release/builds-15.conf 2804 2025-04-21 17:34:35Z cperciva $
4 + #
5 + # usage
6 + # zfs destroy -vrf zroot/releng
7 + # zfs create -o sync=disabled -o mountpoint=/releng zroot/releng
8 + # zfs create -o mountpoint=/releng zroot/releng
9 + # ./thermite.sh -b -d -c dch.conf
10 +
11 + . "${__BUILDCONFDIR}/main.conf"
12 +
13 + # overrides
14 + SRC_UPDATE_SKIP=1
15 + PORTS_UPDATE_SKIP=1
16 + DOC_UPDATE_SKIP=1
17 +
18 + # use local git repositories
19 + GITROOT="file:///usr"
20 + GITSRC="src"
21 + GITDOC="../projects/freebsd/doc"
22 + GITPORTS="ports"
23 +
24 + releasesrc="main"
25 + heads=15
26 + stables=
27 +
28 + revs="${heads} ${stables}"
29 + archs="amd64 aarch64"
30 + types="snap"
31 +
32 + x86_kernels="GENERIC"
33 + kernels="${x86_kernels}"
34 +
35 + portsbranch="quarterly"
36 + srcbranch="${releasesrc}"
37 + docbranch="main"
38 +
39 + # enable merging of OCI images into multi-architectures
40 + OCI_MERGE=1
41 + # where to put the merged OCI images
42 + OCI_DESTDIR="/releng/oci-images"
43 + # what are the images we need to merge
44 + oci_images="static dynamic runtime"
45 + # the architectures are already defined previously

podmanic(file created)

@@ -0,0 +1,164 @@
1 + #!/usr/libexec/flua
2 +
3 + -- check if we have enough arguments
4 + if #arg < 3 then
5 + print("usage: podmanic <aarch64.txz> <amd64.txz> <manifest> [tags...]")
6 + os.exit(1)
7 + end
8 +
9 + function execute_command(command)
10 + print("executing: " .. command)
11 + local handle = io.popen(command)
12 + local result = handle:read("*a")
13 + local success, _, code = handle:close()
14 + return result:gsub("%s+$", ""), success, code
15 + end
16 +
17 + function import_image(txz_path)
18 + print("importing " .. txz_path)
19 + local output, success = execute_command("podman load -i=" .. txz_path)
20 +
21 + if not success then
22 + print("FATAL: failed to import " .. txz_path)
23 + print("command output: " .. output)
24 + print("make sure the file exists and is a valid container image")
25 + os.exit(1)
26 + end
27 +
28 + -- extract the image id from the output
29 + -- output format is typically: "Loaded image: localhost/freebsd-static:14.2-stable-amd64"
30 + local image_id = output:match("Loaded image: ([^\n]+)")
31 +
32 + if not image_id then
33 + -- if still not found, try to list images and find the most recently loaded one
34 + print("FATAL: could not determine image id from output")
35 + print("command output: " .. output)
36 + print("failed to extract image id using pattern matching and fallback method")
37 + os.exit(1)
38 + end
39 +
40 + print("found image: " .. image_id)
41 + return image_id
42 + end
43 +
44 + function create_manifest(manifest_name, image_ids)
45 + -- First, try to remove any existing manifest with the same name
46 + local rm_cmd = "podman manifest rm " .. manifest_name .. " 2>/dev/null || true"
47 + execute_command(rm_cmd)
48 +
49 + -- Create empty manifest
50 + local create_cmd = "podman manifest create " .. manifest_name
51 + local output, success = execute_command(create_cmd)
52 +
53 + if not success then
54 + print("FATAL: Failed to create manifest: " .. output)
55 + os.exit(1)
56 + end
57 +
58 + -- add each image to the manifest
59 + for i, image_id in ipairs(image_ids) do
60 + local arch = i == 1 and "arm64" or "amd64"
61 + local add_cmd = "podman manifest add --arch " .. arch .. " " .. manifest_name .. " " .. image_id
62 + local add_output, add_success = execute_command(add_cmd)
63 +
64 + if not add_success then
65 + print("FATAL: failed to add " .. image_id .. " to manifest: " .. add_output)
66 + os.exit(1)
67 + end
68 + end
69 +
70 + print("successfully created manifest: " .. manifest_name)
71 + return manifest_name
72 + end
73 +
74 + function apply_tags(manifest_name, tags)
75 + for _, tag in ipairs(tags) do
76 + local tag_cmd = "podman tag " .. manifest_name .. " " .. tag
77 + local output, success = execute_command(tag_cmd)
78 +
79 + if not success then
80 + print("ERROR: failed to apply tag " .. tag .. ": " .. output)
81 + else
82 + print("applied tag: " .. tag)
83 +
84 + -- also push the additional tags if we're pushing to a registry
85 + -- if manifest_name:match("^[^/]+%.") or manifest_name:match("^localhost/") then
86 + -- local push_tag_cmd = "podman manifest push --all " .. tag .. " " .. tag
87 + -- local push_output, push_success = execute_command(push_tag_cmd)
88 +
89 + -- if push_success then
90 + -- print("successfully pushed tag: " .. tag)
91 + -- else
92 + -- print("ERROR: failed to push tag " .. tag .. ": " .. push_output)
93 + -- end
94 + -- end
95 + end
96 + end
97 + end
98 +
99 + function export_manifest(manifest_name)
100 + local file = manifest_name .. ".oci"
101 + -- remove all text up to / from filename, and replace all : with -
102 + file = file:gsub("^.+/", "")
103 + file = file:gsub(":", "-")
104 + local push_cmd = "podman manifest push --all " .. manifest_name .. " oci-archive:" .. file
105 + local output, success = execute_command(push_cmd)
106 +
107 + if not success then
108 + print("FATAL: failed to push manifest " .. manifest_name .. ": " .. output)
109 + print("- make sure you're logged in via podman login")
110 + return false
111 + end
112 +
113 + print("successfully pushed manifest: " .. manifest_name .. " to " .. file)
114 + return true
115 + end
116 +
117 + -- get arguments
118 + local manifest_name = arg[3]
119 + local txz_arm64 = arg[1] -- arm64 image
120 + local txz_amd64 = arg[2] -- amd64 image
121 + local tags = {}
122 +
123 + -- collect additional tags
124 + for i = 4, #arg do
125 + table.insert(tags, arg[i])
126 + end
127 +
128 + print("podmanic is processing:")
129 + print("- manifest: " .. manifest_name)
130 + print("- arm64 image: " .. txz_arm64)
131 + print("- amd64 image: " .. txz_amd64)
132 + if #tags > 0 then
133 + print("- additional tags: " .. table.concat(tags, ", "))
134 + end
135 + print("")
136 +
137 + -- import images
138 + local image_arm64 = import_image(txz_arm64)
139 + local image_amd64 = import_image(txz_amd64)
140 +
141 + if not image_arm64 or not image_amd64 then
142 + print("FATAL: failed to import one or more images")
143 + os.exit(1)
144 + end
145 +
146 + -- create manifest
147 + local manifest = create_manifest(manifest_name, {image_arm64, image_amd64})
148 + if not manifest then
149 + print("FATAL: failed to create or populate manifest")
150 + os.exit(1)
151 + end
152 +
153 + -- apply additional tags
154 + if #tags > 0 then
155 + apply_tags(manifest_name, tags)
156 + end
157 +
158 + -- export manifest
159 + if not export_manifest(manifest_name) then
160 + print("FATAL: failed to publish manifest")
161 + os.exit(1)
162 + end
163 +
164 + print("OK: great success")

thermite.sh(file created)

@@ -0,0 +1,86 @@
1 +
2 + # Merge OCI images into a single image for subsequent upload.
3 + merge_oci_images() {
4 + source_config || return 0
5 + info "Merging OCI images"
6 +
7 + # define paths for aarch64 and amd64 container images to merge
8 + # /releng/15-aarch64-GENERIC-snap/R/ociimages/container-image-$type.txz
9 + # /releng/15-amd64-GENERIC-snap/R/ociimages/container-image-$type.txz
10 + aarch64_path="/${zfs_mount}/${heads}-aarch64-GENERIC-${type}/R/ociimages/container-image"
11 + amd64_path="/${zfs_mount}/${heads}-amd64-GENERIC-${type}/R/ociimages/container-image"
12 +
13 + # define destination for merged OCI images, probably should be a dataset
14 + test -d ${OCI_DESTDIR} || mkdir -p ${OCI_DESTDIR}
15 +
16 + # podmanic FreeBSD-14.3-PRERELEASE-arm64-aarch64-container-image-$t.txz \
17 + # FreeBSD-14.3-PRERELEASE-amd64-container-image-$t.txz \
18 + # freebsd-$t-14.3-prerelease.oci
19 + for t in ${oci_images}; do
20 + aarch64="${aarch64_path}-${t}.txz"
21 + amd64="${amd64_path}-${t}.txz"
22 + oci="${OCI_DESTDIR}/freebsd-${t}-${heads}"
23 + podmanic $aarch64 $amd64 $oci >> ${logdir}/${_build}.log 2>&1
24 + sha256 ${oci}.oci > ${oci}.oci.sha256 >> ${logdir}/${_build}.log 2>&1
25 + done
26 + return 0
27 + }
28 +
29 + main() {
30 + releasesrc="main"
31 + export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
32 +
33 + while getopts "bc:du" opt; do
34 + case ${opt} in
35 + b)
36 + DOBUILD=YES
37 + ;;
38 + c)
39 + CONF=${OPTARG}
40 + [ -e ${CONF} ] && . $(realpath ${CONF})
41 + ;;
42 + d)
43 + debug=1
44 + ;;
45 + u)
46 + DOUPLOAD=YES
47 + ;;
48 + \?)
49 + usage
50 + ;;
51 + esac
52 + done
53 + shift $(($OPTIND - 1))
54 + [ -z ${CONF} ] && usage
55 + if [ "$DOBUILD" ]; then
56 + check_zfs_mounts
57 + use_zfs=1
58 + check_use_zfs
59 + runall prebuild_setup
60 + runall truncate_logs
61 + runall zfs_mount_src
62 + runall build_chroots
63 + runall install_chroots
64 + runall zfs_clone_chroots
65 + zfs_finish_bootstrap
66 + runall ${parallel}build_release
67 + wait
68 + fi
69 + if [ "$OCI_MERGE" ]; then
70 + merge_oci_images
71 + # TODO bug_colin_to_sign_them
72 + fi
73 + if [ "$DOUPLOAD" ]; then
74 + runall upload_mount_devfs
75 + runall upload_ec2_ami
76 + runall upload_gce_image
77 + runall upload_vagrant_image
78 + runall upload_azure_image
79 + runall upload_oracle_image
80 + wait
81 + runall upload_umount_devfs
82 + fi
83 + send_completed_email
84 + }
85 +
86 + main "$@"
Newer Older