-
Notifications
You must be signed in to change notification settings - Fork 193
Expand file tree
/
Copy pathdeploy.rs
More file actions
1400 lines (1287 loc) · 51.7 KB
/
deploy.rs
File metadata and controls
1400 lines (1287 loc) · 51.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! # Write deployments merging image with configmap
//!
//! Create a merged filesystem tree with the image and mounted configmaps.
use std::collections::HashSet;
use std::io::{BufRead, Write};
use std::process::Command;
use anyhow::{Context, Result, anyhow};
use bootc_kernel_cmdline::utf8::CmdlineOwned;
use cap_std::fs::{Dir, MetadataExt};
use cap_std_ext::cap_std;
use cap_std_ext::dirext::CapStdExtDirExt;
use fn_error_context::context;
use ostree::{gio, glib};
use ostree_container::OstreeImageReference;
use ostree_ext::container as ostree_container;
use ostree_ext::container::store::{ImageImporter, ImportProgress, PrepareResult, PreparedImport};
use ostree_ext::oci_spec::image::{Descriptor, Digest};
use ostree_ext::ostree::Deployment;
use ostree_ext::ostree::{self, Sysroot};
use ostree_ext::sysroot::SysrootLock;
use ostree_ext::tokio_util::spawn_blocking_cancellable_flatten;
use std::os::fd::AsFd;
use crate::progress_jsonl::{Event, ProgressWriter, SubTaskBytes, SubTaskStep};
use crate::spec::ImageReference;
use crate::spec::{BootOrder, HostSpec};
use crate::status::labels_of_config;
use crate::store::Storage;
use crate::utils::async_task_with_spinner;
// TODO use https://github.com/ostreedev/ostree-rs-ext/pull/493/commits/afc1837ff383681b947de30c0cefc70080a4f87a
const BASE_IMAGE_PREFIX: &str = "ostree/container/baseimage/bootc";
/// Create an ImageProxyConfig with bootc's user agent prefix set.
///
/// This allows registries to distinguish "image pulls for bootc client runs"
/// from other skopeo/containers-image users.
pub(crate) fn new_proxy_config() -> ostree_ext::containers_image_proxy::ImageProxyConfig {
ostree_ext::containers_image_proxy::ImageProxyConfig {
user_agent_prefix: Some(format!("bootc/{}", env!("CARGO_PKG_VERSION"))),
..Default::default()
}
}
/// Set on an ostree commit if this is a derived commit
const BOOTC_DERIVED_KEY: &str = "bootc.derived";
/// Variant of HostSpec but required to be filled out
pub(crate) struct RequiredHostSpec<'a> {
pub(crate) image: &'a ImageReference,
}
/// State of a locally fetched image
pub(crate) struct ImageState {
pub(crate) manifest_digest: Digest,
pub(crate) version: Option<String>,
pub(crate) ostree_commit: String,
}
impl<'a> RequiredHostSpec<'a> {
/// Given a (borrowed) host specification, "unwrap" its internal
/// options, giving a spec that is required to have a base container image.
pub(crate) fn from_spec(spec: &'a HostSpec) -> Result<Self> {
let image = spec
.image
.as_ref()
.ok_or_else(|| anyhow::anyhow!("Missing image in specification"))?;
Ok(Self { image })
}
}
impl From<ostree_container::store::LayeredImageState> for ImageState {
fn from(value: ostree_container::store::LayeredImageState) -> Self {
let version = value.version().map(|v| v.to_owned());
let ostree_commit = value.get_commit().to_owned();
Self {
manifest_digest: value.manifest_digest,
version,
ostree_commit,
}
}
}
impl ImageState {
/// Fetch the manifest corresponding to this image. May not be available in all backends.
pub(crate) fn get_manifest(
&self,
repo: &ostree::Repo,
) -> Result<Option<ostree_ext::oci_spec::image::ImageManifest>> {
ostree_container::store::query_image_commit(repo, &self.ostree_commit)
.map(|v| Some(v.manifest))
}
}
/// Wrapper for pulling a container image, wiring up status output.
pub(crate) async fn new_importer(
repo: &ostree::Repo,
imgref: &ostree_container::OstreeImageReference,
) -> Result<ostree_container::store::ImageImporter> {
let config = new_proxy_config();
let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?;
imp.require_bootable();
Ok(imp)
}
/// Wrapper for pulling a container image with a custom proxy config (e.g. for unified storage).
pub(crate) async fn new_importer_with_config(
repo: &ostree::Repo,
imgref: &ostree_container::OstreeImageReference,
config: ostree_ext::containers_image_proxy::ImageProxyConfig,
) -> Result<ostree_container::store::ImageImporter> {
let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?;
imp.require_bootable();
Ok(imp)
}
pub(crate) fn check_bootc_label(config: &ostree_ext::oci_spec::image::ImageConfiguration) {
if let Some(label) =
labels_of_config(config).and_then(|labels| labels.get(crate::metadata::BOOTC_COMPAT_LABEL))
{
match label.as_str() {
crate::metadata::COMPAT_LABEL_V1 => {}
o => crate::journal::journal_print(
libsystemd::logging::Priority::Warning,
&format!(
"notice: Unknown {} value {}",
crate::metadata::BOOTC_COMPAT_LABEL,
o
),
),
}
} else {
crate::journal::journal_print(
libsystemd::logging::Priority::Warning,
&format!(
"notice: Image is missing label: {}",
crate::metadata::BOOTC_COMPAT_LABEL
),
)
}
}
fn descriptor_of_progress(p: &ImportProgress) -> &Descriptor {
match p {
ImportProgress::OstreeChunkStarted(l) => l,
ImportProgress::OstreeChunkCompleted(l) => l,
ImportProgress::DerivedLayerStarted(l) => l,
ImportProgress::DerivedLayerCompleted(l) => l,
}
}
fn prefix_of_progress(p: &ImportProgress) -> &'static str {
match p {
ImportProgress::OstreeChunkStarted(_) | ImportProgress::OstreeChunkCompleted(_) => {
"ostree chunk"
}
ImportProgress::DerivedLayerStarted(_) | ImportProgress::DerivedLayerCompleted(_) => {
"layer"
}
}
}
/// Configuration for layer progress printing
struct LayerProgressConfig {
layers: tokio::sync::mpsc::Receiver<ostree_container::store::ImportProgress>,
layer_bytes: tokio::sync::watch::Receiver<Option<ostree_container::store::LayerProgress>>,
digest: Box<str>,
n_layers_to_fetch: usize,
layers_total: usize,
bytes_to_download: u64,
bytes_total: u64,
prog: ProgressWriter,
quiet: bool,
}
/// Write container fetch progress to standard output.
async fn handle_layer_progress_print(mut config: LayerProgressConfig) -> ProgressWriter {
let start = std::time::Instant::now();
let mut total_read = 0u64;
let bar = indicatif::MultiProgress::new();
if config.quiet {
bar.set_draw_target(indicatif::ProgressDrawTarget::hidden());
}
let layers_bar = bar.add(indicatif::ProgressBar::new(
config.n_layers_to_fetch.try_into().unwrap(),
));
let byte_bar = bar.add(indicatif::ProgressBar::new(0));
// let byte_bar = indicatif::ProgressBar::new(0);
// byte_bar.set_draw_target(indicatif::ProgressDrawTarget::hidden());
layers_bar.set_style(
indicatif::ProgressStyle::default_bar()
.template("{prefix} {bar} {pos}/{len} {wide_msg}")
.unwrap(),
);
let taskname = "Fetching layers";
layers_bar.set_prefix(taskname);
layers_bar.set_message("");
byte_bar.set_prefix("Fetching");
byte_bar.set_style(
indicatif::ProgressStyle::default_bar()
.template(
" └ {prefix} {bar} {binary_bytes}/{binary_total_bytes} ({binary_bytes_per_sec}) {wide_msg}",
)
.unwrap()
);
let mut subtasks = vec![];
let mut subtask: SubTaskBytes = Default::default();
loop {
tokio::select! {
// Always handle layer changes first.
biased;
layer = config.layers.recv() => {
if let Some(l) = layer {
let layer = descriptor_of_progress(&l);
let layer_type = prefix_of_progress(&l);
let short_digest = &layer.digest().digest()[0..21];
let layer_size = layer.size();
if l.is_starting() {
// Reset the progress bar
byte_bar.reset_elapsed();
byte_bar.reset_eta();
byte_bar.set_length(layer_size);
byte_bar.set_message(format!("{layer_type} {short_digest}"));
subtask = SubTaskBytes {
subtask: layer_type.into(),
description: format!("{layer_type}: {short_digest}").clone().into(),
id: short_digest.to_string().clone().into(),
bytes_cached: 0,
bytes: 0,
bytes_total: layer_size,
};
} else {
byte_bar.set_position(layer_size);
layers_bar.inc(1);
total_read = total_read.saturating_add(layer_size);
// Emit an event where bytes == total to signal completion.
subtask.bytes = layer_size;
subtasks.push(subtask.clone());
config.prog.send(Event::ProgressBytes {
task: "pulling".into(),
description: format!("Pulling Image: {}", config.digest).into(),
id: (*config.digest).into(),
bytes_cached: config.bytes_total - config.bytes_to_download,
bytes: total_read,
bytes_total: config.bytes_to_download,
steps_cached: (config.layers_total - config.n_layers_to_fetch) as u64,
steps: layers_bar.position(),
steps_total: config.n_layers_to_fetch as u64,
subtasks: subtasks.clone(),
}).await;
}
} else {
// If the receiver is disconnected, then we're done
break
};
},
r = config.layer_bytes.changed() => {
if r.is_err() {
// If the receiver is disconnected, then we're done
break
}
let bytes = {
let bytes = config.layer_bytes.borrow_and_update();
bytes.as_ref().cloned()
};
if let Some(bytes) = bytes {
byte_bar.set_position(bytes.fetched);
subtask.bytes = byte_bar.position();
config.prog.send_lossy(Event::ProgressBytes {
task: "pulling".into(),
description: format!("Pulling Image: {}", config.digest).into(),
id: (*config.digest).into(),
bytes_cached: config.bytes_total - config.bytes_to_download,
bytes: total_read + byte_bar.position(),
bytes_total: config.bytes_to_download,
steps_cached: (config.layers_total - config.n_layers_to_fetch) as u64,
steps: layers_bar.position(),
steps_total: config.n_layers_to_fetch as u64,
subtasks: subtasks.clone().into_iter().chain([subtask.clone()]).collect(),
}).await;
}
}
}
}
byte_bar.finish_and_clear();
layers_bar.finish_and_clear();
if let Err(e) = bar.clear() {
tracing::warn!("clearing bar: {e}");
}
let end = std::time::Instant::now();
let elapsed = end.duration_since(start);
let persec = total_read as f64 / elapsed.as_secs_f64();
let persec = indicatif::HumanBytes(persec as u64);
if let Err(e) = bar.println(&format!(
"Fetched layers: {} in {} ({}/s)",
indicatif::HumanBytes(total_read),
indicatif::HumanDuration(elapsed),
persec,
)) {
tracing::warn!("writing to stdout: {e}");
}
// Since the progress notifier closed, we know import has started
// use as a heuristic to begin import progress
// Cannot be lossy or it is dropped
config
.prog
.send(Event::ProgressSteps {
task: "importing".into(),
description: "Importing Image".into(),
id: (*config.digest).into(),
steps_cached: 0,
steps: 0,
steps_total: 1,
subtasks: [SubTaskStep {
subtask: "importing".into(),
description: "Importing Image".into(),
id: "importing".into(),
completed: false,
}]
.into(),
})
.await;
// Return the writer
config.prog
}
/// Gather all bound images in all deployments, then prune the image store,
/// using the gathered images as the roots (that will not be GC'd).
pub(crate) async fn prune_container_store(sysroot: &Storage) -> Result<()> {
let ostree = sysroot.get_ostree()?;
let deployments = ostree.deployments();
let mut all_bound_images = Vec::new();
for deployment in deployments {
let bound = crate::boundimage::query_bound_images_for_deployment(ostree, &deployment)?;
all_bound_images.extend(bound.into_iter());
// Also include the host image itself
// Note: Use just the image name (not the full transport:image format) because
// podman's image names don't include the transport prefix.
if let Some(host_image) = crate::status::boot_entry_from_deployment(ostree, &deployment)?
.image
.map(|i| i.image)
{
all_bound_images.push(crate::boundimage::BoundImage {
image: host_image.image.clone(),
auth_file: None,
});
}
}
// Convert to a hashset of just the image names
let image_names = HashSet::from_iter(all_bound_images.iter().map(|img| img.image.as_str()));
let pruned = sysroot
.get_ensure_imgstore()?
.prune_except_roots(&image_names)
.await?;
tracing::debug!("Pruned images: {}", pruned.len());
Ok(())
}
/// Verify there is sufficient disk space to pull an image.
///
/// This checks the available space on the filesystem containing the OSTree repository
/// against the number of bytes that need to be fetched for the image.
pub(crate) fn check_disk_space(
repo_fd: impl AsFd,
image_meta: &PreparedImportMeta,
imgref: &ImageReference,
) -> Result<()> {
let stat = rustix::fs::fstatvfs(repo_fd)?;
let bytes_avail: u64 = stat.f_bsize * stat.f_bavail;
tracing::trace!("bytes_avail: {bytes_avail}");
if image_meta.bytes_to_fetch > bytes_avail {
anyhow::bail!(
"Insufficient free space for {image} (available: {bytes_avail} required: {bytes_to_fetch})",
bytes_avail = ostree_ext::glib::format_size(bytes_avail),
bytes_to_fetch = ostree_ext::glib::format_size(image_meta.bytes_to_fetch),
image = imgref.image,
);
}
Ok(())
}
pub(crate) struct PreparedImportMeta {
pub imp: ImageImporter,
pub prep: Box<PreparedImport>,
pub digest: Digest,
pub n_layers_to_fetch: usize,
pub layers_total: usize,
pub bytes_to_fetch: u64,
pub bytes_total: u64,
}
pub(crate) enum PreparedPullResult {
Ready(Box<PreparedImportMeta>),
AlreadyPresent(Box<ImageState>),
}
pub(crate) async fn prepare_for_pull(
repo: &ostree::Repo,
imgref: &ImageReference,
target_imgref: Option<&OstreeImageReference>,
) -> Result<PreparedPullResult> {
let imgref_canonicalized = imgref.clone().canonicalize()?;
tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}");
let ostree_imgref = &OstreeImageReference::from(imgref_canonicalized);
let mut imp = new_importer(repo, ostree_imgref).await?;
if let Some(target) = target_imgref {
imp.set_target(target);
}
let prep = match imp.prepare().await? {
PrepareResult::AlreadyPresent(c) => {
println!("No changes in {imgref:#} => {}", c.manifest_digest);
return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into())));
}
PrepareResult::Ready(p) => p,
};
check_bootc_label(&prep.config);
if let Some(warning) = prep.deprecated_warning() {
ostree_ext::cli::print_deprecated_warning(warning).await;
}
ostree_ext::cli::print_layer_status(&prep);
let layers_to_fetch = prep.layers_to_fetch().collect::<Result<Vec<_>>>()?;
let prepared_image = PreparedImportMeta {
imp,
n_layers_to_fetch: layers_to_fetch.len(),
layers_total: prep.all_layers().count(),
bytes_to_fetch: layers_to_fetch.iter().map(|(l, _)| l.layer.size()).sum(),
bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(),
digest: prep.manifest_digest.clone(),
prep,
};
Ok(PreparedPullResult::Ready(Box::new(prepared_image)))
}
/// Check whether the image exists in bootc's unified container storage.
///
/// This is used for auto-detection: if the image already exists in bootc storage
/// (e.g., from a previous `bootc image set-unified` or LBI pull), we can use
/// the unified storage path for faster imports.
///
/// Returns true if the image exists in bootc storage.
pub(crate) async fn image_exists_in_unified_storage(
store: &Storage,
imgref: &ImageReference,
) -> Result<bool> {
let imgstore = store.get_ensure_imgstore()?;
let image_ref_str = imgref.to_transport_image()?;
imgstore.exists(&image_ref_str).await
}
/// Unified approach: Use bootc's CStorage to pull the image, then prepare from containers-storage.
/// This reuses the same infrastructure as LBIs.
pub(crate) async fn prepare_for_pull_unified(
repo: &ostree::Repo,
imgref: &ImageReference,
target_imgref: Option<&OstreeImageReference>,
store: &Storage,
) -> Result<PreparedPullResult> {
// Get or initialize the bootc container storage (same as used for LBIs)
let imgstore = store.get_ensure_imgstore()?;
let image_ref_str = imgref.to_transport_image()?;
// Always pull to ensure we have the latest image, whether from a remote
// registry or a locally rebuilt image
tracing::info!(
"Unified pull: pulling from transport '{}' to bootc storage",
&imgref.transport
);
// Pull the image to bootc storage using the same method as LBIs
// Show a spinner since podman pull can take a while and doesn't output progress
let pull_msg = format!("Pulling {} to bootc storage", &image_ref_str);
async_task_with_spinner(&pull_msg, async move {
imgstore
.pull(&image_ref_str, crate::podstorage::PullMode::Always)
.await
})
.await?;
// Now create a containers-storage reference to read from bootc storage
tracing::info!("Unified pull: now importing from containers-storage transport");
let containers_storage_imgref = ImageReference {
transport: "containers-storage".to_string(),
image: imgref.image.clone(),
signature: imgref.signature.clone(),
};
let ostree_imgref = OstreeImageReference::from(containers_storage_imgref);
// Configure the importer to use bootc storage as an additional image store
let mut config = new_proxy_config();
let mut cmd = Command::new("skopeo");
// Use the physical path to bootc storage from the Storage struct
let storage_path = format!(
"{}/{}",
store.physical_root_path,
crate::podstorage::CStorage::subpath()
);
crate::podstorage::set_additional_image_store(&mut cmd, &storage_path);
config.skopeo_cmd = Some(cmd);
// Use the preparation flow with the custom config
let mut imp = new_importer_with_config(repo, &ostree_imgref, config).await?;
if let Some(target) = target_imgref {
imp.set_target(target);
}
let prep = match imp.prepare().await? {
PrepareResult::AlreadyPresent(c) => {
println!("No changes in {imgref:#} => {}", c.manifest_digest);
return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into())));
}
PrepareResult::Ready(p) => p,
};
check_bootc_label(&prep.config);
if let Some(warning) = prep.deprecated_warning() {
ostree_ext::cli::print_deprecated_warning(warning).await;
}
ostree_ext::cli::print_layer_status(&prep);
let layers_to_fetch = prep.layers_to_fetch().collect::<Result<Vec<_>>>()?;
// Log that we're importing a new image from containers-storage
const PULLING_NEW_IMAGE_ID: &str = "6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0";
tracing::info!(
message_id = PULLING_NEW_IMAGE_ID,
bootc.image.reference = &imgref.image,
bootc.image.transport = "containers-storage",
bootc.original_transport = &imgref.transport,
bootc.status = "importing_from_storage",
"Importing image from bootc storage: {}",
ostree_imgref
);
let prepared_image = PreparedImportMeta {
imp,
n_layers_to_fetch: layers_to_fetch.len(),
layers_total: prep.all_layers().count(),
bytes_to_fetch: layers_to_fetch.iter().map(|(l, _)| l.layer.size()).sum(),
bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(),
digest: prep.manifest_digest.clone(),
prep,
};
Ok(PreparedPullResult::Ready(Box::new(prepared_image)))
}
/// Unified pull: Use podman to pull to containers-storage, then read from there
pub(crate) async fn pull_unified(
repo: &ostree::Repo,
imgref: &ImageReference,
target_imgref: Option<&OstreeImageReference>,
quiet: bool,
prog: ProgressWriter,
store: &Storage,
) -> Result<Box<ImageState>> {
match prepare_for_pull_unified(repo, imgref, target_imgref, store).await? {
PreparedPullResult::AlreadyPresent(existing) => {
// Log that the image was already present (Debug level since it's not actionable)
const IMAGE_ALREADY_PRESENT_ID: &str = "5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9";
tracing::debug!(
message_id = IMAGE_ALREADY_PRESENT_ID,
bootc.image.reference = &imgref.image,
bootc.image.transport = &imgref.transport,
bootc.status = "already_present",
"Image already present: {}",
imgref
);
Ok(existing)
}
PreparedPullResult::Ready(prepared_image_meta) => {
// To avoid duplicate success logs, pass a containers-storage imgref to the importer
let cs_imgref = ImageReference {
transport: "containers-storage".to_string(),
image: imgref.image.clone(),
signature: imgref.signature.clone(),
};
pull_from_prepared(&cs_imgref, quiet, prog, *prepared_image_meta).await
}
}
}
#[context("Pulling")]
pub(crate) async fn pull_from_prepared(
imgref: &ImageReference,
quiet: bool,
prog: ProgressWriter,
mut prepared_image: PreparedImportMeta,
) -> Result<Box<ImageState>> {
let layer_progress = prepared_image.imp.request_progress();
let layer_byte_progress = prepared_image.imp.request_layer_progress();
let digest = prepared_image.digest.clone();
let digest_imp = prepared_image.digest.clone();
let printer = tokio::task::spawn(async move {
handle_layer_progress_print(LayerProgressConfig {
layers: layer_progress,
layer_bytes: layer_byte_progress,
digest: digest.as_ref().into(),
n_layers_to_fetch: prepared_image.n_layers_to_fetch,
layers_total: prepared_image.layers_total,
bytes_to_download: prepared_image.bytes_to_fetch,
bytes_total: prepared_image.bytes_total,
prog,
quiet,
})
.await
});
let import = prepared_image.imp.import(prepared_image.prep).await;
let prog = printer.await?;
// Both the progress and the import are done, so import is done as well
prog.send(Event::ProgressSteps {
task: "importing".into(),
description: "Importing Image".into(),
id: digest_imp.clone().as_ref().into(),
steps_cached: 0,
steps: 1,
steps_total: 1,
subtasks: [SubTaskStep {
subtask: "importing".into(),
description: "Importing Image".into(),
id: "importing".into(),
completed: true,
}]
.into(),
})
.await;
let import = import?;
let imgref_canonicalized = imgref.clone().canonicalize()?;
tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}");
// Log successful import completion (skip if using unified storage to avoid double logging)
let is_unified_path = imgref.transport == "containers-storage";
if !is_unified_path {
const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8";
tracing::info!(
message_id = IMPORT_COMPLETE_JOURNAL_ID,
bootc.image.reference = &imgref.image,
bootc.image.transport = &imgref.transport,
bootc.manifest_digest = import.manifest_digest.as_ref(),
bootc.ostree_commit = &import.merge_commit,
"Successfully imported image: {}",
imgref
);
}
if let Some(msg) =
ostree_container::store::image_filtered_content_warning(&import.filtered_files)
.context("Image content warning")?
{
tracing::info!("{}", msg);
}
Ok(Box::new((*import).into()))
}
/// Wrapper for pulling a container image, wiring up status output.
pub(crate) async fn pull(
repo: &ostree::Repo,
imgref: &ImageReference,
target_imgref: Option<&OstreeImageReference>,
quiet: bool,
prog: ProgressWriter,
) -> Result<Box<ImageState>> {
match prepare_for_pull(repo, imgref, target_imgref).await? {
PreparedPullResult::AlreadyPresent(existing) => {
// Log that the image was already present (Debug level since it's not actionable)
const IMAGE_ALREADY_PRESENT_ID: &str = "5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9";
tracing::debug!(
message_id = IMAGE_ALREADY_PRESENT_ID,
bootc.image.reference = &imgref.image,
bootc.image.transport = &imgref.transport,
bootc.status = "already_present",
"Image already present: {}",
imgref
);
Ok(existing)
}
PreparedPullResult::Ready(prepared_image_meta) => {
// Check disk space before attempting to pull
check_disk_space(repo.dfd_borrow(), &prepared_image_meta, imgref)?;
// Log that we're pulling a new image
const PULLING_NEW_IMAGE_ID: &str = "6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0";
tracing::info!(
message_id = PULLING_NEW_IMAGE_ID,
bootc.image.reference = &imgref.image,
bootc.image.transport = &imgref.transport,
bootc.status = "pulling_new",
"Pulling new image: {}",
imgref
);
Ok(pull_from_prepared(imgref, quiet, prog, *prepared_image_meta).await?)
}
}
}
pub(crate) async fn wipe_ostree(sysroot: Sysroot) -> Result<()> {
tokio::task::spawn_blocking(move || {
sysroot
.write_deployments(&[], gio::Cancellable::NONE)
.context("removing deployments")
})
.await??;
Ok(())
}
pub(crate) async fn cleanup(sysroot: &Storage) -> Result<()> {
// Log the cleanup operation to systemd journal
const CLEANUP_JOURNAL_ID: &str = "2f1a0b9c8d7e6f5a4b3c2d1e0f9a8b7c6";
tracing::info!(
message_id = CLEANUP_JOURNAL_ID,
"Starting cleanup of old images and deployments"
);
let bound_prune = prune_container_store(sysroot);
// We create clones (just atomic reference bumps) here to move to the thread.
let ostree = sysroot.get_ostree_cloned()?;
let repo = ostree.repo();
let repo_prune =
ostree_ext::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| {
let locked_sysroot = &SysrootLock::from_assumed_locked(&ostree);
let cancellable = Some(cancellable);
let repo = &repo;
let txn = repo.auto_transaction(cancellable)?;
let repo = txn.repo();
// Regenerate our base references. First, we delete the ones that exist
for ref_entry in repo
.list_refs_ext(
Some(BASE_IMAGE_PREFIX),
ostree::RepoListRefsExtFlags::NONE,
cancellable,
)
.context("Listing refs")?
.keys()
{
repo.transaction_set_refspec(ref_entry, None);
}
// Then, for each deployment which is derived (e.g. has configmaps) we synthesize
// a base ref to ensure that it's not GC'd.
for (i, deployment) in ostree.deployments().into_iter().enumerate() {
let commit = deployment.csum();
if let Some(base) = get_base_commit(repo, &commit)? {
repo.transaction_set_refspec(&format!("{BASE_IMAGE_PREFIX}/{i}"), Some(&base));
}
}
let pruned =
ostree_container::deploy::prune(locked_sysroot).context("Pruning images")?;
if !pruned.is_empty() {
let size = glib::format_size(pruned.objsize);
println!(
"Pruned images: {} (layers: {}, objsize: {})",
pruned.n_images, pruned.n_layers, size
);
} else {
tracing::debug!("Nothing to prune");
}
Ok(())
});
// We run these in parallel mostly because we can.
tokio::try_join!(repo_prune, bound_prune)?;
Ok(())
}
/// If commit is a bootc-derived commit (e.g. has configmaps), return its base.
#[context("Finding base commit")]
pub(crate) fn get_base_commit(repo: &ostree::Repo, commit: &str) -> Result<Option<String>> {
let commitv = repo.load_commit(commit)?.0;
let commitmeta = commitv.child_value(0);
let commitmeta = &glib::VariantDict::new(Some(&commitmeta));
let r = commitmeta.lookup::<String>(BOOTC_DERIVED_KEY)?;
Ok(r)
}
#[context("Writing deployment")]
async fn deploy(
sysroot: &Storage,
from: MergeState,
image: &ImageState,
origin: &glib::KeyFile,
lock_finalization: bool,
) -> Result<Deployment> {
// Compute the kernel argument overrides. In practice today this API is always expecting
// a merge deployment. The kargs code also always looks at the booted root (which
// is a distinct minor issue, but not super important as right now the install path
// doesn't use this API).
let (stateroot, override_kargs) = match &from {
MergeState::MergeDeployment(deployment) => {
let kargs = crate::bootc_kargs::get_kargs(sysroot, &deployment, image)?;
(deployment.stateroot().into(), Some(kargs))
}
MergeState::Reset { stateroot, kargs } => (stateroot.clone(), Some(kargs.clone())),
};
// Clone all the things to move to worker thread
let ostree = sysroot.get_ostree_cloned()?;
// ostree::Deployment is incorrectly !Send 😢 so convert it to an integer
let merge_deployment = from.as_merge_deployment();
let merge_deployment = merge_deployment.map(|d| d.index() as usize);
let ostree_commit = image.ostree_commit.to_string();
// GKeyFile also isn't Send! So we serialize that as a string...
let origin_data = origin.to_data();
let r = async_task_with_spinner(
"Deploying",
spawn_blocking_cancellable_flatten(move |cancellable| -> Result<_> {
let ostree = ostree;
let stateroot = Some(stateroot);
let mut opts = ostree::SysrootDeployTreeOpts::default();
// Set finalization lock if requested
opts.locked = lock_finalization;
// Because the C API expects a Vec<&str>, convert the Cmdline to string slices.
// The references borrow from the Cmdline, which outlives this usage.
let override_kargs_refs = override_kargs
.as_ref()
.map(|kargs| kargs.iter_str().collect::<Vec<_>>());
if let Some(kargs) = override_kargs_refs.as_ref() {
opts.override_kernel_argv = Some(kargs);
}
let deployments = ostree.deployments();
let merge_deployment = merge_deployment.map(|m| &deployments[m]);
let origin = glib::KeyFile::new();
origin.load_from_data(&origin_data, glib::KeyFileFlags::NONE)?;
let d = ostree.stage_tree_with_options(
stateroot.as_deref(),
&ostree_commit,
Some(&origin),
merge_deployment,
&opts,
Some(cancellable),
)?;
Ok(d.index())
}),
)
.await?;
// SAFETY: We must have a staged deployment
let ostree = sysroot.get_ostree()?;
let staged = ostree.staged_deployment().unwrap();
assert_eq!(staged.index(), r);
Ok(staged)
}
#[context("Generating origin")]
fn origin_from_imageref(imgref: &ImageReference) -> Result<glib::KeyFile> {
let origin = glib::KeyFile::new();
let imgref = OstreeImageReference::from(imgref.clone());
origin.set_string(
"origin",
ostree_container::deploy::ORIGIN_CONTAINER,
imgref.to_string().as_str(),
);
Ok(origin)
}
/// The source of data for staging a new deployment
#[derive(Debug)]
pub(crate) enum MergeState {
/// Use the provided merge deployment
MergeDeployment(Deployment),
/// Don't use a merge deployment, but only this
/// provided initial state.
Reset {
stateroot: String,
kargs: CmdlineOwned,
},
}
impl MergeState {
/// Initialize using the default merge deployment for the given stateroot.
pub(crate) fn from_stateroot(sysroot: &Storage, stateroot: &str) -> Result<Self> {
let ostree = sysroot.get_ostree()?;
let merge_deployment = ostree.merge_deployment(Some(stateroot)).ok_or_else(|| {
anyhow::anyhow!("No merge deployment found for stateroot {stateroot}")
})?;
Ok(Self::MergeDeployment(merge_deployment))
}
/// Cast this to a merge deployment case.
pub(crate) fn as_merge_deployment(&self) -> Option<&Deployment> {
match self {
Self::MergeDeployment(d) => Some(d),
Self::Reset { .. } => None,
}
}
}
/// Stage (queue deployment of) a fetched container image.
#[context("Staging")]
pub(crate) async fn stage(
sysroot: &Storage,
from: MergeState,
image: &ImageState,
spec: &RequiredHostSpec<'_>,
prog: ProgressWriter,
lock_finalization: bool,
) -> Result<()> {
// Log the staging operation to systemd journal with comprehensive upgrade information
const STAGE_JOURNAL_ID: &str = "8f7a2b1c3d4e5f6a7b8c9d0e1f2a3b4c";
tracing::info!(
message_id = STAGE_JOURNAL_ID,
bootc.image.reference = &spec.image.image,
bootc.image.transport = &spec.image.transport,
bootc.manifest_digest = image.manifest_digest.as_ref(),
"Staging image for deployment: {} (digest: {})",
spec.image,
image.manifest_digest
);
let mut subtask = SubTaskStep {
subtask: "merging".into(),
description: "Merging Image".into(),
id: "fetching".into(),
completed: false,
};
let mut subtasks = vec![];
prog.send(Event::ProgressSteps {
task: "staging".into(),
description: "Deploying Image".into(),
id: image.manifest_digest.clone().as_ref().into(),
steps_cached: 0,
steps: 0,
steps_total: 3,
subtasks: subtasks
.clone()
.into_iter()
.chain([subtask.clone()])
.collect(),
})
.await;
subtask.completed = true;
subtasks.push(subtask.clone());
subtask.subtask = "deploying".into();
subtask.id = "deploying".into();
subtask.description = "Deploying Image".into();
subtask.completed = false;
prog.send(Event::ProgressSteps {
task: "staging".into(),
description: "Deploying Image".into(),
id: image.manifest_digest.clone().as_ref().into(),
steps_cached: 0,
steps: 1,
steps_total: 3,
subtasks: subtasks
.clone()
.into_iter()
.chain([subtask.clone()])
.collect(),
})
.await;
let origin = origin_from_imageref(spec.image)?;
let deployment =
crate::deploy::deploy(sysroot, from, image, &origin, lock_finalization).await?;
subtask.completed = true;
subtasks.push(subtask.clone());
subtask.subtask = "bound_images".into();
subtask.id = "bound_images".into();
subtask.description = "Pulling Bound Images".into();
subtask.completed = false;
prog.send(Event::ProgressSteps {
task: "staging".into(),
description: "Deploying Image".into(),
id: image.manifest_digest.clone().as_ref().into(),
steps_cached: 0,
steps: 1,
steps_total: 3,
subtasks: subtasks
.clone()
.into_iter()
.chain([subtask.clone()])
.collect(),
})
.await;
crate::boundimage::pull_bound_images(sysroot, &deployment).await?;
subtask.completed = true;
subtasks.push(subtask.clone());
subtask.subtask = "cleanup".into();
subtask.id = "cleanup".into();
subtask.description = "Removing old images".into();
subtask.completed = false;
prog.send(Event::ProgressSteps {
task: "staging".into(),