diff --git a/src/api2/tape/drive.rs b/src/api2/tape/drive.rs
index 633a236d..53b14bfd 100644
--- a/src/api2/tape/drive.rs
+++ b/src/api2/tape/drive.rs
@@ -677,9 +677,6 @@ pub async fn read_label(drive: String, inventorize: Option<bool>) -> Result<Medi
         let label = if let Some(ref set) = media_id.media_set_label {
             let key = &set.encryption_key_fingerprint;
 
-            if let Err(err) = drive.set_encryption(key.clone().map(|fp| (fp, set.uuid.clone()))) {
-                eprintln!("unable to load encryption key: {}", err); // best-effort only
-            }
             MediaIdFlat {
                 ctime: media_id.label.ctime,
                 encryption_key_fingerprint: key.as_ref().map(|fp| fp.signature()),
@@ -1358,12 +1355,6 @@ pub fn catalog_media(
                         inventory.store(media_id.clone(), false)?;
                         return Ok(());
                     }
-                    let encrypt_fingerprint = set
-                        .encryption_key_fingerprint
-                        .clone()
-                        .map(|fp| (fp, set.uuid.clone()));
-
-                    drive.set_encryption(encrypt_fingerprint)?;
 
                     let _pool_lock = lock_media_pool(TAPE_STATUS_DIR, &set.pool)?;
                     let media_set_lock = lock_media_set(TAPE_STATUS_DIR, &set.uuid, None)?;
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index 7b6c8978..47e0586b 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -1029,12 +1029,6 @@ fn restore_snapshots_to_tmpdir(
                     media_set_uuid
                 );
             }
-            let encrypt_fingerprint = set.encryption_key_fingerprint.clone().map(|fp| {
-                task_log!(worker, "Encryption key fingerprint: {}", fp);
-                (fp, set.uuid.clone())
-            });
-
-            drive.set_encryption(encrypt_fingerprint)?;
         }
     }
 
@@ -1279,12 +1273,6 @@ pub fn request_and_restore_media(
                     media_set_uuid
                 );
             }
-            let encrypt_fingerprint = set
-                .encryption_key_fingerprint
-                .clone()
-                .map(|fp| (fp, set.uuid.clone()));
-
-            drive.set_encryption(encrypt_fingerprint)?;
         }
     }
 
diff --git a/src/tape/drive/lto/mod.rs b/src/tape/drive/lto/mod.rs
index 98087b25..c30b2c94 100644
--- a/src/tape/drive/lto/mod.rs
+++ b/src/tape/drive/lto/mod.rs
@@ -269,6 +269,13 @@ impl TapeDriver for LtoTapeHandle {
 
         self.sync()?; // sync data to tape
 
+        let encrypt_fingerprint = media_set_label
+            .encryption_key_fingerprint
+            .clone()
+            .map(|fp| (fp, media_set_label.uuid.clone()));
+
+        self.set_encryption(encrypt_fingerprint)?;
+
         Ok(())
     }
 
diff --git a/src/tape/drive/mod.rs b/src/tape/drive/mod.rs
index 97673282..90be5f9e 100644
--- a/src/tape/drive/mod.rs
+++ b/src/tape/drive/mod.rs
@@ -182,6 +182,15 @@ pub trait TapeDriver {
             bail!("got unexpected data after media set label");
         }
 
+        drop(reader);
+
+        let encrypt_fingerprint = media_set_label
+            .encryption_key_fingerprint
+            .clone()
+            .map(|fp| (fp, media_set_label.uuid.clone()));
+
+        self.set_encryption(encrypt_fingerprint)?;
+
         media_id.media_set_label = Some(media_set_label);
 
         Ok((Some(media_id), key_config))
@@ -203,6 +212,9 @@ pub trait TapeDriver {
     /// We use the media_set_uuid to XOR the secret key with the
     /// uuid (first 16 bytes), so that each media set uses an unique
     /// key for encryption.
+    ///
+    /// Should be called as part of write_media_set_label or read_label,
+    /// so this should not be called manually.
     fn set_encryption(
         &mut self,
         key_fingerprint: Option<(Fingerprint, Uuid)>,
diff --git a/src/tape/pool_writer/mod.rs b/src/tape/pool_writer/mod.rs
index d1add203..6fd4f72a 100644
--- a/src/tape/pool_writer/mod.rs
+++ b/src/tape/pool_writer/mod.rs
@@ -270,15 +270,6 @@ impl PoolWriter {
 
         self.catalog_set.lock().unwrap().append_catalog(catalog)?;
 
-        let media_set = media.media_set_label().unwrap();
-
-        let encrypt_fingerprint = media_set
-            .encryption_key_fingerprint
-            .clone()
-            .map(|fp| (fp, media_set.uuid.clone()));
-
-        drive.set_encryption(encrypt_fingerprint)?;
-
         self.status = Some(PoolWriterState {
             drive,
             media_uuid: media_uuid.clone(),