From 6fa355b319356cac4d15760abdcbbe14c4ddc43c Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:36:50 +0000 Subject: [PATCH 01/10] ceph-rgw: keep existing account details when storing the RGW keys AccountDetailsDao.persist() removes every detail of the account before it writes the given ones, so the first bucket of an account removed its account-scoped settings. --- .../storage/datastore/driver/CephObjectStoreDriverImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index b3ad2d9552f1..a1fd543503ff 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -236,7 +236,7 @@ public boolean createUser(long accountId, long storeId) { Map details = new HashMap<>(); details.put(CEPH_ACCESS_KEY, credentials.getAccessKey()); details.put(CEPH_SECRET_KEY, credentials.getSecretKey()); - _accountDetailsDao.persist(accountId, details); + _accountDetailsDao.update(accountId, details); return true; } catch (Exception e) { throw new CloudRuntimeException(e); From 55645739490f36f4edf7ce2f924013040df5d696 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:38:05 +0000 Subject: [PATCH 02/10] ceph-rgw: encrypt the RGW keys of an account and keep them per object store The keys were stored in plain text under one name per account, so a second Ceph object store replaced the keys of the first one. They are now stored encrypted with the ID of the store in their name; keys stored before are still read. Secret keys in the account details are no longer returned by listAccounts. --- .../driver/CephObjectStoreDriverImpl.java | 30 ++++++++++++++----- .../main/java/com/cloud/api/ApiDBUtils.java | 1 + 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index a1fd543503ff..df6886056c6f 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -37,8 +37,10 @@ import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; import com.cloud.user.Account; +import com.cloud.user.AccountDetailVO; import com.cloud.user.AccountDetailsDao; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; @@ -107,8 +109,8 @@ public Bucket createBucket(Bucket bucket, boolean objectLock) { } try { s3client.createBucket(bucketName); - String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); - String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); + String accessKey = getAccountKey(accountId, storeId, CEPH_ACCESS_KEY); + String secretKey = getAccountKey(accountId, storeId, CEPH_SECRET_KEY); ObjectStoreVO store = _storeDao.findById(storeId); BucketVO bucketVO = _bucketDao.findById(bucket.getId()); bucketVO.setAccessKey(accessKey); @@ -234,8 +236,8 @@ public boolean createUser(long accountId, long storeId) { S3Credential credentials = newUser.getS3Credentials().get(0); Map details = new HashMap<>(); - details.put(CEPH_ACCESS_KEY, credentials.getAccessKey()); - details.put(CEPH_SECRET_KEY, credentials.getSecretKey()); + details.put(CEPH_ACCESS_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getAccessKey())); + details.put(CEPH_SECRET_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getSecretKey())); _accountDetailsDao.update(accountId, details); return true; } catch (Exception e) { @@ -346,9 +348,23 @@ private String getStoreURL(long storeId) { protected AmazonS3 getS3Client(long storeId, long accountId) { String url = getStoreURL(storeId); - String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); - String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); - return this.getS3Client(url, accessKey, secretKey); + return this.getS3Client(url, getAccountKey(accountId, storeId, CEPH_ACCESS_KEY), getAccountKey(accountId, storeId, CEPH_SECRET_KEY)); + } + + /** + * Returns the RGW key of the account on the given store. Keys stored before they were kept per store + * have no store ID in their name and are not encrypted. + */ + protected String getAccountKey(long accountId, long storeId, String name) { + AccountDetailVO detail = _accountDetailsDao.findDetail(accountId, name + "-" + storeId); + if (detail != null) { + return DBEncryptionUtil.decrypt(detail.getValue()); + } + detail = _accountDetailsDao.findDetail(accountId, name); + if (detail == null) { + throw new CloudRuntimeException("No Ceph RGW " + name + " found for account " + accountId); + } + return detail.getValue(); } protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) { AmazonS3 client = AmazonS3ClientBuilder.standard() diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index af2965d4a6d4..65e84b894ed5 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1564,6 +1564,7 @@ public static Map getDomainDetails(long domainId) { public static Map getAccountDetails(long accountId) { Map details = s_accountDetailsDao.findDetails(accountId); + details.keySet().removeIf(key -> key.toLowerCase().contains("secretkey")); return details.isEmpty() ? null : details; } From 7286217890002d99de07ae83eb831f6e8bf06fbe Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:38:49 +0000 Subject: [PATCH 03/10] ceph-rgw: apply bucket encryption and object lock Both were accepted and recorded on the bucket, but never set in RGW. --- .../driver/CephObjectStoreDriverImpl.java | 18 +++++++++++++++--- .../driver/CephObjectStoreDriverImplTest.java | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index df6886056c6f..66d6ef41ab49 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -27,7 +27,13 @@ import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; import com.amazonaws.services.s3.model.BucketVersioningConfiguration; +import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; +import com.amazonaws.services.s3.model.SSEAlgorithm; +import com.amazonaws.services.s3.model.ServerSideEncryptionByDefault; +import com.amazonaws.services.s3.model.ServerSideEncryptionConfiguration; +import com.amazonaws.services.s3.model.ServerSideEncryptionRule; +import com.amazonaws.services.s3.model.SetBucketEncryptionRequest; import com.amazonaws.services.s3.model.SetBucketPolicyRequest; import com.amazonaws.services.s3.model.GetBucketPolicyRequest; import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; @@ -108,7 +114,7 @@ public Bucket createBucket(Bucket bucket, boolean objectLock) { throw new CloudRuntimeException(e); } try { - s3client.createBucket(bucketName); + s3client.createBucket(new CreateBucketRequest(bucketName).withObjectLockEnabledForBucket(objectLock)); String accessKey = getAccountKey(accountId, storeId, CEPH_ACCESS_KEY); String secretKey = getAccountKey(accountId, storeId, CEPH_SECRET_KEY); ObjectStoreVO store = _storeDao.findById(storeId); @@ -247,12 +253,18 @@ public boolean createUser(long accountId, long storeId) { @Override public boolean setBucketEncryption(BucketTO bucket, long storeId) { - return false; + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getSecretKey()); + client.setBucketEncryption(new SetBucketEncryptionRequest().withBucketName(bucket.getName()) + .withServerSideEncryptionConfiguration(new ServerSideEncryptionConfiguration().withRules( + new ServerSideEncryptionRule().withApplyServerSideEncryptionByDefault( + new ServerSideEncryptionByDefault().withSSEAlgorithm(SSEAlgorithm.AES256))))); + return true; } @Override public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { - return false; + getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getSecretKey()).deleteBucketEncryption(bucket.getName()); + return true; } @Override diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java index 33919107b5ae..52123ad67056 100644 --- a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.datastore.driver; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CreateBucketRequest; import com.cloud.agent.api.to.BucketTO; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; @@ -97,7 +98,7 @@ public void testCreateBucket() throws Exception { Bucket bucketRet = cephObjectStoreDriverImpl.createBucket(bucket, false); assertEquals(bucketRet.getName(), bucket.getName()); verify(rgwClient, times(1)).doesBucketExistV2(anyString()); - verify(rgwClient, times(1)).createBucket(anyString()); + verify(rgwClient, times(1)).createBucket(any(CreateBucketRequest.class)); } @Test From 44e645a5b621ad4a315b4d67d3835d742e8efe98 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:40:00 +0000 Subject: [PATCH 04/10] ceph-rgw: refuse to delete a bucket which is not empty The admin API removed the bucket with all its objects. The S3 API fails with BucketNotEmpty instead. --- .../storage/datastore/driver/CephObjectStoreDriverImpl.java | 4 +--- .../datastore/driver/CephObjectStoreDriverImplTest.java | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 66d6ef41ab49..f5179a0a741d 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -148,10 +148,8 @@ public List listBuckets(long storeId) { @Override public boolean deleteBucket(BucketTO bucket, long storeId) { - RgwAdmin rgwAdmin = getRgwAdminClient(storeId); - try { - rgwAdmin.removeBucket(bucket.getName()); + getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getSecretKey()).deleteBucket(bucket.getName()); } catch (Exception e) { throw new CloudRuntimeException(e); } diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java index 52123ad67056..68f98aee68b5 100644 --- a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java @@ -105,9 +105,9 @@ public void testCreateBucket() throws Exception { public void testDeleteBucket() throws Exception { String bucketName = "test-bucket"; BucketTO bucket = new BucketTO(bucketName); - doReturn(rgwAdmin).when(cephObjectStoreDriverImpl).getRgwAdminClient(anyLong()); + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(any(), any(), any()); boolean success = cephObjectStoreDriverImpl.deleteBucket(bucket, 1L); assertTrue(success); - verify(rgwAdmin, times(1)).removeBucket(anyString()); + verify(rgwClient, times(1)).deleteBucket(anyString()); } } From aa60d85db38d7e0dbc0e2ef4920d5cc868124c73 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:40:52 +0000 Subject: [PATCH 05/10] ceph-rgw: report failures of the RGW admin API instead of hiding them - a failed lookup of the RGW user was taken for a missing user - an existing RGW user without stored keys made the bucket creation fail with a NullPointerException, its keys are stored now - the error of an existing bucket was wrapped twice - adding an object store without credentials failed without the cause --- .../driver/CephObjectStoreDriverImpl.java | 46 ++++++------------- .../CephObjectStoreLifeCycleImpl.java | 8 ++-- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index f5179a0a741d..4240f689ece2 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -63,7 +63,6 @@ import javax.inject.Inject; import java.util.List; import java.util.ArrayList; -import java.util.Optional; import java.util.Map; import java.util.HashMap; @@ -102,16 +101,8 @@ public Bucket createBucket(Bucket bucket, boolean objectLock) { long accountId = bucket.getAccountId(); AmazonS3 s3client = getS3Client(storeId, accountId); - try { - if (s3client.doesBucketExistV2(bucketName)) { - throw new CloudRuntimeException("Bucket already exists with the name: " + bucketName); - } - } catch (AmazonS3Exception e) { - if (e.getStatusCode() != 404) { - throw new CloudRuntimeException(e); - } - } catch (Exception e) { - throw new CloudRuntimeException(e); + if (s3client.doesBucketExistV2(bucketName)) { + throw new CloudRuntimeException("Bucket already exists with the name: " + bucketName); } try { s3client.createBucket(new CreateBucketRequest(bucketName).withObjectLockEnabledForBucket(objectLock)); @@ -222,31 +213,22 @@ public boolean createUser(long accountId, long storeId) { String username = account.getUuid(); logger.debug("Attempting to create Ceph RGW user for account {} with UUID {}", account, username); + User user; try { - Optional user = rgwAdmin.getUserInfo(username); - if (user.isPresent()) { - logger.info("User already exists in Ceph RGW: " + username); - return true; - } else { - logger.debug("User does not exist. Creating user in Ceph RGW: " + username); - } + user = rgwAdmin.getUserInfo(username).orElseGet(() -> rgwAdmin.createUser(username)); } catch (Exception e) { - logger.debug("Get user info failed for user {} with exception {}. Proceeding with user creation.", username, e.getMessage()); + throw new CloudRuntimeException("Failed to get or create Ceph RGW user " + username, e); } - - try { - rgwAdmin.createUser(username); - User newUser = rgwAdmin.getUserInfo(username).get(); - S3Credential credentials = newUser.getS3Credentials().get(0); - - Map details = new HashMap<>(); - details.put(CEPH_ACCESS_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getAccessKey())); - details.put(CEPH_SECRET_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getSecretKey())); - _accountDetailsDao.update(accountId, details); - return true; - } catch (Exception e) { - throw new CloudRuntimeException(e); + if (user.getS3Credentials().isEmpty()) { + throw new CloudRuntimeException("Ceph RGW user " + username + " has no S3 credentials"); } + S3Credential credentials = user.getS3Credentials().get(0); + + Map details = new HashMap<>(); + details.put(CEPH_ACCESS_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getAccessKey())); + details.put(CEPH_SECRET_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getSecretKey())); + _accountDetailsDao.update(accountId, details); + return true; } @Override diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java index f0b0ecf5c61a..f41ba3e42570 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.twonote.rgwadmin4j.RgwAdmin; @@ -63,7 +64,9 @@ public DataStore initialize(Map dsInfos) { String accessKey = details.get("accesskey"); String secretKey = details.get("secretkey"); - + if (StringUtils.isAnyBlank(accessKey, secretKey)) { + throw new CloudRuntimeException("Ceph RGW Admin credentials are missing"); + } Map objectStoreParameters = new HashMap(); objectStoreParameters.put("name", name); @@ -86,8 +89,7 @@ public DataStore initialize(Map dsInfos) { logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); logger.info("Successfully connected to Ceph RGW: " + url); } catch (Exception e) { - logger.debug("Error while initializing Ceph RGW Object Store: " + e.getMessage()); - throw new RuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL"); + throw new CloudRuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL", e); } ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); From cce4b6b7cbc4337e7ab78f20d34c6b12f94424f7 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:41:26 +0000 Subject: [PATCH 06/10] ceph-rgw: reuse the S3 and RGW admin clients A new client, with its own connection pool, was built for every call and never closed. --- .../driver/CephObjectStoreDriverImpl.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 4240f689ece2..7387206f3e13 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -65,6 +65,7 @@ import java.util.ArrayList; import java.util.Map; import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { @@ -89,6 +90,9 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { private static final String CEPH_ACCESS_KEY = "ceph-rgw-accesskey"; private static final String CEPH_SECRET_KEY = "ceph-rgw-secretkey"; + private final Map s3Clients = new ConcurrentHashMap<>(); + private final Map rgwAdminClients = new ConcurrentHashMap<>(); + @Override public DataStoreTO getStoreTO(DataStore store) { return null; @@ -321,15 +325,11 @@ protected RgwAdmin getRgwAdminClient(long storeId) { String url = store.getUrl(); String accessKey = storeDetails.get(ACCESS_KEY); String secretKey = storeDetails.get(SECRET_KEY); - RgwAdmin admin = new RgwAdminBuilder() + return rgwAdminClients.computeIfAbsent(url + accessKey + secretKey, key -> new RgwAdminBuilder() .accessKey(accessKey) .secretKey(secretKey) .endpoint(url + "/admin") - .build(); - if (admin == null) { - throw new CloudRuntimeException("Error while creating Ceph RGW client"); - } - return admin; + .build()); } private String getStoreURL(long storeId) { @@ -359,18 +359,13 @@ protected String getAccountKey(long accountId, long storeId, String name) { return detail.getValue(); } protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) { - AmazonS3 client = AmazonS3ClientBuilder.standard() + return s3Clients.computeIfAbsent(url + accessKey + secretKey, key -> AmazonS3ClientBuilder.standard() .enablePathStyleAccess() .withCredentials( new AWSStaticCredentialsProvider( new BasicAWSCredentials(accessKey, secretKey))) .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration(url, "us-east-1")) - .build(); - - if (client == null) { - throw new CloudRuntimeException("Error while creating Ceph RGW S3 client"); - } - return client; + .build()); } } From d47f1308e5093a97549320533ab11d15b515470d Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:42:17 +0000 Subject: [PATCH 07/10] ceph-rgw: warn about a RGW URL without HTTPS and strip its trailing slash The admin access key and the names of all buckets are no longer logged when the object store is added. --- .../lifecycle/CephObjectStoreLifeCycleImpl.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java index f41ba3e42570..55eea48b126c 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -35,7 +35,6 @@ import javax.inject.Inject; import java.util.HashMap; -import java.util.List; import java.util.Map; public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { @@ -53,7 +52,7 @@ public CephObjectStoreLifeCycleImpl() { @SuppressWarnings("unchecked") @Override public DataStore initialize(Map dsInfos) { - String url = (String)dsInfos.get("url"); + String url = StringUtils.stripEnd((String)dsInfos.get("url"), "/"); String name = (String)dsInfos.get("name"); Long size = (Long)dsInfos.get("size"); String providerName = (String)dsInfos.get("providerName"); @@ -77,7 +76,9 @@ public DataStore initialize(Map dsInfos) { objectStoreParameters.put("accesskey", accessKey); objectStoreParameters.put("secretkey", secretKey); - logger.info("Attempting to connect to Ceph RGW at " + url + " with access key " + accessKey); + if (!url.startsWith("https://")) { + logger.warn("Ceph RGW at {} is not using HTTPS, requests to it are not protected in transit", url); + } RgwAdmin rgwAdmin = new RgwAdminBuilder() .accessKey(accessKey) @@ -85,8 +86,7 @@ public DataStore initialize(Map dsInfos) { .endpoint(url + "/admin") .build(); try { - List buckets = rgwAdmin.listBucket(); - logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); + rgwAdmin.listBucket(); logger.info("Successfully connected to Ceph RGW: " + url); } catch (Exception e) { throw new CloudRuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL", e); From 9a10135e06e87d9d7460fc95897335897cdaf10d Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:42:52 +0000 Subject: [PATCH 08/10] ceph-rgw: create the RGW user with the name of the account as display name --- .../storage/datastore/driver/CephObjectStoreDriverImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 7387206f3e13..00176d9118b1 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -219,7 +219,7 @@ public boolean createUser(long accountId, long storeId) { logger.debug("Attempting to create Ceph RGW user for account {} with UUID {}", account, username); User user; try { - user = rgwAdmin.getUserInfo(username).orElseGet(() -> rgwAdmin.createUser(username)); + user = rgwAdmin.getUserInfo(username).orElseGet(() -> rgwAdmin.createUser(username, Map.of("display-name", account.getAccountName()))); } catch (Exception e) { throw new CloudRuntimeException("Failed to get or create Ceph RGW user " + username, e); } From 044c49f1545a23938a86193841ee2ab5ee88dcb0 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:43:25 +0000 Subject: [PATCH 09/10] ceph-rgw: don't allow anonymous listing of a public bucket The public policy allowed everyone to list the objects of the bucket, it now only allows reading them. A policy other than public or private was applied as private and is refused now. --- .../datastore/driver/CephObjectStoreDriverImpl.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 00176d9118b1..763185bdf8d4 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -39,6 +39,7 @@ import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.exception.InvalidParameterValueException; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; @@ -171,15 +172,6 @@ public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { builder.append("{\n"); builder.append(" \"Statement\": [\n"); builder.append(" {\n"); - builder.append(" \"Action\": [\n"); - builder.append(" \"s3:GetBucketLocation\",\n"); - builder.append(" \"s3:ListBucket\"\n"); - builder.append(" ],\n"); - builder.append(" \"Effect\": \"Allow\",\n"); - builder.append(" \"Principal\": \"*\",\n"); - builder.append(" \"Resource\": \"arn:aws:s3:::" + bucket.getName() + "\"\n"); - builder.append(" },\n"); - builder.append(" {\n"); builder.append(" \"Action\": \"s3:GetObject\",\n"); builder.append(" \"Effect\": \"Allow\",\n"); builder.append(" \"Principal\": \"*\",\n"); @@ -189,6 +181,8 @@ public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { builder.append(" \"Version\": \"2012-10-17\"\n"); builder.append("}\n"); policyConfig = builder.toString(); + } else if (!policy.equalsIgnoreCase("private")) { + throw new InvalidParameterValueException("Unsupported bucket policy: " + policy); } else { logger.debug("Setting private policy on bucket " + bucket.getName()); policyConfig = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; From 5747ea39ce763b7e51464ad41d6b656f0fdcdd06 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Sat, 21 Nov 2026 06:44:59 +0000 Subject: [PATCH 10/10] ceph-rgw: add unit tests for the RGW user, its keys and the bucket policy --- .../driver/CephObjectStoreDriverImplTest.java | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java index 68f98aee68b5..54c4226deb82 100644 --- a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java @@ -18,6 +18,16 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.SetBucketEncryptionRequest; +import com.amazonaws.services.s3.model.SetBucketPolicyRequest; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.exception.CloudRuntimeException; +import org.mockito.ArgumentCaptor; +import org.twonote.rgwadmin4j.model.S3Credential; +import org.twonote.rgwadmin4j.model.User; +import java.util.List; +import java.util.Map; +import java.util.Optional; import com.cloud.agent.api.to.BucketTO; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; @@ -39,11 +49,15 @@ import org.twonote.rgwadmin4j.RgwAdmin; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -110,4 +124,82 @@ public void testDeleteBucket() throws Exception { assertTrue(success); verify(rgwClient, times(1)).deleteBucket(anyString()); } + + private User mockRgwUser(boolean existing) { + User user = mock(User.class); + S3Credential credential = mock(S3Credential.class); + when(credential.getAccessKey()).thenReturn("access"); + when(credential.getSecretKey()).thenReturn("secret"); + when(user.getS3Credentials()).thenReturn(List.of(credential)); + when(account.getUuid()).thenReturn("uuid"); + when(account.getAccountName()).thenReturn("name"); + when(accountDao.findById(1L)).thenReturn(account); + doReturn(rgwAdmin).when(cephObjectStoreDriverImpl).getRgwAdminClient(anyLong()); + when(rgwAdmin.getUserInfo("uuid")).thenReturn(existing ? Optional.of(user) : Optional.empty()); + return user; + } + + @Test + public void testCreateUserStoresKeysPerStoreWithoutReplacingDetails() { + User user = mockRgwUser(false); + when(rgwAdmin.createUser(eq("uuid"), any())).thenReturn(user); + assertTrue(cephObjectStoreDriverImpl.createUser(1L, 2L)); + verify(accountDetailsDao).update(1L, Map.of("ceph-rgw-accesskey-2", "access", "ceph-rgw-secretkey-2", "secret")); + verify(accountDetailsDao, never()).persist(anyLong(), any()); + } + + @Test + public void testCreateUserStoresKeysOfExistingUser() { + mockRgwUser(true); + assertTrue(cephObjectStoreDriverImpl.createUser(1L, 2L)); + verify(rgwAdmin, never()).createUser(any(), any()); + verify(accountDetailsDao).update(eq(1L), any()); + } + + @Test(expected = CloudRuntimeException.class) + public void testCreateUserFailsWhenLookupFails() { + mockRgwUser(true); + when(rgwAdmin.getUserInfo("uuid")).thenThrow(new RuntimeException("connection refused")); + cephObjectStoreDriverImpl.createUser(1L, 2L); + } + + @Test + public void testGetAccountKeyFallsBackToKeyWithoutStore() { + when(accountDetailsDao.findDetail(1L, "ceph-rgw-secretkey")).thenReturn(new AccountDetailVO(1L, "ceph-rgw-secretkey", "old")); + assertEquals("old", cephObjectStoreDriverImpl.getAccountKey(1L, 2L, "ceph-rgw-secretkey")); + } + + @Test(expected = CloudRuntimeException.class) + public void testGetAccountKeyFailsWithoutKey() { + cephObjectStoreDriverImpl.getAccountKey(1L, 2L, "ceph-rgw-secretkey"); + } + + @Test(expected = CloudRuntimeException.class) + public void testCreateBucketFailsWhenBucketExists() { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(anyLong(), anyLong()); + when(rgwClient.doesBucketExistV2(anyString())).thenReturn(true); + cephObjectStoreDriverImpl.createBucket(bucket, false); + } + + @Test + public void testPublicPolicyDoesNotAllowListing() { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(any(), any(), any()); + cephObjectStoreDriverImpl.setBucketPolicy(new BucketTO("test-bucket"), "public", 1L); + ArgumentCaptor request = ArgumentCaptor.forClass(SetBucketPolicyRequest.class); + verify(rgwClient).setBucketPolicy(request.capture()); + assertTrue(request.getValue().getPolicyText().contains("s3:GetObject")); + assertFalse(request.getValue().getPolicyText().contains("s3:ListBucket")); + } + + @Test(expected = InvalidParameterValueException.class) + public void testUnknownPolicyIsRefused() { + cephObjectStoreDriverImpl.setBucketPolicy(new BucketTO("test-bucket"), "everyone", 1L); + } + + @Test + public void testSetBucketEncryption() { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(any(), any(), any()); + assertTrue(cephObjectStoreDriverImpl.setBucketEncryption(new BucketTO("test-bucket"), 1L)); + verify(rgwClient).setBucketEncryption(any(SetBucketEncryptionRequest.class)); + } }