diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index b3ad2d9552f1..763185bdf8d4 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -27,18 +27,27 @@ import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; import com.amazonaws.services.s3.model.BucketVersioningConfiguration; +import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; +import com.amazonaws.services.s3.model.SSEAlgorithm; +import com.amazonaws.services.s3.model.ServerSideEncryptionByDefault; +import com.amazonaws.services.s3.model.ServerSideEncryptionConfiguration; +import com.amazonaws.services.s3.model.ServerSideEncryptionRule; +import com.amazonaws.services.s3.model.SetBucketEncryptionRequest; import com.amazonaws.services.s3.model.SetBucketPolicyRequest; import com.amazonaws.services.s3.model.GetBucketPolicyRequest; import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.exception.InvalidParameterValueException; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; import com.cloud.user.Account; +import com.cloud.user.AccountDetailVO; import com.cloud.user.AccountDetailsDao; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; @@ -55,9 +64,9 @@ import javax.inject.Inject; import java.util.List; import java.util.ArrayList; -import java.util.Optional; import java.util.Map; import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { @@ -82,6 +91,9 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { private static final String CEPH_ACCESS_KEY = "ceph-rgw-accesskey"; private static final String CEPH_SECRET_KEY = "ceph-rgw-secretkey"; + private final Map s3Clients = new ConcurrentHashMap<>(); + private final Map rgwAdminClients = new ConcurrentHashMap<>(); + @Override public DataStoreTO getStoreTO(DataStore store) { return null; @@ -94,21 +106,13 @@ public Bucket createBucket(Bucket bucket, boolean objectLock) { long accountId = bucket.getAccountId(); AmazonS3 s3client = getS3Client(storeId, accountId); - try { - if (s3client.doesBucketExistV2(bucketName)) { - throw new CloudRuntimeException("Bucket already exists with the name: " + bucketName); - } - } catch (AmazonS3Exception e) { - if (e.getStatusCode() != 404) { - throw new CloudRuntimeException(e); - } - } catch (Exception e) { - throw new CloudRuntimeException(e); + if (s3client.doesBucketExistV2(bucketName)) { + throw new CloudRuntimeException("Bucket already exists with the name: " + bucketName); } try { - s3client.createBucket(bucketName); - String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); - String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); + s3client.createBucket(new CreateBucketRequest(bucketName).withObjectLockEnabledForBucket(objectLock)); + String accessKey = getAccountKey(accountId, storeId, CEPH_ACCESS_KEY); + String secretKey = getAccountKey(accountId, storeId, CEPH_SECRET_KEY); ObjectStoreVO store = _storeDao.findById(storeId); BucketVO bucketVO = _bucketDao.findById(bucket.getId()); bucketVO.setAccessKey(accessKey); @@ -140,10 +144,8 @@ public List listBuckets(long storeId) { @Override public boolean deleteBucket(BucketTO bucket, long storeId) { - RgwAdmin rgwAdmin = getRgwAdminClient(storeId); - try { - rgwAdmin.removeBucket(bucket.getName()); + getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getSecretKey()).deleteBucket(bucket.getName()); } catch (Exception e) { throw new CloudRuntimeException(e); } @@ -170,15 +172,6 @@ public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { builder.append("{\n"); builder.append(" \"Statement\": [\n"); builder.append(" {\n"); - builder.append(" \"Action\": [\n"); - builder.append(" \"s3:GetBucketLocation\",\n"); - builder.append(" \"s3:ListBucket\"\n"); - builder.append(" ],\n"); - builder.append(" \"Effect\": \"Allow\",\n"); - builder.append(" \"Principal\": \"*\",\n"); - builder.append(" \"Resource\": \"arn:aws:s3:::" + bucket.getName() + "\"\n"); - builder.append(" },\n"); - builder.append(" {\n"); builder.append(" \"Action\": \"s3:GetObject\",\n"); builder.append(" \"Effect\": \"Allow\",\n"); builder.append(" \"Principal\": \"*\",\n"); @@ -188,6 +181,8 @@ public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { builder.append(" \"Version\": \"2012-10-17\"\n"); builder.append("}\n"); policyConfig = builder.toString(); + } else if (!policy.equalsIgnoreCase("private")) { + throw new InvalidParameterValueException("Unsupported bucket policy: " + policy); } else { logger.debug("Setting private policy on bucket " + bucket.getName()); policyConfig = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; @@ -216,41 +211,38 @@ public boolean createUser(long accountId, long storeId) { String username = account.getUuid(); logger.debug("Attempting to create Ceph RGW user for account {} with UUID {}", account, username); + User user; try { - Optional user = rgwAdmin.getUserInfo(username); - if (user.isPresent()) { - logger.info("User already exists in Ceph RGW: " + username); - return true; - } else { - logger.debug("User does not exist. Creating user in Ceph RGW: " + username); - } + user = rgwAdmin.getUserInfo(username).orElseGet(() -> rgwAdmin.createUser(username, Map.of("display-name", account.getAccountName()))); } catch (Exception e) { - logger.debug("Get user info failed for user {} with exception {}. Proceeding with user creation.", username, e.getMessage()); + throw new CloudRuntimeException("Failed to get or create Ceph RGW user " + username, e); } - - try { - rgwAdmin.createUser(username); - User newUser = rgwAdmin.getUserInfo(username).get(); - S3Credential credentials = newUser.getS3Credentials().get(0); - - Map details = new HashMap<>(); - details.put(CEPH_ACCESS_KEY, credentials.getAccessKey()); - details.put(CEPH_SECRET_KEY, credentials.getSecretKey()); - _accountDetailsDao.persist(accountId, details); - return true; - } catch (Exception e) { - throw new CloudRuntimeException(e); + if (user.getS3Credentials().isEmpty()) { + throw new CloudRuntimeException("Ceph RGW user " + username + " has no S3 credentials"); } + S3Credential credentials = user.getS3Credentials().get(0); + + Map details = new HashMap<>(); + details.put(CEPH_ACCESS_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getAccessKey())); + details.put(CEPH_SECRET_KEY + "-" + storeId, DBEncryptionUtil.encrypt(credentials.getSecretKey())); + _accountDetailsDao.update(accountId, details); + return true; } @Override public boolean setBucketEncryption(BucketTO bucket, long storeId) { - return false; + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getSecretKey()); + client.setBucketEncryption(new SetBucketEncryptionRequest().withBucketName(bucket.getName()) + .withServerSideEncryptionConfiguration(new ServerSideEncryptionConfiguration().withRules( + new ServerSideEncryptionRule().withApplyServerSideEncryptionByDefault( + new ServerSideEncryptionByDefault().withSSEAlgorithm(SSEAlgorithm.AES256))))); + return true; } @Override public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { - return false; + getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getSecretKey()).deleteBucketEncryption(bucket.getName()); + return true; } @Override @@ -327,15 +319,11 @@ protected RgwAdmin getRgwAdminClient(long storeId) { String url = store.getUrl(); String accessKey = storeDetails.get(ACCESS_KEY); String secretKey = storeDetails.get(SECRET_KEY); - RgwAdmin admin = new RgwAdminBuilder() + return rgwAdminClients.computeIfAbsent(url + accessKey + secretKey, key -> new RgwAdminBuilder() .accessKey(accessKey) .secretKey(secretKey) .endpoint(url + "/admin") - .build(); - if (admin == null) { - throw new CloudRuntimeException("Error while creating Ceph RGW client"); - } - return admin; + .build()); } private String getStoreURL(long storeId) { @@ -346,23 +334,32 @@ private String getStoreURL(long storeId) { protected AmazonS3 getS3Client(long storeId, long accountId) { String url = getStoreURL(storeId); - String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); - String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); - return this.getS3Client(url, accessKey, secretKey); + return this.getS3Client(url, getAccountKey(accountId, storeId, CEPH_ACCESS_KEY), getAccountKey(accountId, storeId, CEPH_SECRET_KEY)); + } + + /** + * Returns the RGW key of the account on the given store. Keys stored before they were kept per store + * have no store ID in their name and are not encrypted. + */ + protected String getAccountKey(long accountId, long storeId, String name) { + AccountDetailVO detail = _accountDetailsDao.findDetail(accountId, name + "-" + storeId); + if (detail != null) { + return DBEncryptionUtil.decrypt(detail.getValue()); + } + detail = _accountDetailsDao.findDetail(accountId, name); + if (detail == null) { + throw new CloudRuntimeException("No Ceph RGW " + name + " found for account " + accountId); + } + return detail.getValue(); } protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) { - AmazonS3 client = AmazonS3ClientBuilder.standard() + return s3Clients.computeIfAbsent(url + accessKey + secretKey, key -> AmazonS3ClientBuilder.standard() .enablePathStyleAccess() .withCredentials( new AWSStaticCredentialsProvider( new BasicAWSCredentials(accessKey, secretKey))) .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration(url, "us-east-1")) - .build(); - - if (client == null) { - throw new CloudRuntimeException("Error while creating Ceph RGW S3 client"); - } - return client; + .build()); } } diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java index f0b0ecf5c61a..55eea48b126c 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.twonote.rgwadmin4j.RgwAdmin; @@ -34,7 +35,6 @@ import javax.inject.Inject; import java.util.HashMap; -import java.util.List; import java.util.Map; public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { @@ -52,7 +52,7 @@ public CephObjectStoreLifeCycleImpl() { @SuppressWarnings("unchecked") @Override public DataStore initialize(Map dsInfos) { - String url = (String)dsInfos.get("url"); + String url = StringUtils.stripEnd((String)dsInfos.get("url"), "/"); String name = (String)dsInfos.get("name"); Long size = (Long)dsInfos.get("size"); String providerName = (String)dsInfos.get("providerName"); @@ -63,7 +63,9 @@ public DataStore initialize(Map dsInfos) { String accessKey = details.get("accesskey"); String secretKey = details.get("secretkey"); - + if (StringUtils.isAnyBlank(accessKey, secretKey)) { + throw new CloudRuntimeException("Ceph RGW Admin credentials are missing"); + } Map objectStoreParameters = new HashMap(); objectStoreParameters.put("name", name); @@ -74,7 +76,9 @@ public DataStore initialize(Map dsInfos) { objectStoreParameters.put("accesskey", accessKey); objectStoreParameters.put("secretkey", secretKey); - logger.info("Attempting to connect to Ceph RGW at " + url + " with access key " + accessKey); + if (!url.startsWith("https://")) { + logger.warn("Ceph RGW at {} is not using HTTPS, requests to it are not protected in transit", url); + } RgwAdmin rgwAdmin = new RgwAdminBuilder() .accessKey(accessKey) @@ -82,12 +86,10 @@ public DataStore initialize(Map dsInfos) { .endpoint(url + "/admin") .build(); try { - List buckets = rgwAdmin.listBucket(); - logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); + rgwAdmin.listBucket(); logger.info("Successfully connected to Ceph RGW: " + url); } catch (Exception e) { - logger.debug("Error while initializing Ceph RGW Object Store: " + e.getMessage()); - throw new RuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL"); + throw new CloudRuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL", e); } ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java index 33919107b5ae..54c4226deb82 100644 --- a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java @@ -17,6 +17,17 @@ package org.apache.cloudstack.storage.datastore.driver; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.amazonaws.services.s3.model.SetBucketEncryptionRequest; +import com.amazonaws.services.s3.model.SetBucketPolicyRequest; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.exception.CloudRuntimeException; +import org.mockito.ArgumentCaptor; +import org.twonote.rgwadmin4j.model.S3Credential; +import org.twonote.rgwadmin4j.model.User; +import java.util.List; +import java.util.Map; +import java.util.Optional; import com.cloud.agent.api.to.BucketTO; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; @@ -38,11 +49,15 @@ import org.twonote.rgwadmin4j.RgwAdmin; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -97,16 +112,94 @@ public void testCreateBucket() throws Exception { Bucket bucketRet = cephObjectStoreDriverImpl.createBucket(bucket, false); assertEquals(bucketRet.getName(), bucket.getName()); verify(rgwClient, times(1)).doesBucketExistV2(anyString()); - verify(rgwClient, times(1)).createBucket(anyString()); + verify(rgwClient, times(1)).createBucket(any(CreateBucketRequest.class)); } @Test public void testDeleteBucket() throws Exception { String bucketName = "test-bucket"; BucketTO bucket = new BucketTO(bucketName); - doReturn(rgwAdmin).when(cephObjectStoreDriverImpl).getRgwAdminClient(anyLong()); + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(any(), any(), any()); boolean success = cephObjectStoreDriverImpl.deleteBucket(bucket, 1L); assertTrue(success); - verify(rgwAdmin, times(1)).removeBucket(anyString()); + verify(rgwClient, times(1)).deleteBucket(anyString()); + } + + private User mockRgwUser(boolean existing) { + User user = mock(User.class); + S3Credential credential = mock(S3Credential.class); + when(credential.getAccessKey()).thenReturn("access"); + when(credential.getSecretKey()).thenReturn("secret"); + when(user.getS3Credentials()).thenReturn(List.of(credential)); + when(account.getUuid()).thenReturn("uuid"); + when(account.getAccountName()).thenReturn("name"); + when(accountDao.findById(1L)).thenReturn(account); + doReturn(rgwAdmin).when(cephObjectStoreDriverImpl).getRgwAdminClient(anyLong()); + when(rgwAdmin.getUserInfo("uuid")).thenReturn(existing ? Optional.of(user) : Optional.empty()); + return user; + } + + @Test + public void testCreateUserStoresKeysPerStoreWithoutReplacingDetails() { + User user = mockRgwUser(false); + when(rgwAdmin.createUser(eq("uuid"), any())).thenReturn(user); + assertTrue(cephObjectStoreDriverImpl.createUser(1L, 2L)); + verify(accountDetailsDao).update(1L, Map.of("ceph-rgw-accesskey-2", "access", "ceph-rgw-secretkey-2", "secret")); + verify(accountDetailsDao, never()).persist(anyLong(), any()); + } + + @Test + public void testCreateUserStoresKeysOfExistingUser() { + mockRgwUser(true); + assertTrue(cephObjectStoreDriverImpl.createUser(1L, 2L)); + verify(rgwAdmin, never()).createUser(any(), any()); + verify(accountDetailsDao).update(eq(1L), any()); + } + + @Test(expected = CloudRuntimeException.class) + public void testCreateUserFailsWhenLookupFails() { + mockRgwUser(true); + when(rgwAdmin.getUserInfo("uuid")).thenThrow(new RuntimeException("connection refused")); + cephObjectStoreDriverImpl.createUser(1L, 2L); + } + + @Test + public void testGetAccountKeyFallsBackToKeyWithoutStore() { + when(accountDetailsDao.findDetail(1L, "ceph-rgw-secretkey")).thenReturn(new AccountDetailVO(1L, "ceph-rgw-secretkey", "old")); + assertEquals("old", cephObjectStoreDriverImpl.getAccountKey(1L, 2L, "ceph-rgw-secretkey")); + } + + @Test(expected = CloudRuntimeException.class) + public void testGetAccountKeyFailsWithoutKey() { + cephObjectStoreDriverImpl.getAccountKey(1L, 2L, "ceph-rgw-secretkey"); + } + + @Test(expected = CloudRuntimeException.class) + public void testCreateBucketFailsWhenBucketExists() { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(anyLong(), anyLong()); + when(rgwClient.doesBucketExistV2(anyString())).thenReturn(true); + cephObjectStoreDriverImpl.createBucket(bucket, false); + } + + @Test + public void testPublicPolicyDoesNotAllowListing() { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(any(), any(), any()); + cephObjectStoreDriverImpl.setBucketPolicy(new BucketTO("test-bucket"), "public", 1L); + ArgumentCaptor request = ArgumentCaptor.forClass(SetBucketPolicyRequest.class); + verify(rgwClient).setBucketPolicy(request.capture()); + assertTrue(request.getValue().getPolicyText().contains("s3:GetObject")); + assertFalse(request.getValue().getPolicyText().contains("s3:ListBucket")); + } + + @Test(expected = InvalidParameterValueException.class) + public void testUnknownPolicyIsRefused() { + cephObjectStoreDriverImpl.setBucketPolicy(new BucketTO("test-bucket"), "everyone", 1L); + } + + @Test + public void testSetBucketEncryption() { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(any(), any(), any()); + assertTrue(cephObjectStoreDriverImpl.setBucketEncryption(new BucketTO("test-bucket"), 1L)); + verify(rgwClient).setBucketEncryption(any(SetBucketEncryptionRequest.class)); } } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index af2965d4a6d4..65e84b894ed5 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1564,6 +1564,7 @@ public static Map getDomainDetails(long domainId) { public static Map getAccountDetails(long accountId) { Map details = s_accountDetailsDao.findDetails(accountId); + details.keySet().removeIf(key -> key.toLowerCase().contains("secretkey")); return details.isEmpty() ? null : details; }