diff --git a/cubicweb_s3storage/storages.py b/cubicweb_s3storage/storages.py index d3ee11166a5b2016f4f6d52c223e273306ed7d94_Y3ViaWN3ZWJfczNzdG9yYWdlL3N0b3JhZ2VzLnB5..9087316504dad4250aadcf66b5e957efd68507ed_Y3ViaWN3ZWJfczNzdG9yYWdlL3N0b3JhZ2VzLnB5 100644 --- a/cubicweb_s3storage/storages.py +++ b/cubicweb_s3storage/storages.py @@ -58,7 +58,7 @@ # FIXME need a way to check that the attribute is actually edited try: suffixed_key = self.suffixed_key(key) - return self.download(suffixed_key) + return self.get_s3_object(cnx, suffixed_key) except Exception: pass try: @@ -62,7 +62,7 @@ except Exception: pass try: - return self.download(key) + return self.get_s3_object(cnx, key) except Exception as ex: source.critical("can't retrieve S3 object %s: %s", key, ex) return None @@ -76,7 +76,7 @@ if PY3: key = key.decode('utf-8') try: - return self.download(key) + return self.get_s3_object(entity._cw, key) except Exception: return None @@ -87,6 +87,8 @@ self.entity_deleted(entity, attr) else: oldkey = self.get_s3_key(entity, attr) + if oldkey is not None: + oldkey, _ = self.parse_key(oldkey) key = self.new_s3_key(entity, attr) # copy Binary value, workaround for boto3 bug # https://github.com/boto/s3transfer/issues/80 @@ -137,6 +139,7 @@ if key is None: # no key to remove return + key, _ = self.parse_key(key) self.delay_deletion(entity, attr, key) def delay_deletion(self, entity, attr, key): @@ -172,6 +175,13 @@ Binary(id_string.encode('utf-8'))) return entity + def parse_key(self, key): + try: + key, version = key.rsplit(self.KEY_SEPARATOR, 1) + except ValueError: + return key, None + return key, version + def format_version_id_suffix(self, key, version_id): """ Format the string that will store key and version id @@ -207,8 +217,31 @@ def suffixed_key(self, key): return key + self.suffix - def download(self, key): - result = self.s3cnx.get_object(Bucket=self.bucket, Key=key) + def get_s3_object(self, cnx, key): + """ + :param cnx: (Object) + :param key: (string) + get s3 stored attribute for key + handle the case of versioned object + """ + versioning_activated = cnx.repo.config[ + 's3-activate-object-versioning' + ] + # check first : does the key contain a '<separator>' + key, version_id = self.parse_key(key) + + # if object-versioning is activated use the version_id + if versioning_activated and version_id is not None: + return self.download(key, VersionId=version_id) + return self.download(key) + + def download(self, key, **kwargs): + """ + :param key: (string) + :param kwargs: (dict) Keys must be compatible with method S3.Client.put_object + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object + """ + result = self.s3cnx.get_object(Bucket=self.bucket, Key=key, **kwargs) self.info('Downloaded %s/%s from S3', self.bucket, key) return Binary(result['Body'].read())