Новая версия botocore нарушает интеграционный тест

Представьте следующие функции, которые должны загружать и копировать что-то на S3

class TestAttachments(TestCase):
    # YAML is the only serializer supporting binary
    @override_env(AWS_DEFAULT_REGION='eu-west-1')  # So the test doesn't fail depending on env. vars
    @my_vcr.use_cassette(serializer='yaml')
    def test_copy_attachments_to_sent_folder(self):
        with self.assertRaises(
                CopyAttachmentException,
                msg="Failed to copy attachments to sent folder for attachment URL: http://example.com/foo.jpg"
        ) as cm:
            copy_attachments_to_sent_folder(["http://example.com/foo.jpg"])
        self.assertEqual(cm.exception.__cause__.__class__, InvalidAttachmentURL)

        TEST_UUIDS = ["uuid_0"]
        with patch.object(uuid, 'uuid4', side_effect=TEST_UUIDS):
            result = copy_attachments_to_sent_folder([
                f"https://{settings.MESSAGE_ATTACHMENTS_S3_BUCKET}.s3.amazonaws.com/attachments/Test+video.mov"
            ])

        self.assertEqual(
            [AttachmentMetadata(
                s3_key=f"attachments/sent/{TEST_UUIDS[0]}/video/quicktime/Test video.mov",
                filename="Test video.mov",
                mime_type="video/quicktime",
                size=178653,
            )],
            result
        )

Необходимо протестировать следующую функцию:

def copy_attachments_to_sent_folder(urls: List[str]) -> List[AttachmentMetadata]:
    # Copy the attachment to the sent folder in parallel
    with futures.ThreadPoolExecutor(max_workers=4) as executor:
        urls_by_future = {executor.submit(copy_attachment_to_sent_folder, url): url for url in urls}
        results_by_url = {}
        for future in futures.as_completed(urls_by_future.keys()):
            try:
                results_by_url[urls_by_future[future]] = future.result()
            except Exception as e:
                raise CopyAttachmentException(
                    f"Failed to copy attachments to sent folder for attachment URL: {urls_by_future[future]}"
                ) from e

        # The futures can complete out-of-order, so we need to re-order them to match the original order here
        return [results_by_url[url] for url in urls]

В итоге внутри используется эта функция:

def copy_attachment_to_sent_folder(url: str) -> AttachmentMetadata:
    aws_session = attachments_aws_session()
    s3_client = aws_session.client("s3")
    parse_result = urlparse(url, allow_fragments=False)

    # Allow both with/without AWS region in hostname
    attachment_hostname_regex = fr"{settings.MESSAGE_ATTACHMENTS_S3_BUCKET}\.s3\.(.+?\.)?amazonaws\.com"

    if not (
        parse_result.hostname is not None and re.fullmatch(attachment_hostname_regex, parse_result.hostname) is not None
        and parse_result.scheme == "https"
    ):
        raise InvalidAttachmentURL(f"URL {url} is not a valid attachment URL")

    path = unquote_plus(parse_result.path)
    key = path.lstrip("/")
    _, _, filename = key.rpartition("/")

    object_metadata = get_s3_object_metadata(aws_session=aws_session, object_bucket=settings.MESSAGE_ATTACHMENTS_S3_BUCKET,
                                             object_key=key)

    s3_resource = aws_session.resource("s3")

    # Place images in their own path so that provider1 only has access to these files
    destination_key = f"attachments/sent/{uuid.uuid4()}/{object_metadata.mime_type}/{filename}"

    try:
        # Copy to sent attachments folder and set content-type
        response = s3_resource.Object(settings.MESSAGE_ATTACHMENTS_S3_BUCKET, destination_key).copy_from(
            # Tell S3 to set cache header to "Forever"
            Expires=datetime(2100, 1, 1),
            CopySource={"Bucket": settings.MESSAGE_ATTACHMENTS_S3_BUCKET, "Key": key},
            ACL="private",
            MetadataDirective="REPLACE",
            # Set mime type on the destination file
            ContentType=object_metadata.mime_type,
            # Only copy it if the user did not modify the file since we fetched it to detect mimetype :)
            # S3 allows overwriting of the files. A user could potentially overwrite
            # the already uploaded file with a file of a different type after they upload it and after we detect the
            # mimetype, but before we copy it to the destination. Setting CopySourceIfUnmodifiedSince prevents that.
            CopySourceIfUnmodifiedSince=object_metadata.last_modified,
        )
    except ClientError as e:
        raise CopyAttachmentException from e

    if response.get("CopyObjectResult", {}).get("ETag", None) is None:
        raise CopyAttachmentException(
            f"Copy of object '{key}' to '{destination_key}' was not successful. Response: {response}"
        )

    return AttachmentMetadata(
        s3_key=destination_key,
        filename=filename,
        mime_type=object_metadata.mime_type,
        size=object_metadata.byte_size,
    )

Это отлично работает на botocore==1.23.46

Но как только мы обновляем его, мы получаем следующую ошибку:

Error
Traceback (most recent call last):
  File "/tl/test/messaging/attachments.py", line 105, in copy_attachments_to_sent_folder
    results_by_url[urls_by_future[future]] = future.result()
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/concurrent/futures/_base.py", line 439, in result
    return self.__get_result()
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/tl/test/messaging/attachments.py", line 64, in copy_attachment_to_sent_folder
    destination_key = f"attachments/sent/{uuid.uuid4()}/{object_metadata.mime_type}/{filename}"
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/unittest/mock.py", line 1104, in __call__
    return self._mock_call(*args, **kwargs)
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/unittest/mock.py", line 1108, in _mock_call
    return self._execute_mock_call(*args, **kwargs)
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/unittest/mock.py", line 1165, in _execute_mock_call
    result = next(effect)
StopIteration

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/tl/.pyenv/versions/3.10.2/lib/python3.10/contextlib.py", line 79, in inner
    return func(*args, **kwds)
  File "/tl/test/venv/lib/python3.10/site-packages/vcr/cassette.py", line 100, in __call__
    return type(self)(self.cls, args_getter)._execute_function(function, args, kwargs)
  File "/tl/test/venv/lib/python3.10/site-packages/vcr/cassette.py", line 114, in _execute_function
    return self._handle_function(fn=handle_function)
  File "/tl/test/venv/lib/python3.10/site-packages/vcr/cassette.py", line 138, in _handle_function
    return fn(cassette)
  File "/tl/test/venv/lib/python3.10/site-packages/vcr/cassette.py", line 107, in handle_function
    return function(*args, **kwargs)
  File "/tl/test/messaging/test_attachments.py", line 27, in test_copy_attachments_to_sent_folder
    result = copy_attachments_to_sent_folder([
  File "/tl/test/messaging/attachments.py", line 107, in copy_attachments_to_sent_folder
    raise CopyAttachmentException(
messaging.attachments.CopyAttachmentException: Failed to copy attachments to sent folder for attachment URL: https://test-message-attachments.s3.amazonaws.com/attachments/Test+video.mov

Я предполагаю, что это связано с destination_key = f"attachments/sent/{uuid.uuid4()}/{object_metadata.mime_type}/{filename}", но я не уверен, что я делаю неправильно? И почему именно это работает на botocore==<1.2.46, а не после.

Вернуться на верх