Can't save BlobFields when using both django_gcp and django-reverse-admin
I'm trying to use both django-gcp to store large images as BlobFields, and django-reverse-admin so I can edit all of my data inline. My models look like this:
class SceneContent(models.Model):
site_title = RichCharField(
max_length=25, verbose_name="Site Title (25 char)", null=True
)
site_description = RichTextField(
max_length=300,
verbose_name="Site Description (300 char)",
null=True,
validators=[MaxLengthValidator(300)],
)
site_image = BlobField(
# blank=True,
verbose_name="Site Image (3840 x 2160 px)",
get_destination_path=get_destination_path_image,
store_key="media",
null=True,
validators=[BlobImageExtensionValidator()],
overwrite_mode="update",
)
object_title_1 = RichCharField(
max_length=25, verbose_name="Object Title 1 (25 char)", null=True
)
object_description_1 = RichTextField(
max_length=300,
verbose_name="Object Description 1 (300 char)",
null=True,
)
object_image_1 = BlobField(
# blank=True,
verbose_name="Object Image 1 (3840 x 2160 px)",
get_destination_path=get_destination_path_image,
store_key="media",
null=True,
validators=[BlobImageExtensionValidator()],
overwrite_mode="update",
)
object_title_2 = RichCharField(
max_length=25, verbose_name="Object Title 2 (25 char)", null=True
)
object_description_2 = RichTextField(
max_length=300,
verbose_name="Object Description 2 (300 char)",
null=True,
)
object_image_2 = BlobField(
# blank=True,
verbose_name="Object Image 2 (3840 x 2160 px)",
get_destination_path=get_destination_path_image,
store_key="media",
null=True,
validators=[BlobImageExtensionValidator()],
overwrite_mode="update",
)
def __str__(self):
return self.site_title
class Site(models.Model):
scene_title = RichCharField(
max_length=12, verbose_name="Title (12 char)", null=True
)
pronunciation_guide = RichCharField(
max_length=25, verbose_name="Site Pronunciation Guide (25 char)", null=True
)
scene_subtitle = RichCharField(
max_length=40, verbose_name="Subtitle (40 char)", null=True
)
scene_selection_image = BlobField(
# blank=True,
verbose_name="Background Image (1280 x 2160 px)",
get_destination_path=get_destination_path_image,
store_key="media",
null=True,
validators=[BlobImageExtensionValidator()],
overwrite_mode="update",
)
scene_content = models.OneToOneField(
SceneContent, on_delete=models.CASCADE, related_name="scene_content", null=True
)
def __str__(self):
return self.scene_title
class Meta:
verbose_name = "Site"
And I'm using django-reverse-admin so that when I create a Site in my admin interface, I create my SceneContent inline.
However, when I save the model in my admin interface, I'm getting a MissingBlobError. It appears that when django-gcp goes to copy_blob, the blob is no longer available in its temporary location. My guess is that django-reverse-admin is causing the save order of the OneToOne relationship and parent model to be different than django-gcp is expecting (or the parent models gets saved/validated multiple times), and this is causing the issue.
I've noticed that the parent model (Site) has no issue with saving its BlobField, and I get errors ONLY when saving the BlobField fields in my OneToOne model.
The error is below:
Internal Server Error: /admin/mesoamerica/site/64/change/
Traceback (most recent call last):
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django_gcp\storage\operations.py", line 52, in copy_blob
destination_blob = source_bucket.copy_blob(
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\google\cloud\storage\bucket.py", line 1910, in copy_blob
copy_result = client._post_resource(
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\google\cloud\storage\client.py", line 627, in _post_resource
return self._connection.api_request(
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\google\cloud\storage\_http.py", line 72, in api_request
return call()
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\google\cloud\_http\__init__.py", line 494, in api_request
raise exceptions.from_http_response(response)
google.api_core.exceptions.NotFound: 404 POST https://storage.googleapis.com/storage/v1/b/my-museum-multimedia/o/_tmp%2Fd1263eb1-0469-4ea4-ace0-272d8ecdefba/copyTo/b/my-museum-multimedia/o/mesoamerica%2Fimages%2FCh1-e8c868fa-f800-47d7-8b67-0e0d0676f2de.png?prettyPrint=false: No such object:
my-museum-multimedia/_tmp/d1263eb1-0469-4ea4-ace0-272d8ecdefba
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\core\handlers\exception.py", line 55, in inner
response = get_response(request)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\core\handlers\base.py", line 197, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\contrib\admin\options.py", line 688, in wrapper
return self.admin_site.admin_view(view)(*args, **kwargs)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\utils\decorators.py", line 134, in _wrapper_view
response = view_func(request, *args, **kwargs)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\views\decorators\cache.py", line 62, in _wrapper_view_func
response = view_func(request, *args, **kwargs)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\contrib\admin\sites.py", line 242, in inner
return view(request, *args, **kwargs)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django_reverse_admin\__init__.py", line 207, in change_view
return self._changeform_view(request, object_id, form_url, extra_context)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django_reverse_admin\__init__.py", line 275, in _changeform_view
self._save_object(request, new_object, form, formsets, add)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django_reverse_admin\__init__.py", line 214, in _save_object
self.save_related(request, form, formsets, change=not add)
File "D:\Projects\myproject-django-cms\projects\cms\mesoamerica\admin.py", line 56, in save_related
with transaction.atomic():
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\db\transaction.py", line 307, in __exit__
connection.set_autocommit(True)
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\db\backends\base\base.py", line 501, in set_autocommit
self.run_and_clear_commit_hooks()
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django\db\backends\base\base.py", line 779, in run_and_clear_commit_hooks
func()
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django_gcp\storage\fields.py", line 255, in on_commit_valid
copy_blob(
File "D:\Projects\myproject-django-cms\projects\cms\env\lib\site-packages\django_gcp\storage\operations.py", line 86, in copy_blob
raise MissingBlobError(
django_gcp.exceptions.MissingBlobError: Could not complete copy: source blob _tmp/d1263eb1-0469-4ea4-ace0-272d8ecdefba does not exist in bucket my-museum-multimedia
My first thought was to modify the save order; I updated my wsgi.py to include:
def _changeform_view_reverted(self, request, object_id, form_url, extra_context):
add = object_id is None
model = self.model
opts = model._meta
if add:
if not self.has_add_permission(request):
raise PermissionDenied
obj = None
else:
obj = self.get_object(request, unquote(object_id))
if request.method == "POST":
if not self.has_change_permission(request, obj):
raise PermissionDenied
else:
if not self.has_view_or_change_permission(request, obj):
raise PermissionDenied
if obj is None:
return self._get_obj_does_not_exist_redirect(request, opts, object_id)
formsets = []
model_form = self.get_form(request, obj=obj, change=not add)
if request.method == "POST":
form = model_form(request.POST, request.FILES, instance=obj)
form_validated = form.is_valid()
if form_validated:
new_object = self.save_form(request, form, change=not add)
else:
new_object = form.instance
logger.info(
f"Parent model {new_object} saved or validated. Proceeding to formsets."
)
formsets, inline_instances = self._create_formsets(
request, new_object, change=not add
)
formset_inline_tuples = zip(formsets, self.get_inline_instances(request))
formset_inline_tuples = _remove_blank_reverse_inlines(
new_object, formset_inline_tuples
)
formsets = [t[0] for t in formset_inline_tuples]
# Step 1: Save the parent model first without inlines
if form_validated:
try:
# Save the parent model first
self.save_model(request, new_object, form, not add)
logger.info(f"Parent model {new_object} saved. Saving inlines next.")
# Step 2: Save normal inlines (non-reverse inlines)
for formset, inline in formset_inline_tuples:
if not isinstance(inline, ReverseInlineModelAdmin):
self.save_formset(request, form, formset, change=not add)
logger.info(f"Saved normal formset for {inline}.")
# Step 3: Save reverse inlines with BlobFields after the parent model is saved
for formset, inline in formset_inline_tuples:
if isinstance(inline, ReverseInlineModelAdmin):
if formset.is_valid(): # Ensure the formset is valid
logger.info(
f"Reverse inline formset for {inline.__class__.__name__} is valid."
)
# Save only the reverse inline forms that have changes
for form_instance in formset.forms:
if (
form_instance.has_changed()
): # Check if the form has changes
logger.info(
f"Form {form_instance.__class__.__name__} with instance ID {form_instance.instance.pk if form_instance.instance.pk else 'New'} has changes and will be saved."
)
# Delay saving BlobField to set the relationship first
obj = form_instance.save(
commit=False
) # Don't save yet
# Set the OneToOneField or ForeignKey relationship on the parent model
setattr(new_object, inline.parent_fk_name, obj)
logger.info(
f"SceneContent relationship set: {inline.parent_fk_name} -> {obj}"
)
# Do not save the object yet to avoid the BlobField being prematurely saved
else:
logger.info(
f"Form {form_instance.__class__.__name__} has no changes and will not be saved."
)
# Step 4: Now save all reverse inline objects (including BlobFields) after the parent model and related objects
for formset, inline in formset_inline_tuples:
if isinstance(inline, ReverseInlineModelAdmin):
saved_objects = [] # List to keep track of newly saved objects
changed_objects = [] # List to keep track of changed objects
deleted_objects = [] # List to keep track of deleted objects
forms = [f for f in formset if f.has_changed()]
if forms:
for form_instance in forms:
logger.info(
f"Saving reverse inline object {form_instance.__class__.__name__} (including BlobField)."
)
# Final save here, including BlobField
form_instance.instance.save()
if form_instance.instance.pk is None:
saved_objects.append(
form_instance.instance
) # Track new objects
else:
changed_fields = (
form_instance.changed_data
) # Track changed fields
changed_objects.append(
(form_instance.instance, changed_fields)
) # Track changed objects and fields
# Manually add the saved, changed, and deleted objects to the formset attributes
formset.new_objects = saved_objects
formset.changed_objects = changed_objects
formset.deleted_objects = (
deleted_objects # Add the deleted objects
)
# Re-save the parent object (new_object) to persist the OneToOneField relationship
new_object.save()
logger.info(f"All reverse inline objects saved.")
# Step 5: Log the change and return the appropriate response
change_message = self.construct_change_message(
request, form, formsets, add
)
if add:
self.log_addition(request, new_object, change_message)
return self.response_add(request, new_object)
else:
self.log_change(request, new_object, change_message)
return self.response_change(request, new_object)
except AttemptedOverwriteError:
form.add_error(
None,
"A file you uploaded has the same name as an existing file. Please check your filenames and try again.",
)
logger.error("Attempted file overwrite error.")
else:
logger.error("Form validation failed.")
form_validated = False
else:
# Prepare the dict of initial data from the request.
initial = dict(request.GET.items())
for k in initial:
try:
f = opts.get_field(k)
except FieldDoesNotExist:
continue
if isinstance(f, models.ManyToManyField):
initial[k] = initial[k].split(",")
if add:
form = model_form(initial=initial)
prefixes = {}
for FormSet, inline in self.get_formsets_with_inlines(request):
prefix = FormSet.get_default_prefix()
prefixes[prefix] = prefixes.get(prefix, 0) + 1
if prefixes[prefix] != 1:
prefix = "%s-%s" % (prefix, prefixes[prefix])
formset = FormSet(instance=self.model(), prefix=prefix)
formsets.append(formset)
else:
form = model_form(instance=obj)
formsets, inline_instances = self._create_formsets(
request, obj, change=True
)
if not add and not self.has_change_permission(request, obj):
readonly_fields = flatten_fieldsets(self.get_fieldsets(request, obj))
else:
readonly_fields = self.get_readonly_fields(request, obj)
adminForm = helpers.AdminForm(
form,
list(self.get_fieldsets(request)),
self.prepopulated_fields,
readonly_fields=readonly_fields,
model_admin=self,
)
media = self.media + adminForm.media
inline_admin_formsets = self.get_inline_formsets(
request, formsets, self.get_inline_instances(request), obj
)
for inline_formset in inline_admin_formsets:
media = media + inline_formset.media
context = self.admin_site.each_context(request)
reverse_admin_context = {
"title": _(("Change %s", "Add %s")[add]) % force_str(opts.verbose_name),
"adminform": adminForm,
"is_popup": False,
"object_id": object_id,
"original": obj,
"media": mark_safe(media),
"inline_admin_formsets": inline_admin_formsets,
"errors": helpers.AdminErrorList(form, formsets),
"app_label": opts.app_label,
}
context.update(reverse_admin_context)
context.update(extra_context or {})
return self.render_change_form(
request,
context,
form_url=form_url,
add=add,
change=not add,
obj=obj,
)
ReverseModelAdmin._changeform_view = _changeform_view_reverted
But this didn't resolve the problem - it actually made it so that I'd get MissingBlobErrors when creating the parent model, and then the OneToOne model would save properly.
What else could I try to resolve this issue?