Multiple storage backends
Since my project deals with a lot of audio that I've been serving as static files, and since all of my other static files (css, js, svg, fonts, jpg, mp4, etc.) are really small file sizes (from 497 bytes to 667 kb) that Digital Oceans Spaces recommends be served another way, I've decided to just serve static files from the same server that is serving my django project, and I implemented a model for audio files to be served from Digital Oceans Spaces.
In summary, I am still using the django-storages library, but I have one way of serving static files (standard staticfiles strategy that requires running collectstatic command everytime a static file changes or is added) and two different ways of serving media files (my site's media files and user media files). Below is a peek at my settings.py and a simplified view of my models.py so you can see the code I used to implement this static/media files strategy.
Couple of notes:
- using django-environ package for environment variables
- 'default' is the storage scheme (with
user_media_storage_backend) that is for user media files
- 'appmedia' is the storage scheme (with
app_media_storage_backend) for my site's media files
- I like separating the two simply because I can set separate Digital Oceans spaces buckets for either type of media files (and set different overwrite settings)
- the
USE_SPACES environment variable can be set to False to use the regular 'django.core.files.storage.FileSystemStorage' storage backend for all media files (for development environment, if desired)
- You'll see I'm using callable functions to set the file paths and filenames for ImageField and FileField (see Django documentation for FileField)
- You'll also see that I'm prefixing my file names with a random sequence of ascii letters so that calls to any given resource don't have to search through as many objects to find the right object in Spaces (as recommended in Digital Oceans Spaces docs)
settings.py
from pathlib import Path
import environ
env = environ.Env(
# set casting, default value
DEBUG=(bool, False)
)
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Take environment variables from .env file
import os
environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
# Apps added (3rd party)
'storages',
# Apps built and added by me
## This app has user profile model
'profile',
## This app has model that will use non-user-generated audio files
'dialogues'
]
# cuz it will be string in env file, do this to get a boolean
USE_SPACES = env("USE_SPACES") == "True"
if USE_SPACES:
# Hafta make dictionaries to go in STORAGES dictionary cuz this is new way to do it according to django-storages docs (https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html)
## First make base dictionary object cuz I'm gonna just extend it with the options that are actually different
do_region = env('DO_REGION_NAME')
do_endpoint = f'https://{do_region}.digitaloceanspaces.com'
do_secret_key = env('DO_SECRET_ACCESS_KEY')
do_access_key = env('DO_ACCESS_KEY_ID')
user_media_cloud_storage_backend = {
'BACKEND': 'storages.backends.s3.S3Storage',
'OPTIONS': {
'endpoint_url': do_endpoint,
'region_name': do_region,
'bucket_name': env('DO_USER_STORAGE_BUCKET_NAME'),
'location': 'media',
'secret_key': do_secret_key,
'access_key': do_access_key,
'object_parameters': {'CacheControl': 'max-age=86400'},
'default_acl': 'private',
'querystring_auth': True,
'file_overwrite': False,
}
}
app_media_cloud_storage_backend = {
'BACKEND': 'storages.backends.s3.S3Storage',
'OPTIONS': {
'endpoint_url': do_endpoint,
'region_name': do_region,
'bucket_name': env('DO_APP_STORAGE_BUCKET_NAME'),
'location': 'media',
'secret_key': do_secret_key,
'access_key': do_access_key,
'object_parameters': {'CacheControl': 'max-age=86400'},
'default_acl': 'private',
'querystring_auth': True,
'file_overwrite': True,
}
}
## then backend_with_options goes into the big STORAGES dictionary that the django-storages docs ask for (Django 4.2 and greater)
STORAGES = {
'staticfiles': {'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage'},
'default': user_media_cloud_storage_backend, # this 'default' is the same as media
'appmedia': terp_media_cloud_storage_backend,
}
# so staticfiles app will work (in dev and production environments)
STATIC_URL = 'static/'
STATIC_ROOT = '/var/www/example.com/static/'
else:
STORAGES = {
'default': {
'BACKEND': 'django.core.files.storage.FileSystemStorage',
},
'staticfiles': {
'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage',
},
'appmedia': {
'BACKEND': 'django.core.files.storage.FileSystemStorage',
},
}
STATIC_URL = '/static/'
STATIC_ROOT = BASE_DIR / 'staticfiles'
MEDIA_URL = '/media/'
MEDIA_ROOT = BASE_DIR / 'mediafiles'
STATICFILES_DIRS = [
os.path.join(BASE_DIR, 'static/'),
]
profile > models.py
This ImageField in my user profile model will use the storage backend I defined as 'default':
from django.db import models
import random
from string import ascii_letters
# Returns five-char random string to be prepended to filename (so computer can find file faster as suggested here: https://docs.digitalocean.com/products/spaces/concepts/best-practices/)
def genRandomString():
randomPrefix = ''.join(random.choice(ascii_letters) for i in range(5))
return randomPrefix
# with how this function works, there would need to be validation on the forms that handle this file upload so that they only accept jpg and png files
def profile_pic_file_path(instance, filename):
randomString = genRandomString()
jpg_extensions = ('.jpg', '.JPG', '.jpeg', '.JPEG')
png_extensions = ('.png', '.PNG')
if filename.endswith(jpg_extensions):
ext = '.jpg'
elif filename.endswith(png_extensions):
ext = '.png'
# this is just a catch-all in case a non-jpg or -png file slips by validation
else:
ext = filename
return f'profile_pics/{randomString}_userslug_{instance.slug}_pic{ext}'
class UserProfile(models.Model):
# ... bunch of fields ...
profile_pic = models.ImageField(null=True, blank=True, upload_to=profile_pic_file_path)
dialogues > models.py
Things to notice:
- I have to load the
storages module so that I can select for my non-default method of storing files
- here I'm not as worried about dealing with file extension issues because myself, not users, will be the one uploading these files
- my function to set the file path and name can access related objects, which is handy (see Django documentation for FileField).
from django.db import models
from django.core.files.storage import storages
import random
from string import ascii_letters
def select_storage():
return storages['appmedia']
# Returns two-char random string to be prepended to filename (so computer can find file faster as suggested here: https://docs.digitalocean.com/products/spaces/concepts/best-practices/)
def genRandomPrefix():
randomPrefix = ''.join(random.choice(ascii_letters) for i in range(2))
return randomPrefix
def dialogue_audio_file_path(instance, filename):
dialogue_id = instance.original_dialogue.dialogue_slug
randomPrefix = genRandomPrefix()
return f'dialogue_audios/{dialogue_id}/{randomPrefix}-{filename}'
class DialogueAudioFile(models.Model):
audio_file = models.FileField(
storage=select_storage, upload_to=dialogue_audio_file_path
)
original_dialogue = models.ForeignKey(
Dialogue, on_delete=models.CASCADE,
)
