Django project - Redis connection "kombu.exceptions.OperationalError: invalid username-password pair or user is disabled."
Hello I'm trying to deploy my django app on railway. This app is using Celery on Redis. When I deploy the project the logs indicate: [enter image description here][1] As we see the initital connection is to redis is successful. However I as soons as I trigger the task (from my tasks.py file): the connection is lost: [enter image description here][2]
The error indicates a "invalid username-password pair or user is disabled.". Nevertheless, I don't understand because my REDIS_URL is the same used for the initial connection when the project is deployed. In my logs I get extra info: [enter image description here][3] [1]: https://i.sstatic.net/3yAjMwlD.png [2]: https://i.sstatic.net/Cb0cY3Lr.png [3]: https://i.sstatic.net/XWwOvWdc.png
tasks.py
# mobile_subscriptions/tasks.py
from celery import shared_task
import time
import logging
logger = logging.getLogger(__name__)
@shared_task
def debug_task():
try:
logger.info('Demo task started!')
time.sleep(10)
logger.info('Demo task completed!')
return 'Demo task completed!'
except Exception as e:
logger.error(f"Unexpected error in debug task: {e}")
raise
celery.py:
# comparaplan/celery.py
import os
from celery import Celery
from dotenv import load_dotenv
load_dotenv()
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'comparaplan.settings')
celery_app = Celery('comparaplan')
celery_app.config_from_object('django.conf:settings', namespace='CELERY')
celery_app.autodiscover_tasks()
celery_app.conf.task_routes = {
'mobile_subscriptions.tasks.debug_task': {'queue': 'cloud_queue'},
}
celery_app.conf.update(
result_expires=60,
)
settings.py
"""
Django settings for comparaplan project.
"""
import os
import sys
import time
from pathlib import Path
from dotenv import load_dotenv
import dj_database_url
import logging
import importlib
from django.conf import settings
from redis import Redis
from redis.connection import ConnectionPool, ConnectionError
from redis.exceptions import TimeoutError, RedisError
from .redis_utils import get_redis_pool, get_redis_client
# Load environment variables
load_dotenv()
# API Keys and Credentials
OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY', '')
EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER')
EMAIL_PASSWORD = os.environ.get('EMAIL_PASSWORD')
DB_PASSWORD = os.environ.get('DB_PASSWORD')
# Redis Configuration
REDIS_HOST = os.environ.get('REDISHOST')
REDIS_PORT = os.environ.get('REDISPORT')
REDIS_USER = os.environ.get('REDISUSER')
REDIS_PASSWORD = os.environ.get('REDISPASSWORD')
REDIS_PUBLIC_URL = os.environ.get('REDIS_PUBLIC_URL')
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
# Railway Configuration
RAILWAY_RUN_UID = os.environ.get('RAILWAY_RUN_UID')
RAILWAY_RUN_AS_ROOT = os.environ.get('RAILWAY_RUN_AS_ROOT')
# Constants for retry
MAX_RETRIES = 5
RETRY_DELAY = 5
CONNECT_TIMEOUT = 30
SOCKET_TIMEOUT = 30
CONNECTION_POOL_MAX_CONNECTIONS = 200
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# SECURITY WARNING: don't run with debug turned on in production!
# DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true'
DEBUG=False
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY')
if not SECRET_KEY and DEBUG:
SECRET_KEY = 'django-insecure-dev-key-for-local-development-only'
ALLOWED_HOSTS = ['comparaplan-production.up.railway.app', '127.0.0.1']
CSRF_TRUSTED_ORIGINS = ['https://comparaplan-production.up.railway.app']
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django_celery_beat',
'members',
'contact',
'prospectAlert',
'mobile_subscriptions',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'whitenoise.middleware.WhiteNoiseMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'comparaplan.urls'
WSGI_APPLICATION = 'comparaplan.wsgi.application'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [BASE_DIR / 'templates'],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
# Database Configuration
DATABASE_URL = os.environ.get('DATABASE_URL')
if DATABASE_URL:
DATABASES = {
'default': dj_database_url.parse(DATABASE_URL)
}
else:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': os.environ.get('DB_NAME', 'railway'),
'USER': os.environ.get('DB_USER', 'postgres'),
'PASSWORD': os.environ.get('DB_PASSWORD'),
'HOST': os.environ.get('DB_HOST', 'localhost'),
'PORT': os.environ.get('DB_PORT', '5432'),
}
}
# Password validation
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
STATICFILES_DIRS = [
os.path.join(BASE_DIR, "static")
]
STATIC_URL = 'static/'
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
# Default primary key field type
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# Celery Configuration
CELERY_BROKER_URL = REDIS_URL
CELERY_RESULT_BACKEND = REDIS_URL
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'UTC'
CELERY_BROKER_TRANSPORT_OPTIONS = {
'socket_timeout': CONNECT_TIMEOUT,
'socket_connect_timeout': CONNECT_TIMEOUT,
'retry_on_timeout': True,
'max_retries': MAX_RETRIES,
'connection_attempts': MAX_RETRIES,
'interval_start': RETRY_DELAY,
'interval_max': RETRY_DELAY * 3,
'backoff_rate': 2,
}
CELERY_BROKER_CONNECTION_RETRY=True
CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
CELERY_TASK_TRACK_STARTED = True
CELERY_BROKER_CONNECTION_MAX_RETRIES=10
CELERY_TASK_TIME_LIMIT = 30 * 60
# Cache Configuration
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": os.environ.get('REDIS_URL'), # Use REDIS_URL from env
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"CONNECTION_POOL_KWARGS": None, # Use the shared pool
"PICKLE_VERSION": -1,
}
}
}
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
# Email Configuration
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_HOST = 'smtp.gmail.com'
EMAIL_PORT = 587
EMAIL_USE_TLS = True
EMAIL_HOST_USER = EMAIL_HOST_USER
EMAIL_HOST_PASSWORD = EMAIL_PASSWORD
DEFAULT_FROM_EMAIL = EMAIL_HOST_USER
# Logging Configuration
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'verbose',
},
},
'formatters': {
'verbose': {
'format': '[%(levelname)s] %(asctime)s %(module)s - %(message)s',
},
},
'loggers': {
'celery': {
'handlers': ['console'],
'level': 'INFO',
},
'django': {
'handlers': ['console'],
'level': 'INFO',
},
'comparaplan': {
'handlers': ['console'],
'level': 'INFO',
},
'redis': {
'handlers': ['console'],
'level': 'INFO',
},
},
}
# Base logging configuration
logging.basicConfig(
level=logging.INFO,
format='[%(levelname)s] %(asctime)s %(module)s - %(message)s',
handlers=[logging.StreamHandler(sys.stdout)]
)
# Get logger for this module
logger = logging.getLogger(__name__)
def get_redis_connection(url=None):
"""
Establishes a robust connection to Redis with retry logic and connection pooling.
"""
if url is None:
url = os.environ.get('REDIS_URL')
if not url:
logger.critical("SETTINGS: REDIS_URL not found in environment variables!")
return None
logger.info(f"SETTINGS.PY {os.environ.get('REDIS_URL')}")
for attempt in range(MAX_RETRIES):
try:
logger.info(f"SETTINGS: Attempting Redis connection (attempt {attempt + 1}/{MAX_RETRIES})...")
connection_pool = ConnectionPool.from_url(
url,
max_connections=CONNECTION_POOL_MAX_CONNECTIONS,
decode_responses=True,
socket_connect_timeout=CONNECT_TIMEOUT,
socket_timeout=SOCKET_TIMEOUT,
)
client = Redis(connection_pool=connection_pool)
client.ping()
logger.info("SETTINGS: Redis connection successful.")
return client
except (ConnectionError, TimeoutError, RedisError) as e:
logger.error(f"SETTINGS: Redis connection failed: {e}")
if attempt < MAX_RETRIES - 1:
logger.info(f"SETTINGS: Retrying in {RETRY_DELAY} seconds...")
time.sleep(RETRY_DELAY)
else:
logger.critical("SETTINGS: Max retries reached. Redis connection failed.")
return None
def test_redis_connection():
"""Test Redis connection on Django startup."""
redis_client = get_redis_connection()
if redis_client:
logger.info("SETTINGS: Redis connection test successful on startup.")
else:
logger.error("SETTINGS: Redis connection test failed on startup.")
# Do NOT raise an exception here. Django should start, and the
# application should attempt to reconnect as needed.
test_redis_connection()
Did anyone already encounter this issue?