Почему мое промежуточное программное обеспечение Django перенаправляет googlebot вместо того, чтобы напрямую разрешить его?

class BlockDirectAPIAccessMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        protected_paths = ['/api/socialinfo', '/api/crypto-data']

        if any(request.path.startswith(path) for path in protected_paths):
              
            googlebot_user_agents = [
                "Googlebot", "Googlebot-Image", "Googlebot-News", "Googlebot-Video",
                "Storebot-Google", "Google-InspectionTool", "GoogleOther", "Google-Extended"
            ]
            # Check for Googlebot (allow access)
            user_agent = request.META.get('HTTP_USER_AGENT', '').lower() # Convert to lowercase for case-insensitive matching
            if any(ua.lower() in user_agent for ua in googlebot_user_agents):
                return self.get_response(request)  

            # Allow requests from your own domain (optional, but good practice)
            referer = request.META.get('HTTP_REFERER', '')
            if 'mysite.com' in referer:
                return self.get_response(request)

            # Allow local development (if needed)
            if 'localhost' in referer or '127.0.0.1' in referer:
                return self.get_response(request)

            # Block other direct access
            return HttpResponseForbidden('Access denied.')

        # Allow access to non-protected paths
        return self.get_response(request)

Я тестирую предоставление доступа googlebot к API, но получаю 301 ответ

HTTP/1.1 301 Moved Permanently
Date: Thu, 16 May 2024 19:46:45 GMT
Server: Apache/2.4.29 (Ubuntu)
Strict-Transport-Security: max-age=63072000; includeSubdomains
X-Frame-Options: DENY
X-Content-Type-Options: nosniff
Location: /api/socialinfo/?beginRange=0&endRange=19
X-Content-Type-Options: nosniff
Referrer-Policy: same-origin
Cross-Origin-Opener-Policy: same-origin
Content-Type: text/html; charset=utf-8

Вернуться на верх