Apply all patches up to CVE-2023-36053

2023-07-24 15:46:31 -07:00 · 2023-07-24 15:46:31 -07:00 · c9de5aa27b
commit c9de5aa27b
parent 9d67bfadf8
13 changed files with 186 additions and 19 deletions
--- a/django/conf/global_settings.py
+++ b/django/conf/global_settings.py
@ -295,6 +295,10 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 2621440  # i.e. 2.5 MB
 # SuspiciousOperation (TooManyFieldsSent) is raised.
 DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000

+# Maximum number of files encoded in a multipart upload that will be read
+# before a SuspiciousOperation (TooManyFilesSent) is raised.
+DATA_UPLOAD_MAX_NUMBER_FILES = 100
+
 # Directory in which upload streamed files will be temporarily saved. A value of
 # `None` will make Django use the operating system's default temporary directory
 # (i.e. "/tmp" on *nix systems).
--- a/django/core/exceptions.py
+++ b/django/core/exceptions.py
@ -61,6 +61,14 @@ class TooManyFieldsSent(SuspiciousOperation):
    pass


+class TooManyFilesSent(SuspiciousOperation):
+    """
+    The number of fields in a GET or POST request exceeded
+    settings.DATA_UPLOAD_MAX_NUMBER_FILES.
+    """
+    pass
+
+
 class RequestDataTooBig(SuspiciousOperation):
    """
    The size of the request (excluding any file uploads) exceeded
--- a/django/core/handlers/exception.py
+++ b/django/core/handlers/exception.py
@ -9,7 +9,7 @@ from django.conf import settings
 from django.core import signals
 from django.core.exceptions import (
    PermissionDenied, RequestDataTooBig, SuspiciousOperation,
-    TooManyFieldsSent,
+    TooManyFieldsSent, TooManyFilesSent,
 )
 from django.http import Http404
 from django.http.multipartparser import MultiPartParserError
@ -68,7 +68,7 @@ def response_for_exception(request, exc):
        response = get_exception_response(request, get_resolver(get_urlconf()), 400, exc)

    elif isinstance(exc, SuspiciousOperation):
-        if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent)):
+        if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent, TooManyFilesSent)):
            # POST data can't be accessed again, otherwise the original
            # exception would be raised.
            request._mark_post_parse_error()
--- a/django/core/validators.py
+++ b/django/core/validators.py
@ -105,6 +105,8 @@ class URLValidator(RegexValidator):
        r'\Z', re.IGNORECASE)
    message = _('Enter a valid URL.')
    schemes = ['http', 'https', 'ftp', 'ftps']
+    unsafe_chars = frozenset('\t\r\n')
+    max_length = 2048

    def __init__(self, schemes=None, **kwargs):
        super(URLValidator, self).__init__(**kwargs)
@ -193,8 +195,14 @@ class EmailValidator(object):

    def __call__(self, value):
        value = force_text(value)
+        if not isinstance(value, str) or len(value) > self.max_length:
+            raise ValidationError(self.message, code=self.code, params={'value': value})
+        if self.unsafe_chars.intersection(value):
+            raise ValidationError(self.message, code=self.code)

-        if not value or '@' not in value:
+        # The maximum length of an email is 320 characters per RFC 3696
+        # section 3.
+        if not value or '@' not in value or len(value) > 320:
            raise ValidationError(self.message, code=self.code)

        user_part, domain_part = value.rsplit('@', 1)
--- a/django/forms/fields.py
+++ b/django/forms/fields.py
@ -531,6 +531,12 @@ class EmailField(CharField):
    widget = EmailInput
    default_validators = [validators.validate_email]

+    def __init__(self, *args, **kwargs):
+        # The default maximum length of an email is 320 characters per RFC 3696
+        # section 3.
+        kwargs.setdefault("max_length", 320)
+        super(EmailField, self).__init__(*args, strip=True, **kwargs)
+
    def clean(self, value):
        value = self.to_python(value).strip()
        return super(EmailField, self).clean(value)
--- a/django/http/multipartparser.py
+++ b/django/http/multipartparser.py
@ -13,7 +13,7 @@ import sys

 from django.conf import settings
 from django.core.exceptions import (
-    RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent,
+    RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent, TooManyFilesSent,
 )
 from django.core.files.uploadhandler import (
    SkipFile, StopFutureHandlers, StopUpload,
@ -40,6 +40,7 @@ class InputStreamExhausted(Exception):
 RAW = "raw"
 FILE = "file"
 FIELD = "field"
+FIELD_TYPES = frozenset([FIELD, RAW])

 _BASE64_DECODE_ERROR = TypeError if six.PY2 else binascii.Error

@ -107,6 +108,22 @@ class MultiPartParser(object):
        self._upload_handlers = upload_handlers

    def parse(self):
+        # Call the actual parse routine and close all open files in case of
+        # errors. This is needed because if exceptions are thrown the
+        # MultiPartParser will not be garbage collected immediately and
+        # resources would be kept alive. This is only needed for errors because
+        # the Request object closes all uploaded files at the end of the
+        # request.
+        try:
+            return self._parse()
+        except Exception:
+            if hasattr(self, "_files"):
+                for _, files in self._files.lists():
+                    for fileobj in files:
+                        fileobj.close()
+            raise
+
+    def _parse(self):
        """
        Parse the POST data and break it into a FILES MultiValueDict and a POST
        MultiValueDict.
@ -151,6 +168,8 @@ class MultiPartParser(object):
        num_bytes_read = 0
        # To count the number of keys in the request.
        num_post_keys = 0
+        # To count the number of files in the request.
+        num_files = 0
        # To limit the amount of data read from the request.
        read_size = None

@ -163,6 +182,20 @@ class MultiPartParser(object):
                    self.handle_file_complete(old_field_name, counters)
                    old_field_name = None

+                if (
+                    item_type in FIELD_TYPES and
+                    settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
+                ):
+                    # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
+                    num_post_keys += 1
+                    # 2 accounts for empty raw fields before and after the
+                    # last boundary.
+                    if settings.DATA_UPLOAD_MAX_NUMBER_FIELDS + 2 < num_post_keys:
+                        raise TooManyFieldsSent(
+                            "The number of GET/POST parameters exceeded "
+                            "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS."
+                        )
+
                try:
                    disposition = meta_data['content-disposition'][1]
                    field_name = disposition['name'].strip()
@ -210,6 +243,16 @@ class MultiPartParser(object):
                    self._post.appendlist(field_name,
                                          force_text(data, encoding, errors='replace'))
                elif item_type == FILE:
+                    # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES.
+                    num_files += 1
+                    if (
+                        settings.DATA_UPLOAD_MAX_NUMBER_FILES is not None and
+                        num_files > settings.DATA_UPLOAD_MAX_NUMBER_FILES
+                    ):
+                        raise TooManyFilesSent(
+                            "The number of files exceeded "
+                            "settings.DATA_UPLOAD_MAX_NUMBER_FILES."
+                        )
                    # This is a file, use the handler...
                    file_name = disposition.get('filename')
                    if file_name:
@ -275,8 +318,13 @@ class MultiPartParser(object):
                        # Handle file upload completions on next iteration.
                        old_field_name = field_name
                else:
-                    # If this is neither a FIELD or a FILE, just exhaust the stream.
-                    exhaust(stream)
+                    # If this is neither a FIELD nor a FILE, exhaust the field
+                    # stream. Note: There could be an error here at some point,
+                    # but there will be at least two RAW types (before and
+                    # after the other boundaries). This branch is usually not
+                    # reached at all, because a missing content-disposition
+                    # header will skip the whole boundary.
+                    exhaust(field_stream)
        except StopUpload as e:
            self._close_files()
            if not e.connection_reset:
--- a/django/http/request.py
+++ b/django/http/request.py
@ -12,7 +12,9 @@ from django.core.exceptions import (
    DisallowedHost, ImproperlyConfigured, RequestDataTooBig,
 )
 from django.core.files import uploadhandler
-from django.http.multipartparser import MultiPartParser, MultiPartParserError
+from django.http.multipartparser import (
+    MultiPartParser, MultiPartParserError, TooManyFilesSent,
+)
 from django.utils import six
 from django.utils.datastructures import ImmutableList, MultiValueDict
 from django.utils.encoding import (
@ -297,7 +299,7 @@ class HttpRequest(object):
                data = self
            try:
                self._post, self._files = self.parse_file_upload(self.META, data)
-            except MultiPartParserError:
+            except (MultiPartParserError, TooManyFilesSent):
                # An error occurred while parsing POST data. Since when
                # formatting the error the request handler might access
                # self.POST, set self._post and self._file to prevent
--- a/tests/forms_tests/field_tests/test_emailfield.py
+++ b/tests/forms_tests/field_tests/test_emailfield.py
@ -11,7 +11,10 @@ class EmailFieldTest(FormFieldAssertionsMixin, SimpleTestCase):

    def test_emailfield_1(self):
        f = EmailField()
-        self.assertWidgetRendersTo(f, '<input type="email" name="f" id="id_f" required />')
+        self.assertEqual(f.max_length, 320)
+        self.assertWidgetRendersTo(
+            f, '<input type="email" name="f" id="id_f" maxlength="320" required>'
+        )
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
            f.clean('')
        with self.assertRaisesMessage(ValidationError, "'This field is required.'"):
--- a/tests/forms_tests/tests/test_forms.py
+++ b/tests/forms_tests/tests/test_forms.py
@ -416,11 +416,18 @@ class FormsTestCase(SimpleTestCase):
            get_spam = BooleanField()

        f = SignupForm(auto_id=False)
-        self.assertHTMLEqual(str(f['email']), '<input type="email" name="email" required />')
+        self.assertHTMLEqual(
+            str(f["email"]),
+            '<input type="email" name="email" maxlength="320" required />',
+        )
        self.assertHTMLEqual(str(f['get_spam']), '<input type="checkbox" name="get_spam" required />')

        f = SignupForm({'email': 'test@example.com', 'get_spam': True}, auto_id=False)
-        self.assertHTMLEqual(str(f['email']), '<input type="email" name="email" value="test@example.com" required />')
+        self.assertHTMLEqual(
+            str(f["email"]),
+            '<input type="email" name="email" maxlength="320" value="test@example.com" '
+            "required />",
+        )
        self.assertHTMLEqual(
            str(f['get_spam']),
            '<input checked="checked" type="checkbox" name="get_spam" required />',
@ -2649,7 +2656,7 @@ Good luck picking a username that doesn&#39;t already exist.</p>
 <option value="2">Yes</option>
 <option value="3">No</option>
 </select></li>
-<li><label for="id_email">Email:</label> <input type="email" name="email" id="id_email" /></li>
+<li><label for="id_email">Email:</label> <input type="email" name="email" id="id_email" maxlength="320" /></li>
 <li class="required error"><ul class="errorlist"><li>This field is required.</li></ul>
 <label class="required" for="id_age">Age:</label> <input type="number" name="age" id="id_age" required /></li>"""
        )
@ -2665,7 +2672,7 @@ Good luck picking a username that doesn&#39;t already exist.</p>
 <option value="2">Yes</option>
 <option value="3">No</option>
 </select></p>
-<p><label for="id_email">Email:</label> <input type="email" name="email" id="id_email" /></p>
+<p><label for="id_email">Email:</label> <input type="email" name="email" id="id_email" maxlength="320" /></p>
 <ul class="errorlist"><li>This field is required.</li></ul>
 <p class="required error"><label class="required" for="id_age">Age:</label>
 <input type="number" name="age" id="id_age" required /></p>"""
@ -2684,7 +2691,7 @@ Good luck picking a username that doesn&#39;t already exist.</p>
 <option value="3">No</option>
 </select></td></tr>
 <tr><th><label for="id_email">Email:</label></th><td>
-<input type="email" name="email" id="id_email" /></td></tr>
+<input type="email" name="email" id="id_email" maxlength="320" /></td></tr>
 <tr class="required error"><th><label class="required" for="id_age">Age:</label></th>
 <td><ul class="errorlist"><li>This field is required.</li></ul>
 <input type="number" name="age" id="id_age" required /></td></tr>"""
@ -3285,7 +3292,7 @@ Good luck picking a username that doesn&#39;t already exist.</p>
        f = CommentForm(data, auto_id=False, error_class=DivErrorList)
        self.assertHTMLEqual(f.as_p(), """<p>Name: <input type="text" name="name" maxlength="50" /></p>
 <div class="errorlist"><div class="error">Enter a valid email address.</div></div>
-<p>Email: <input type="email" name="email" value="invalid" required /></p>
+<p>Email: <input type="email" name="email" value="invalid" maxlength="320" required /></p>
 <div class="errorlist"><div class="error">This field is required.</div></div>
 <p>Comment: <input type="text" name="comment" required /></p>""")

--- a/tests/handlers/test_exception.py
+++ b/tests/handlers/test_exception.py
@ -1,6 +1,8 @@
 from django.core.handlers.wsgi import WSGIHandler
 from django.test import SimpleTestCase, override_settings
-from django.test.client import FakePayload
+from django.test.client import (
+    BOUNDARY, MULTIPART_CONTENT, FakePayload, encode_multipart,
+)


@override_settings(ROOT_URLCONF='handlers.urls')
@ -27,3 +29,27 @@ class ExceptionHandlerTests(SimpleTestCase):
    def test_data_upload_max_number_fields_exceeded(self):
        response = WSGIHandler()(self.get_suspicious_environ(), lambda *a, **k: None)
        self.assertEqual(response.status_code, 400)
+
+    @override_settings(DATA_UPLOAD_MAX_NUMBER_FILES=2)
+    def test_data_upload_max_number_files_exceeded(self):
+        payload = FakePayload(
+            encode_multipart(
+                BOUNDARY,
+                {
+                    "a.txt": "Hello World!",
+                    "b.txt": "Hello Django!",
+                    "c.txt": "Hello Python!",
+                },
+            )
+        )
+        environ = {
+            "REQUEST_METHOD": "POST",
+            "CONTENT_TYPE": MULTIPART_CONTENT,
+            "CONTENT_LENGTH": len(payload),
+            "wsgi.input": payload,
+            "SERVER_NAME": "test",
+            "SERVER_PORT": "8000",
+        }
+
+        response = WSGIHandler()(environ, lambda *a, **k: None)
+        self.assertEqual(response.status_code, 400)
--- a/tests/requests/test_data_upload_settings.py
+++ b/tests/requests/test_data_upload_settings.py
@ -1,11 +1,14 @@
 from io import BytesIO

-from django.core.exceptions import RequestDataTooBig, TooManyFieldsSent
+from django.core.exceptions import (
+    RequestDataTooBig, TooManyFieldsSent, TooManyFilesSent,
+)
 from django.core.handlers.wsgi import WSGIRequest
 from django.test import SimpleTestCase
 from django.test.client import FakePayload

 TOO_MANY_FIELDS_MSG = 'The number of GET/POST parameters exceeded settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.'
+TOO_MANY_FILES_MSG = 'The number of files exceeded settings.DATA_UPLOAD_MAX_NUMBER_FILES.'
 TOO_MUCH_DATA_MSG = 'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.'


@ -166,6 +169,52 @@ class DataUploadMaxNumberOfFieldsMultipartPost(SimpleTestCase):
            self.request._load_post_and_files()


+class DataUploadMaxNumberOfFilesMultipartPost(SimpleTestCase):
+    def setUp(self):
+        payload = FakePayload(
+            "\r\n".join(
+                [
+                    "--boundary",
+                    (
+                        'Content-Disposition: form-data; name="name1"; '
+                        'filename="name1.txt"'
+                    ),
+                    "",
+                    "value1",
+                    "--boundary",
+                    (
+                        'Content-Disposition: form-data; name="name2"; '
+                        'filename="name2.txt"'
+                    ),
+                    "",
+                    "value2",
+                    "--boundary--",
+                ]
+            )
+        )
+        self.request = WSGIRequest(
+            {
+                "REQUEST_METHOD": "POST",
+                "CONTENT_TYPE": "multipart/form-data; boundary=boundary",
+                "CONTENT_LENGTH": len(payload),
+                "wsgi.input": payload,
+            }
+        )
+
+    def test_number_exceeded(self):
+        with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=1):
+            with self.assertRaisesMessage(TooManyFilesSent, TOO_MANY_FILES_MSG):
+                self.request._load_post_and_files()
+
+    def test_number_not_exceeded(self):
+        with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=2):
+            self.request._load_post_and_files()
+
+    def test_no_limit(self):
+        with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=None):
+            self.request._load_post_and_files()
+
+
 class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase):
    def setUp(self):
        payload = FakePayload("\r\n".join(['a=1&a=2;a=3', '']))
--- a/tests/requests/tests.py
+++ b/tests/requests/tests.py
@ -236,12 +236,12 @@ class RequestsTests(SimpleTestCase):
    def test_far_expiration(self):
        "Cookie will expire when an distant expiration time is provided"
        response = HttpResponse()
-        response.set_cookie('datetime', expires=datetime(2028, 1, 1, 4, 5, 6))
+        response.set_cookie('datetime', expires=datetime(2038, 1, 1, 4, 5, 6))
        datetime_cookie = response.cookies['datetime']
        self.assertIn(
            datetime_cookie['expires'],
            # Slight time dependency; refs #23450
-            ('Sat, 01-Jan-2028 04:05:06 GMT', 'Sat, 01-Jan-2028 04:05:07 GMT')
+            ('Fri, 01-Jan-2038 04:05:06 GMT', 'Fri, 01-Jan-2038 04:05:07 GMT')
        )

    def test_max_age_expiration(self):
--- a/tests/validators/tests.py
+++ b/tests/validators/tests.py
@ -54,6 +54,7 @@ TEST_DATA = [

    (validate_email, 'example@atm.%s' % ('a' * 64), ValidationError),
    (validate_email, 'example@%s.atm.%s' % ('b' * 64, 'a' * 63), ValidationError),
+    (validate_email, "example@%scom" % (("a" * 63 + ".") * 100), ValidationError),
    (validate_email, None, ValidationError),
    (validate_email, '', ValidationError),
    (validate_email, 'abc', ValidationError),
@ -213,6 +214,11 @@ TEST_DATA = [
    (URLValidator(EXTENDED_SCHEMES), 'git+ssh://git@github.com/example/hg-git.git', None),

    (URLValidator(EXTENDED_SCHEMES), 'git://-invalid.com', ValidationError),
+    (
+        URLValidator(),
+        "http://example." + ("a" * 63 + ".") * 1000 + "com",
+        ValidationError,
+    ),
    # Trailing newlines not accepted
    (URLValidator(), 'http://www.djangoproject.com/\n', ValidationError),
    (URLValidator(), 'http://[::ffff:192.9.5.5]\n', ValidationError),