gh-80302: Override field_size_limit per csv.reader object

Doc/library/csv.rst

@@ -51,7 +51,7 @@ The :mod:`!csv` module defines the following functions:
 .. index::
    single: universal newlines; csv.reader function
 
-.. function:: reader(csvfile, /, dialect='excel', **fmtparams)
+.. function:: reader(csvfile, /, dialect='excel', field_size_limit=None, **fmtparams)
 
    Return a :ref:`reader object <reader-objects>` that will process
    lines from the given *csvfile*.  A csvfile must be an iterable of
@@ -583,6 +583,10 @@ Reader objects have the following public attributes:
    The number of lines read from the source iterator. This is not the same as the
    number of records returned, as records can span multiple lines.
 
+.. attribute:: csvreader.field_size_limit
+
+   The maximum field size allowed by this reader. If :const:`None`,
+   csv.field_size_limit() is used.
 
 DictReader objects have the following public attribute:
 

Lib/test/test_csv.py

@@ -520,6 +520,36 @@ def test_read_bigfield(self):
         finally:
             csv.field_size_limit(limit)
 
+    def test_override_size_limit(self):
+        line = ',,,'
+        reader = csv.reader([line])
+        self.assertEqual(reader.field_size_limit, None)
+        reader = csv.reader([line], field_size_limit=None)
+        self.assertEqual(reader.field_size_limit, None)
+
+        reader = csv.reader([line], field_size_limit=5)
+        self.assertEqual(reader.field_size_limit, 5)
+        reader.field_size_limit = None
+        self.assertEqual(reader.field_size_limit, None)
+        reader.field_size_limit = 6
+        self.assertEqual(reader.field_size_limit, 6)
+        del reader.field_size_limit
+        self.assertEqual(reader.field_size_limit, None)
+
+        with self.assertRaises(ValueError):
+            csv.reader([line], field_size_limit=-1)
+
+        with self.assertRaises(TypeError):
+            csv.reader([line], field_size_limit="string")
+
+        line = 'long_field,3,4,5'
+        reader = csv.reader([line, line], field_size_limit=4)
+        with self.assertRaises(csv.Error):
+            list(reader)
+        reader.field_size_limit = 50
+        self.assertEqual(list(reader), [['long_field', '3', '4', '5']])
+
+
     def test_read_linenum(self):
         r = csv.reader(['line,1', 'line,2', 'line,3'])
         self.assertEqual(r.line_num, 0)

Misc/NEWS.d/next/Library/2019-02-27-16-53-43.bpo-36121.AbiDLw.rst

@@ -0,0 +1,2 @@
+``csv.field_size_limit`` can now be overriden for each csv.Reader
+instance. Contributed by Rémi Lapeyre.

Modules/_csv.c

@@ -139,6 +139,7 @@ typedef struct {
     Py_ssize_t field_len;       /* length of current field */
     bool unquoted_field;        /* true if no quotes around the current field */
     unsigned long line_num;     /* Source-file line number */
+    long field_limit;           /* field limit for current reader */
 } ReaderObj;
 
 typedef struct {
@@ -813,15 +814,19 @@ parse_grow_buff(ReaderObj *self)
 static int
 parse_add_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c)
 {
-    Py_ssize_t field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit);
+    Py_ssize_t field_limit = self->field_limit;
+    if (field_limit == -1) {
+        field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit);
+    }
     if (self->field_len >= field_limit) {
         PyErr_Format(module_state->error_obj,
                      "field larger than field limit (%zd)",
                      field_limit);
         return -1;
     }
-    if (self->field_len == self->field_size && !parse_grow_buff(self))
+    if (self->field_len == self->field_size && !parse_grow_buff(self)) {
         return -1;
+    }
     self->field[self->field_len++] = c;
     return 0;
 }
@@ -1153,6 +1158,46 @@ static struct PyMemberDef Reader_memberlist[] = {
     { NULL }
 };
 
+static PyObject *
+Reader_get_field_size_limit(PyObject *self, void *Py_UNUSED(ignored))
+{
+    ReaderObj *reader = (ReaderObj *)self;
+    if (reader->field_limit == -1) {  // -1 is used as a flag for unset value
+        Py_RETURN_NONE;
+    }
+    else {
+        return PyLong_FromLong(reader->field_limit);
+    }
+}
+
+static int
+Reader_set_field_size_limit(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored))
+{
+    ReaderObj *reader = (ReaderObj *)self;
+    if (arg == NULL || arg == Py_None) {
+        reader->field_limit = -1;
+        return 0;
+    }
+    else {
+        long limit = PyLong_AsLong(arg);
+        if (limit == -1 && PyErr_Occurred()) {
+            return -1;
+        }
+        if (limit <= 0) {
+            PyErr_Format(PyExc_ValueError, "field_size_limit must greater than 0");
+            return -1;
+        }
+        reader->field_limit = limit;
+        return 0;
+    }
+}
+
+static PyGetSetDef Reader_getset[] = {
+    { "field_size_limit", Reader_get_field_size_limit,
+        Reader_set_field_size_limit, PyDoc_STR("field size limit") },
+    { NULL },
+};
+
 #undef R_OFF
 
 
@@ -1163,6 +1208,7 @@ static PyType_Slot Reader_Type_slots[] = {
     {Py_tp_iternext, Reader_iternext},
     {Py_tp_methods, Reader_methods},
     {Py_tp_members, Reader_memberlist},
+    {Py_tp_getset, Reader_getset},
     {Py_tp_clear, Reader_clear},
     {Py_tp_dealloc, Reader_dealloc},
     {0, NULL}
@@ -1206,9 +1252,11 @@ _csv_reader_impl(PyObject *module, PyObject *iterable, PyObject *dialect,
     ReaderObj * self = PyObject_GC_New(
         ReaderObj,
         module_state->reader_type);
+    PyObject *_field_size_limit = NULL;
 
-    if (!self)
+    if (!self) {
         return NULL;
+    }
 
     self->dialect = NULL;
     self->fields = NULL;
@@ -1217,25 +1265,62 @@ _csv_reader_impl(PyObject *module, PyObject *iterable, PyObject *dialect,
     self->field_size = 0;
     self->line_num = 0;
 
+    _field_size_limit = PyUnicode_FromString("field_size_limit");
+    if (_field_size_limit == NULL) {
+        goto fail;
+    }
+    PyObject *field_size_limit = NULL;
+    if (fmtparams != NULL) {
+        field_size_limit = PyDict_GetItemWithError(fmtparams, _field_size_limit);
+        if (PyErr_Occurred()) {
+            goto fail;
+        }
+    }
+    if (field_size_limit == NULL) {
+        self->field_limit = -1;
+    } else if (field_size_limit == Py_None) {
+        self->field_limit = -1;
+        if (PyDict_DelItem(fmtparams, _field_size_limit) < 0) {
+            goto fail;
+        }
+    }
+    else {
+        long limit = PyLong_AsLong(field_size_limit);
+        if (PyErr_Occurred()) {
+            goto fail;
+        }
+        if (limit <= 0) {
+            PyErr_Format(PyExc_ValueError, "field_size_limit must greater than 0");
+            goto fail;
+        }
+        if (PyDict_DelItem(fmtparams, _field_size_limit) < 0) {
+            goto fail;
+        }
+        self->field_limit = limit;
+    }
+    Py_CLEAR(_field_size_limit);
+
     if (parse_reset(self) < 0) {
-        Py_DECREF(self);
-        return NULL;
+        goto fail;
     }
 
     self->input_iter = PyObject_GetIter(iterable);
     if (self->input_iter == NULL) {
-        Py_DECREF(self);
-        return NULL;
+        goto fail;
     }
     self->dialect = (DialectObj *)_call_dialect(module_state, dialect,
                                                 fmtparams);
     if (self->dialect == NULL) {
-        Py_DECREF(self);
-        return NULL;
+        goto fail;
     }
 
     PyObject_GC_Track(self);
     return (PyObject *)self;
+
+fail:
+    Py_XDECREF(_field_size_limit);
+    Py_DECREF(self);
+    return NULL;
 }
 
 /*