diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 52d7bf20228541..29582d2b3cb172 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -51,7 +51,7 @@ The :mod:`!csv` module defines the following functions: .. index:: single: universal newlines; csv.reader function -.. function:: reader(csvfile, /, dialect='excel', **fmtparams) +.. function:: reader(csvfile, /, dialect='excel', field_size_limit=None, **fmtparams) Return a :ref:`reader object ` that will process lines from the given *csvfile*. A csvfile must be an iterable of @@ -583,6 +583,10 @@ Reader objects have the following public attributes: The number of lines read from the source iterator. This is not the same as the number of records returned, as records can span multiple lines. +.. attribute:: csvreader.field_size_limit + + The maximum field size allowed by this reader. If :const:`None`, + csv.field_size_limit() is used. DictReader objects have the following public attribute: diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 36fa7e3572a5ac..8e9d107ec10e8a 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -520,6 +520,36 @@ def test_read_bigfield(self): finally: csv.field_size_limit(limit) + def test_override_size_limit(self): + line = ',,,' + reader = csv.reader([line]) + self.assertEqual(reader.field_size_limit, None) + reader = csv.reader([line], field_size_limit=None) + self.assertEqual(reader.field_size_limit, None) + + reader = csv.reader([line], field_size_limit=5) + self.assertEqual(reader.field_size_limit, 5) + reader.field_size_limit = None + self.assertEqual(reader.field_size_limit, None) + reader.field_size_limit = 6 + self.assertEqual(reader.field_size_limit, 6) + del reader.field_size_limit + self.assertEqual(reader.field_size_limit, None) + + with self.assertRaises(ValueError): + csv.reader([line], field_size_limit=-1) + + with self.assertRaises(TypeError): + csv.reader([line], field_size_limit="string") + + line = 'long_field,3,4,5' + reader = csv.reader([line, line], field_size_limit=4) + with self.assertRaises(csv.Error): + list(reader) + reader.field_size_limit = 50 + self.assertEqual(list(reader), [['long_field', '3', '4', '5']]) + + def test_read_linenum(self): r = csv.reader(['line,1', 'line,2', 'line,3']) self.assertEqual(r.line_num, 0) diff --git a/Misc/NEWS.d/next/Library/2019-02-27-16-53-43.bpo-36121.AbiDLw.rst b/Misc/NEWS.d/next/Library/2019-02-27-16-53-43.bpo-36121.AbiDLw.rst new file mode 100644 index 00000000000000..4028e82a26d913 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-02-27-16-53-43.bpo-36121.AbiDLw.rst @@ -0,0 +1,2 @@ +``csv.field_size_limit`` can now be overriden for each csv.Reader +instance. Contributed by Rémi Lapeyre. diff --git a/Modules/_csv.c b/Modules/_csv.c index 484f7bc0f9d2d4..ce9e4a0627cf52 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -139,6 +139,7 @@ typedef struct { Py_ssize_t field_len; /* length of current field */ bool unquoted_field; /* true if no quotes around the current field */ unsigned long line_num; /* Source-file line number */ + long field_limit; /* field limit for current reader */ } ReaderObj; typedef struct { @@ -813,15 +814,19 @@ parse_grow_buff(ReaderObj *self) static int parse_add_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) { - Py_ssize_t field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); + Py_ssize_t field_limit = self->field_limit; + if (field_limit == -1) { + field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); + } if (self->field_len >= field_limit) { PyErr_Format(module_state->error_obj, "field larger than field limit (%zd)", field_limit); return -1; } - if (self->field_len == self->field_size && !parse_grow_buff(self)) + if (self->field_len == self->field_size && !parse_grow_buff(self)) { return -1; + } self->field[self->field_len++] = c; return 0; } @@ -1153,6 +1158,46 @@ static struct PyMemberDef Reader_memberlist[] = { { NULL } }; +static PyObject * +Reader_get_field_size_limit(PyObject *self, void *Py_UNUSED(ignored)) +{ + ReaderObj *reader = (ReaderObj *)self; + if (reader->field_limit == -1) { // -1 is used as a flag for unset value + Py_RETURN_NONE; + } + else { + return PyLong_FromLong(reader->field_limit); + } +} + +static int +Reader_set_field_size_limit(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored)) +{ + ReaderObj *reader = (ReaderObj *)self; + if (arg == NULL || arg == Py_None) { + reader->field_limit = -1; + return 0; + } + else { + long limit = PyLong_AsLong(arg); + if (limit == -1 && PyErr_Occurred()) { + return -1; + } + if (limit <= 0) { + PyErr_Format(PyExc_ValueError, "field_size_limit must greater than 0"); + return -1; + } + reader->field_limit = limit; + return 0; + } +} + +static PyGetSetDef Reader_getset[] = { + { "field_size_limit", Reader_get_field_size_limit, + Reader_set_field_size_limit, PyDoc_STR("field size limit") }, + { NULL }, +}; + #undef R_OFF @@ -1163,6 +1208,7 @@ static PyType_Slot Reader_Type_slots[] = { {Py_tp_iternext, Reader_iternext}, {Py_tp_methods, Reader_methods}, {Py_tp_members, Reader_memberlist}, + {Py_tp_getset, Reader_getset}, {Py_tp_clear, Reader_clear}, {Py_tp_dealloc, Reader_dealloc}, {0, NULL} @@ -1206,9 +1252,11 @@ _csv_reader_impl(PyObject *module, PyObject *iterable, PyObject *dialect, ReaderObj * self = PyObject_GC_New( ReaderObj, module_state->reader_type); + PyObject *_field_size_limit = NULL; - if (!self) + if (!self) { return NULL; + } self->dialect = NULL; self->fields = NULL; @@ -1217,25 +1265,62 @@ _csv_reader_impl(PyObject *module, PyObject *iterable, PyObject *dialect, self->field_size = 0; self->line_num = 0; + _field_size_limit = PyUnicode_FromString("field_size_limit"); + if (_field_size_limit == NULL) { + goto fail; + } + PyObject *field_size_limit = NULL; + if (fmtparams != NULL) { + field_size_limit = PyDict_GetItemWithError(fmtparams, _field_size_limit); + if (PyErr_Occurred()) { + goto fail; + } + } + if (field_size_limit == NULL) { + self->field_limit = -1; + } else if (field_size_limit == Py_None) { + self->field_limit = -1; + if (PyDict_DelItem(fmtparams, _field_size_limit) < 0) { + goto fail; + } + } + else { + long limit = PyLong_AsLong(field_size_limit); + if (PyErr_Occurred()) { + goto fail; + } + if (limit <= 0) { + PyErr_Format(PyExc_ValueError, "field_size_limit must greater than 0"); + goto fail; + } + if (PyDict_DelItem(fmtparams, _field_size_limit) < 0) { + goto fail; + } + self->field_limit = limit; + } + Py_CLEAR(_field_size_limit); + if (parse_reset(self) < 0) { - Py_DECREF(self); - return NULL; + goto fail; } self->input_iter = PyObject_GetIter(iterable); if (self->input_iter == NULL) { - Py_DECREF(self); - return NULL; + goto fail; } self->dialect = (DialectObj *)_call_dialect(module_state, dialect, fmtparams); if (self->dialect == NULL) { - Py_DECREF(self); - return NULL; + goto fail; } PyObject_GC_Track(self); return (PyObject *)self; + +fail: + Py_XDECREF(_field_size_limit); + Py_DECREF(self); + return NULL; } /*