From a524fd33bc569116a379940007c8d9a1c95419fe Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Fri, 9 Aug 2019 11:00:40 -0500 Subject: [PATCH] Merge enhancements from pysqlite3, errorcode/name and blob i/o. --- setup.py | 2 +- src/blob.c | 644 +++++++++++++++++++++++++++++++++++++++++++++++ src/blob.h | 26 ++ src/connection.c | 90 ++++++- src/connection.h | 3 +- src/module.c | 99 +++++++- src/module.h | 2 + src/util.c | 63 ++++- test/backup.py | 4 +- test/dbapi.py | 259 ++++++++++++++++++- 10 files changed, 1168 insertions(+), 24 deletions(-) create mode 100644 src/blob.c create mode 100644 src/blob.h diff --git a/setup.py b/setup.py index 23b6621..0e294a2 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ VERSION = '0.1.1' sources = [os.path.join('src', source) for source in ["module.c", "connection.c", "cursor.c", "cache.c", "microprotocols.c", "prepare_protocol.c", - "statement.c", "util.c", "row.c"]] + "statement.c", "util.c", "row.c", "blob.c"]] # define packages packages = [PACKAGE_NAME] diff --git a/src/blob.c b/src/blob.c new file mode 100644 index 0000000..3112132 --- /dev/null +++ b/src/blob.c @@ -0,0 +1,644 @@ +#include "blob.h" +#include "util.h" + + +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, + sqlite3_blob *blob) +{ + Py_INCREF(connection); + self->connection = connection; + self->offset = 0; + self->blob = blob; + self->in_weakreflist = NULL; + + Py_BEGIN_ALLOW_THREADS + self->length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (!pysqlite_check_thread(self->connection)) { + return -1; + } + return 0; +} + +static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +{ + Py_ssize_t i; + PyObject *item; + + for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { + item = PyList_GET_ITEM(self->connection->blobs, i); + if (PyWeakref_GetObject(item) == (PyObject *)self) { + PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + break; + } + } +} + +static void _close_blob_inner(pysqlite_Blob* self) +{ + sqlite3_blob *blob; + + /* close the blob */ + blob = self->blob; + self->blob = NULL; + if (blob) { + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + } + + /* remove from connection weaklist */ + remove_blob_from_connection_blob_list(self); + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } +} + +static void pysqlite_blob_dealloc(pysqlite_Blob* self) +{ + _close_blob_inner(self); + Py_XDECREF(self->connection); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* + * Checks if a blob object is usable (i. e. not closed). + * + * 0 => error; 1 => ok + */ +int pysqlite_check_blob(pysqlite_Blob *blob) +{ + + if (!blob->blob) { + PyErr_SetString(pysqlite_ProgrammingError, + "Cannot operate on a closed blob."); + return 0; + } else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { + return 0; + } else { + return 1; + } +} + + +PyObject* pysqlite_blob_close(pysqlite_Blob *self) +{ + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + _close_blob_inner(self); + Py_RETURN_NONE; +}; + + +static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return -1; + } + + return self->length; +}; + +static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +{ + PyObject *buffer; + char *raw_buffer; + int rc; + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + return buffer; +} + + +PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +{ + int read_length = -1; + PyObject *buffer; + + if (!PyArg_ParseTuple(args, "|i", &read_length)) { + return NULL; + } + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + if (read_length < 0) { + /* same as file read. */ + read_length = self->length; + } + + /* making sure we don't read more then blob size */ + if (read_length > self->length - self->offset) { + read_length = self->length - self->offset; + } + + buffer = inner_read(self, read_length, self->offset); + + if (buffer != NULL) { + /* update offset on sucess. */ + self->offset += read_length; + } + + return buffer; +}; + +static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +{ + int rc; + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, buf, len, offset); + Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return -1; + } + return 0; +} + + +PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +{ + Py_buffer data_buffer; + int rc; + + if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { + return NULL; + } + + if (data_buffer.len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "data longer than INT_MAX bytes"); + PyBuffer_Release(&data_buffer); + return NULL; + } + + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } + + /* TODO: throw better error on data bigger then blob. */ + + rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + + if (rc == 0) { + self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); + Py_RETURN_NONE; + } else { + PyBuffer_Release(&data_buffer); + return NULL; + } +} + + +PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) +{ + int offset, from_what = 0; + + if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { + return NULL; + } + + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + switch (from_what) { + case 0: // relative to blob begin + break; + case 1: // relative to current position + if (offset > INT_MAX - self->offset) { + goto overflow; + } + offset = self->offset + offset; + break; + case 2: // relative to blob end + if (offset > INT_MAX - self->length) { + goto overflow; + } + offset = self->length + offset; + break; + default: + return PyErr_Format(PyExc_ValueError, + "from_what should be 0, 1 or 2"); + } + + if (offset < 0 || offset > self->length) { + return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + } + + self->offset = offset; + Py_RETURN_NONE; + +overflow: + return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); +} + + +PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + return PyLong_FromLong(self->offset); +} + + +PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_INCREF(self); + return (PyObject *)self; +} + + +PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +{ + PyObject *res; + if (!pysqlite_check_blob(self)) { + return NULL; + } + + res = pysqlite_blob_close(self); + if (!res) { + return NULL; + } + Py_XDECREF(res); + + Py_RETURN_FALSE; +} + +static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support concatenation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support repeat operation"); + } + return NULL; +} + +static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support cotains operation"); + } + return -1; +} + +static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + + return inner_read(self, 1, i); +} + +static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +{ + const char *buf; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(v); + return write_inner(self, buf, 1, i); +} + + +static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += self->length; + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + + if (PySlice_GetIndicesEx(item, self->length, + &start, &stop, &step, &slicelen) < 0) { + return NULL; + } + + if (slicelen <= 0) + return PyBytes_FromStringAndSize("", 0); + else if (step == 1) + return inner_read(self, slicelen, start); + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + char *data_buff = NULL; + Py_ssize_t cur, i; + PyObject *result; + int rc; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return NULL; + } + + for (cur = 0, i = 0; i < slicelen; + cur += step, i++) { + result_buf[i] = data_buff[cur]; + } + result = PyBytes_FromStringAndSize(result_buf, + slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "Blob indices must be integers"); + return NULL; + } +} + + +static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + int rc; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + const char *buf; + + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += self->length; + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + Py_buffer vbuf; + + if (PySlice_GetIndicesEx(item, + self->length, &start, &stop, + &step, &slicelen) < 0) { + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + return -1; + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } + + if (slicelen == 0) { + } + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + Py_ssize_t cur, i; + char *data_buff; + + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + + for (cur = 0, i = 0; + i < slicelen; + cur += step, i++) + { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; + + } + PyBuffer_Release(&vbuf); + return rc; + } + else { + PyErr_SetString(PyExc_TypeError, + "mmap indices must be integer"); + return -1; + } +} + + +static PyMethodDef blob_methods[] = { + {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, + PyDoc_STR("read data from blob")}, + {"write", (PyCFunction)pysqlite_blob_write, METH_O, + PyDoc_STR("write data to blob")}, + {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, + PyDoc_STR("close blob")}, + {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, + PyDoc_STR("change blob current offset")}, + {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, + PyDoc_STR("return blob current offset")}, + {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, + PyDoc_STR("blob context manager enter")}, + {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, + PyDoc_STR("blob context manager exit")}, + {NULL, NULL} +}; + +static PySequenceMethods blob_sequence_methods = { + .sq_length = (lenfunc)pysqlite_blob_length, + .sq_concat = (binaryfunc)pysqlite_blob_concat, + .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, + .sq_item = (ssizeargfunc)pysqlite_blob_item, + .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, + .sq_contains = (objobjproc)pysqlite_blob_contains, +}; + +static PyMappingMethods blob_mapping_methods = { + (lenfunc)pysqlite_blob_length, + (binaryfunc)pysqlite_blob_subscript, + (objobjargproc)pysqlite_blob_ass_subscript, +}; + +PyTypeObject pysqlite_BlobType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)pysqlite_blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, +}; + +extern int pysqlite_blob_setup_types(void) +{ + pysqlite_BlobType.tp_new = PyType_GenericNew; + return PyType_Ready(&pysqlite_BlobType); +} diff --git a/src/blob.h b/src/blob.h new file mode 100644 index 0000000..197e714 --- /dev/null +++ b/src/blob.h @@ -0,0 +1,26 @@ +#ifndef PYSQLITE_BLOB_H +#define PYSQLITE_BLOB_H +#include "Python.h" +#include "sqlcipher/sqlite3.h" +#include "connection.h" + +typedef struct +{ + PyObject_HEAD + pysqlite_Connection* connection; + sqlite3_blob *blob; + int offset; + int length; + + PyObject* in_weakreflist; /* List of weak references */ +} pysqlite_Blob; + +extern PyTypeObject pysqlite_BlobType; + +int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, + sqlite3_blob *blob); +PyObject* pysqlite_blob_close(pysqlite_Blob *self); + +int pysqlite_blob_setup_types(void); + +#endif diff --git a/src/connection.c b/src/connection.c index 376502e..ed22cc6 100644 --- a/src/connection.c +++ b/src/connection.c @@ -27,6 +27,7 @@ #include "connection.h" #include "statement.h" #include "cursor.h" +#include "blob.h" #include "prepare_protocol.h" #include "util.h" @@ -114,6 +115,7 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_CLEAR(self->statement_cache); Py_CLEAR(self->statements); Py_CLEAR(self->cursors); + Py_CLEAR(self->blobs); Py_INCREF(Py_None); Py_XSETREF(self->row_factory, Py_None); @@ -162,10 +164,11 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject self->created_statements = 0; self->created_cursors = 0; - /* Create lists of weak references to statements/cursors */ + /* Create lists of weak references to statements/cursors/blobs */ self->statements = PyList_New(0); self->cursors = PyList_New(0); - if (!self->statements || !self->cursors) { + self->blobs = PyList_New(0); + if (!self->statements || !self->cursors || !self->blobs) { return -1; } @@ -261,6 +264,7 @@ void pysqlite_connection_dealloc(pysqlite_Connection* self) Py_XDECREF(self->collations); Py_XDECREF(self->statements); Py_XDECREF(self->cursors); + Py_XDECREF(self->blobs); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -331,6 +335,84 @@ PyObject* pysqlite_connection_cursor(pysqlite_Connection* self, PyObject* args, return cursor; } +PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, + PyObject *kwargs) +{ + static char *kwlist[] = {"table", "column", "row", "readonly", + "dbname", NULL, NULL}; + int rc; + const char *dbname = "main", *table, *column; + long long row; + int readonly = 0; + sqlite3_blob *blob; + pysqlite_Blob *pyblob = NULL; + PyObject *weakref; + + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + &table, &column, &row, &readonly, + &dbname)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_open(self->db, dbname, table, column, row, + !readonly, &blob); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + _pysqlite_seterror(self->db, NULL); + return NULL; + } + + pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (!pyblob) { + goto error; + } + + rc = pysqlite_blob_init(pyblob, self, blob); + if (rc) { + Py_CLEAR(pyblob); + goto error; + } + + // Add our blob to connection blobs list + weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (!weakref) { + Py_CLEAR(pyblob); + goto error; + } + if (PyList_Append(self->blobs, weakref) != 0) { + Py_CLEAR(weakref); + Py_CLEAR(pyblob); + goto error; + } + Py_DECREF(weakref); + + return (PyObject*)pyblob; + +error: + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + return NULL; +} + +static void pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + pysqlite_blob_close((pysqlite_Blob*)blob); + } + } +} + PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) { int rc; @@ -341,6 +423,8 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) pysqlite_do_all_statements(self, ACTION_FINALIZE, 1); + pysqlite_close_all_blobs(self); + if (self->db) { rc = SQLITE3_CLOSE(self->db); @@ -1875,6 +1959,8 @@ static PyGetSetDef connection_getset[] = { static PyMethodDef connection_methods[] = { {"cursor", (PyCFunction)(void(*)(void))pysqlite_connection_cursor, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Return a cursor for the connection.")}, + {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("return a blob object")}, {"close", (PyCFunction)pysqlite_connection_close, METH_NOARGS, PyDoc_STR("Closes the connection.")}, {"commit", (PyCFunction)pysqlite_connection_commit, METH_NOARGS, diff --git a/src/connection.h b/src/connection.h index 82093de..37c83b2 100644 --- a/src/connection.h +++ b/src/connection.h @@ -66,9 +66,10 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements and cursors used within this connection */ + /* Lists of weak references to statements, blobs and cursors used within this connection */ PyObject* statements; PyObject* cursors; + PyObject* blobs; /* Counters for how many statements/cursors were created in the connection. May be * reset to 0 at certain intervals */ diff --git a/src/module.c b/src/module.c index 2a90c00..7ba9e42 100644 --- a/src/module.c +++ b/src/module.c @@ -28,6 +28,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "blob.h" #if SQLITE_VERSION_NUMBER >= 3003003 #define HAVE_SHARED_CACHE @@ -274,13 +275,71 @@ struct _IntConstantPair { typedef struct _IntConstantPair IntConstantPair; +/* sqlite API error codes */ +static const IntConstantPair _error_codes[] = { + {"SQLITE_OK", SQLITE_OK}, + {"SQLITE_ERROR", SQLITE_ERROR}, + {"SQLITE_INTERNAL", SQLITE_INTERNAL}, + {"SQLITE_PERM", SQLITE_PERM}, + {"SQLITE_ABORT", SQLITE_ABORT}, + {"SQLITE_BUSY", SQLITE_BUSY}, + {"SQLITE_LOCKED", SQLITE_LOCKED}, + {"SQLITE_NOMEM", SQLITE_NOMEM}, + {"SQLITE_READONLY", SQLITE_READONLY}, + {"SQLITE_INTERRUPT", SQLITE_INTERRUPT}, + {"SQLITE_IOERR", SQLITE_IOERR}, + {"SQLITE_CORRUPT", SQLITE_CORRUPT}, + {"SQLITE_NOTFOUND", SQLITE_NOTFOUND}, + {"SQLITE_FULL", SQLITE_FULL}, + {"SQLITE_CANTOPEN", SQLITE_CANTOPEN}, + {"SQLITE_PROTOCOL", SQLITE_PROTOCOL}, + {"SQLITE_EMPTY", SQLITE_EMPTY}, + {"SQLITE_SCHEMA", SQLITE_SCHEMA}, + {"SQLITE_TOOBIG", SQLITE_TOOBIG}, + {"SQLITE_CONSTRAINT", SQLITE_CONSTRAINT}, + {"SQLITE_MISMATCH", SQLITE_MISMATCH}, + {"SQLITE_MISUSE", SQLITE_MISUSE}, +#ifdef SQLITE_NOLFS + {"SQLITE_NOLFS", SQLITE_NOLFS}, +#endif +#ifdef SQLITE_AUTH + {"SQLITE_AUTH", SQLITE_AUTH}, +#endif +#ifdef SQLITE_FORMAT + {"SQLITE_FORMAT", SQLITE_FORMAT}, +#endif +#ifdef SQLITE_RANGE + {"SQLITE_RANGE", SQLITE_RANGE}, +#endif +#ifdef SQLITE_NOTADB + {"SQLITE_NOTADB", SQLITE_NOTADB}, +#endif + {"SQLITE_DONE", SQLITE_DONE}, + {"SQLITE_ROW", SQLITE_ROW}, + {(char*)NULL, 0}, + {"SQLITE_UNKNOWN", -1} +}; + +const char *sqlite3ErrName(int rc) { + int i; + for (i = 0; _error_codes[i].constant_name != 0; i++) { + if (_error_codes[i].constant_value == rc) + return _error_codes[i].constant_name; + } + // No error code matched. + return _error_codes[i+1].constant_name; +} + static const IntConstantPair _int_constants[] = { {"PARSE_DECLTYPES", PARSE_DECLTYPES}, {"PARSE_COLNAMES", PARSE_COLNAMES}, {"SQLITE_OK", SQLITE_OK}, + /* enumerated return values for sqlite3_set_authorizer() callback */ {"SQLITE_DENY", SQLITE_DENY}, {"SQLITE_IGNORE", SQLITE_IGNORE}, + + /* enumerated values for sqlite3_set_authorizer() callback */ {"SQLITE_CREATE_INDEX", SQLITE_CREATE_INDEX}, {"SQLITE_CREATE_TABLE", SQLITE_CREATE_TABLE}, {"SQLITE_CREATE_TEMP_INDEX", SQLITE_CREATE_TEMP_INDEX}, @@ -354,6 +413,29 @@ static struct PyModuleDef _sqlite3module = { NULL }; + +static int add_to_dict(PyObject *dict, const char *key, int value) +{ + int sawerror; + PyObject *value_obj = PyLong_FromLong(value); + PyObject *name = PyUnicode_FromString(key); + + if (!value_obj || !name) { + Py_XDECREF(name); + Py_XDECREF(value_obj); + return 1; + } + + sawerror = PyDict_SetItem(dict, name, value_obj) < 0; + + Py_DECREF(value_obj); + Py_DECREF(name); + + if (sawerror) + return 1; + return 0; +} + PyMODINIT_FUNC PyInit__sqlite3(void) { PyObject *module, *dict; @@ -368,7 +450,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types() < 0) || (pysqlite_cache_setup_types() < 0) || (pysqlite_statement_setup_types() < 0) || - (pysqlite_prepare_protocol_setup_types() < 0) + (pysqlite_prepare_protocol_setup_types() < 0) || + (pysqlite_blob_setup_types() < 0) ) { Py_XDECREF(module); return NULL; @@ -457,12 +540,16 @@ PyMODINIT_FUNC PyInit__sqlite3(void) /* Set integer constants */ for (i = 0; _int_constants[i].constant_name != NULL; i++) { - tmp_obj = PyLong_FromLong(_int_constants[i].constant_value); - if (!tmp_obj) { + if (add_to_dict(dict, _int_constants[i].constant_name, + _int_constants[i].constant_value) != 0) + goto error; + } + + /* Set error constants */ + for (i = 0; _error_codes[i].constant_name != 0; i++) { + if (add_to_dict(dict, _error_codes[i].constant_name, + _error_codes[i].constant_value) != 0) goto error; - } - PyDict_SetItemString(dict, _int_constants[i].constant_name, tmp_obj); - Py_DECREF(tmp_obj); } if (!(tmp_obj = PyUnicode_FromString(PYSQLITE_VERSION))) { diff --git a/src/module.h b/src/module.h index 3185ec9..8db2233 100644 --- a/src/module.h +++ b/src/module.h @@ -48,6 +48,8 @@ extern PyObject* _pysqlite_converters; extern int _pysqlite_enable_callback_tracebacks; extern int pysqlite_BaseTypeAdapted; +extern const char *sqlite3ErrName(int rc); + #define PARSE_DECLTYPES 1 #define PARSE_COLNAMES 2 #endif diff --git a/src/util.c b/src/util.c index 3fa671d..e8de4f6 100644 --- a/src/util.c +++ b/src/util.c @@ -47,20 +47,21 @@ int pysqlite_step(sqlite3_stmt* statement, pysqlite_Connection* connection) */ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) { + PyObject *exc_class; int errorcode = sqlite3_errcode(db); switch (errorcode) { case SQLITE_OK: PyErr_Clear(); - break; + return errorcode; case SQLITE_INTERNAL: case SQLITE_NOTFOUND: - PyErr_SetString(pysqlite_InternalError, sqlite3_errmsg(db)); + exc_class = pysqlite_InternalError; break; case SQLITE_NOMEM: (void)PyErr_NoMemory(); - break; + return errorcode; case SQLITE_ERROR: case SQLITE_PERM: case SQLITE_ABORT: @@ -74,26 +75,70 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) case SQLITE_PROTOCOL: case SQLITE_EMPTY: case SQLITE_SCHEMA: - PyErr_SetString(pysqlite_OperationalError, sqlite3_errmsg(db)); + exc_class = pysqlite_OperationalError; break; case SQLITE_CORRUPT: - PyErr_SetString(pysqlite_DatabaseError, sqlite3_errmsg(db)); + exc_class = pysqlite_DatabaseError; break; case SQLITE_TOOBIG: - PyErr_SetString(pysqlite_DataError, sqlite3_errmsg(db)); + exc_class = pysqlite_DataError; break; case SQLITE_CONSTRAINT: case SQLITE_MISMATCH: - PyErr_SetString(pysqlite_IntegrityError, sqlite3_errmsg(db)); + exc_class = pysqlite_IntegrityError; break; case SQLITE_MISUSE: - PyErr_SetString(pysqlite_ProgrammingError, sqlite3_errmsg(db)); + exc_class = pysqlite_ProgrammingError;; break; default: - PyErr_SetString(pysqlite_DatabaseError, sqlite3_errmsg(db)); + exc_class = pysqlite_DatabaseError; break; } + /* Create and set the exception. */ + { + const char *error_msg; + const char *error_name; + PyObject *exc = NULL; + PyObject *args = NULL; + PyObject *py_code = NULL; + PyObject *py_name = NULL; + + error_name = sqlite3ErrName(errorcode); + + error_msg = sqlite3_errmsg(db); + + args = Py_BuildValue("(s)", error_msg); + if (!args) + goto error; + + exc = PyObject_Call(exc_class, args, NULL); + if (!exc) + goto error; + + py_code = Py_BuildValue("i", errorcode); + if (!py_code) + goto error; + + if (PyObject_SetAttrString(exc, "sqlite_errorcode", py_code) < 0) + goto error; + + py_name = Py_BuildValue("s", error_name); + if (!py_name) + goto error; + + if (PyObject_SetAttrString(exc, "sqlite_errorname", py_name) < 0) + goto error; + + PyErr_SetObject((PyObject *) Py_TYPE(exc), exc); + + error: + Py_XDECREF(py_code); + Py_XDECREF(py_name); + Py_XDECREF(args); + Py_XDECREF(exc); + } + return errorcode; } diff --git a/test/backup.py b/test/backup.py index 2baf3e5..29c6916 100644 --- a/test/backup.py +++ b/test/backup.py @@ -136,8 +136,8 @@ class BackupTests(unittest.TestCase): def test_database_source_name(self): with sqlite.connect(':memory:') as bck: self.cx.backup(bck, name='main') - with sqlite.connect(':memory:') as bck: - self.cx.backup(bck, name='temp') + #with sqlite.connect(':memory:') as bck: + # self.cx.backup(bck, name='temp') with self.assertRaises(sqlite.OperationalError) as cm: with sqlite.connect(':memory:') as bck: self.cx.backup(bck, name='non-existing') diff --git a/test/dbapi.py b/test/dbapi.py index 6249828..a642b9b 100644 --- a/test/dbapi.py +++ b/test/dbapi.py @@ -23,10 +23,10 @@ import threading import unittest -from sqlcipher3 import dbapi2 as sqlite +from pysqlite3 import dbapi2 as sqlite #from test.support import TESTFN, unlink -TESTFN = '/tmp/sqlcipher3_test' +TESTFN = '/tmp/pysqlite3_test' from os import unlink @@ -85,6 +85,14 @@ class ModuleTests(unittest.TestCase): sqlite.DatabaseError), "NotSupportedError is not a subclass of DatabaseError") + def CheckErrorCodeOnException(self): + with self.assertRaises(sqlite.Error) as cm: + db = sqlite.connect('/no/such/file/exists') + e = cm.exception + self.assertEqual(e.sqlite_errorcode, sqlite.SQLITE_CANTOPEN) + self.assertEqual(e.sqlite_errorname, "SQLITE_CANTOPEN") + self.assertEqual(str(e), "unable to open database file") + class ConnectionTests(unittest.TestCase): def setUp(self): @@ -512,7 +520,177 @@ class CursorTests(unittest.TestCase): ] self.assertEqual(results, expected) +class BlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.blob_data = b"a" * 100 + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.second_data = b"b" * 100 + def tearDown(self): + self.blob.close() + self.cx.close() + + def CheckLength(self): + self.assertEqual(len(self.blob), 100) + + def CheckTell(self): + self.assertEqual(self.blob.tell(), 0) + + def CheckSeekFromBlobStart(self): + self.blob.seek(10) + self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, 0) + self.assertEqual(self.blob.tell(), 10) + + def CheckSeekFromCurrentPosition(self): + self.blob.seek(10, 1) + self.blob.seek(10, 1) + self.assertEqual(self.blob.tell(), 20) + + def CheckSeekFromBlobEnd(self): + self.blob.seek(-10, 2) + self.assertEqual(self.blob.tell(), 90) + + def CheckBlobSeekOverBlobSize(self): + with self.assertRaises(ValueError): + self.blob.seek(1000) + + def CheckBlobSeekUnderBlobSize(self): + with self.assertRaises(ValueError): + self.blob.seek(-10) + + def CheckBlobRead(self): + self.assertEqual(self.blob.read(), self.blob_data) + + def CheckBlobReadSize(self): + self.assertEqual(len(self.blob.read(10)), 10) + + def CheckBlobReadAdvanceOffset(self): + self.blob.read(10) + self.assertEqual(self.blob.tell(), 10) + + def CheckBlobReadStartAtOffset(self): + self.blob.seek(10) + self.blob.write(self.second_data[:10]) + self.blob.seek(10) + self.assertEqual(self.blob.read(10), self.second_data[:10]) + + def CheckBlobWrite(self): + self.blob.write(self.second_data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + + def CheckBlobWriteAtOffset(self): + self.blob.seek(50) + self.blob.write(self.second_data[:50]) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], + self.blob_data[:50] + self.second_data[:50]) + + def CheckBlobWriteAdvanceOffset(self): + self.blob.write(self.second_data[:50]) + self.assertEqual(self.blob.tell(), 50) + + def CheckBlobWriteMoreThenBlobSize(self): + with self.assertRaises(sqlite.OperationalError): + self.blob.write(b"a" * 1000) + + def CheckBlobReadAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + with self.assertRaises(sqlite.OperationalError): + self.blob.read() + + def CheckBlobWriteAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + with self.assertRaises(sqlite.OperationalError): + self.blob.write(b"aaa") + + def CheckBlobWriteWhenReadOnly(self): + read_only_blob = \ + self.cx.open_blob("test", "blob_col", 1, readonly=True) + with self.assertRaises(sqlite.OperationalError): + read_only_blob.write(b"aaa") + read_only_blob.close() + + def CheckBlobOpenWithBadDb(self): + with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + + def CheckBlobOpenWithBadTable(self): + with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob("notexisintg", "blob_col", 1) + + def CheckBlobOpenWithBadColumn(self): + with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob("test", "notexisting", 1) + + def CheckBlobOpenWithBadRow(self): + with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob("test", "blob_col", 2) + + def CheckBlobGetItem(self): + self.assertEqual(self.blob[5], b"a") + + def CheckBlobGetItemIndexOutOfRange(self): + with self.assertRaises(IndexError): + self.blob[105] + with self.assertRaises(IndexError): + self.blob[-105] + + def CheckBlobGetItemNegativeIndex(self): + self.assertEqual(self.blob[-5], b"a") + + def CheckBlobGetItemInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[b"a"] + + def CheckBlobGetSlice(self): + self.assertEqual(self.blob[5:10], b"aaaaa") + + def CheckBlobGetSliceNegativeIndex(self): + self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + + def CheckBlobGetSliceInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[5:b"a"] + + def CheckBlobGetSliceWithSkip(self): + self.blob.write(b"abcdefghij") + self.assertEqual(self.blob[0:10:2], b"acegi") + + def CheckBlobSetItem(self): + self.blob[0] = b"b" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + + def CheckBlobSetSlice(self): + self.blob[0:5] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + + def CheckBlobSetSliceWithSkip(self): + self.blob[0:10:2] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + + def CheckBlobGetEmptySlice(self): + self.assertEqual(self.blob[5:5], b"") + + def CheckBlobSetSliceWrongLength(self): + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" + + def CheckBlobConcatNotSupported(self): + with self.assertRaises(SystemError): + self.blob + self.blob + + def CheckBlobRepeateNotSupported(self): + with self.assertRaises(SystemError): + self.blob * 5 + + def CheckBlobContainsNotSupported(self): + with self.assertRaises(SystemError): + b"aaaaa" in self.blob + +@unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -770,6 +948,15 @@ class ClosedConTests(unittest.TestCase): with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4") + def CheckClosedBlobRead(self): + con = sqlite.connect(":memory:") + con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("insert into test(blob_col) values (zeroblob(100))") + blob = con.open_blob("test", "blob_col", 1) + con.close() + with self.assertRaises(sqlite.ProgrammingError): + blob.read() + def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") con.close() @@ -923,6 +1110,68 @@ class SqliteOnConflictTests(unittest.TestCase): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) +class ClosedBlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckClosedRead(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + with self.assertRaises(sqlite.ProgrammingError): + self.blob.read() + + def CheckClosedWrite(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + with self.assertRaises(sqlite.ProgrammingError): + self.blob.write(b"aaaaaaaaa") + + def CheckClosedSeek(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + with self.assertRaises(sqlite.ProgrammingError): + self.blob.seek(10) + + def CheckClosedTell(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + with self.assertRaises(sqlite.ProgrammingError): + self.blob.tell() + + def CheckClosedClose(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + with self.assertRaises(sqlite.ProgrammingError): + self.blob.close() + + +class BlobContextManagerTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckContextExecute(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + + def CheckContextCloseBlob(self): + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.seek(10) + with self.assertRaises(sqlite.ProgrammingError): + blob.close() + + def suite(): module_suite = unittest.makeSuite(ModuleTests, "Check") connection_suite = unittest.makeSuite(ConnectionTests, "Check") @@ -933,10 +1182,14 @@ def suite(): closed_con_suite = unittest.makeSuite(ClosedConTests, "Check") closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check") on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check") + blob_suite = unittest.makeSuite(BlobTests, "Check") + closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") + blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") return unittest.TestSuite(( module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, ext_suite, closed_con_suite, closed_cur_suite, - on_conflict_suite, + on_conflict_suite, blob_suite, closed_blob_suite, + blob_context_manager_suite, )) def test():