Skip to content

Commit ab994ed

Browse files
committed
Issue #10740: sqlite3 no longer implicitly commit an open transaction before DDL statements
This commit contains the following commits from ghaering/pysqlite: * ghaering/pysqlite@f254c53 * ghaering/pysqlite@796b3af * ghaering/pysqlite@cae87ee * ghaering/pysqlite@3567b31 With the following additions: * Fixed a refcount error * Fixed a compiler warning * Made the string comparison a little more robust * Added a whatsnew entry
1 parent bd48d27 commit ab994ed

File tree

8 files changed

+83
-116
lines changed

8 files changed

+83
-116
lines changed

Doc/library/sqlite3.rst

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -925,9 +925,7 @@ Controlling Transactions
925925

926926
By default, the :mod:`sqlite3` module opens transactions implicitly before a
927927
Data Modification Language (DML) statement (i.e.
928-
``INSERT``/``UPDATE``/``DELETE``/``REPLACE``), and commits transactions
929-
implicitly before a non-DML, non-query statement (i. e.
930-
anything other than ``SELECT`` or the aforementioned).
928+
``INSERT``/``UPDATE``/``DELETE``/``REPLACE``).
931929

932930
So if you are within a transaction and issue a command like ``CREATE TABLE
933931
...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly
@@ -947,6 +945,9 @@ Otherwise leave it at its default, which will result in a plain "BEGIN"
947945
statement, or set it to one of SQLite's supported isolation levels: "DEFERRED",
948946
"IMMEDIATE" or "EXCLUSIVE".
949947

948+
.. versionchanged:: 3.6
949+
:mod:`sqlite3` used to implicitly commit an open transaction before DDL
950+
statements. This is no longer the case.
950951

951952

952953
Using :mod:`sqlite3` efficiently

Doc/whatsnew/3.6.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1195,6 +1195,9 @@ Changes in 'python' Command Behavior
11951195
Changes in the Python API
11961196
-------------------------
11971197

1198+
* :mod:`sqlite3` no longer implicitly commit an open transaction before DDL
1199+
statements.
1200+
11981201
* On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
11991202
is initialized to increase the security.
12001203

Lib/sqlite3/test/transactions.py

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,13 @@ def tearDown(self):
5252
except OSError:
5353
pass
5454

55-
def CheckDMLdoesAutoCommitBefore(self):
55+
def CheckDMLDoesNotAutoCommitBefore(self):
5656
self.cur1.execute("create table test(i)")
5757
self.cur1.execute("insert into test(i) values (5)")
5858
self.cur1.execute("create table test2(j)")
5959
self.cur2.execute("select i from test")
6060
res = self.cur2.fetchall()
61-
self.assertEqual(len(res), 1)
61+
self.assertEqual(len(res), 0)
6262

6363
def CheckInsertStartsTransaction(self):
6464
self.cur1.execute("create table test(i)")
@@ -153,11 +153,6 @@ def setUp(self):
153153
self.con = sqlite.connect(":memory:")
154154
self.cur = self.con.cursor()
155155

156-
def CheckVacuum(self):
157-
self.cur.execute("create table test(i)")
158-
self.cur.execute("insert into test(i) values (5)")
159-
self.cur.execute("vacuum")
160-
161156
def CheckDropTable(self):
162157
self.cur.execute("create table test(i)")
163158
self.cur.execute("insert into test(i) values (5)")
@@ -172,10 +167,35 @@ def tearDown(self):
172167
self.cur.close()
173168
self.con.close()
174169

170+
class TransactionalDDL(unittest.TestCase):
171+
def setUp(self):
172+
self.con = sqlite.connect(":memory:")
173+
174+
def CheckDdlDoesNotAutostartTransaction(self):
175+
# For backwards compatibility reasons, DDL statements should not
176+
# implicitly start a transaction.
177+
self.con.execute("create table test(i)")
178+
self.con.rollback()
179+
result = self.con.execute("select * from test").fetchall()
180+
self.assertEqual(result, [])
181+
182+
def CheckTransactionalDDL(self):
183+
# You can achieve transactional DDL by issuing a BEGIN
184+
# statement manually.
185+
self.con.execute("begin")
186+
self.con.execute("create table test(i)")
187+
self.con.rollback()
188+
with self.assertRaises(sqlite.OperationalError):
189+
self.con.execute("select * from test")
190+
191+
def tearDown(self):
192+
self.con.close()
193+
175194
def suite():
176195
default_suite = unittest.makeSuite(TransactionTests, "Check")
177196
special_command_suite = unittest.makeSuite(SpecialCommandTests, "Check")
178-
return unittest.TestSuite((default_suite, special_command_suite))
197+
ddl_suite = unittest.makeSuite(TransactionalDDL, "Check")
198+
return unittest.TestSuite((default_suite, special_command_suite, ddl_suite))
179199

180200
def test():
181201
runner = unittest.TextTestRunner()

Misc/NEWS

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,9 @@ Core and Builtins
143143
Library
144144
-------
145145

146+
- Issue #10740: sqlite3 no longer implicitly commit an open transaction
147+
before DDL statements.
148+
146149
- Issue #22493: Inline flags now should be used only at the start of the
147150
regular expression. Deprecation warning is emitted if uses them in the
148151
middle of the regular expression.

Modules/_sqlite/cursor.c

Lines changed: 26 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -29,44 +29,6 @@ PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self);
2929

3030
static const char errmsg_fetch_across_rollback[] = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from.";
3131

32-
static pysqlite_StatementKind detect_statement_type(const char* statement)
33-
{
34-
char buf[20];
35-
const char* src;
36-
char* dst;
37-
38-
src = statement;
39-
/* skip over whitepace */
40-
while (*src == '\r' || *src == '\n' || *src == ' ' || *src == '\t') {
41-
src++;
42-
}
43-
44-
if (*src == 0)
45-
return STATEMENT_INVALID;
46-
47-
dst = buf;
48-
*dst = 0;
49-
while (Py_ISALPHA(*src) && (dst - buf) < ((Py_ssize_t)sizeof(buf) - 2)) {
50-
*dst++ = Py_TOLOWER(*src++);
51-
}
52-
53-
*dst = 0;
54-
55-
if (!strcmp(buf, "select")) {
56-
return STATEMENT_SELECT;
57-
} else if (!strcmp(buf, "insert")) {
58-
return STATEMENT_INSERT;
59-
} else if (!strcmp(buf, "update")) {
60-
return STATEMENT_UPDATE;
61-
} else if (!strcmp(buf, "delete")) {
62-
return STATEMENT_DELETE;
63-
} else if (!strcmp(buf, "replace")) {
64-
return STATEMENT_REPLACE;
65-
} else {
66-
return STATEMENT_OTHER;
67-
}
68-
}
69-
7032
static int pysqlite_cursor_init(pysqlite_Cursor* self, PyObject* args, PyObject* kwargs)
7133
{
7234
pysqlite_Connection* connection;
@@ -427,9 +389,9 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
427389
PyObject* func_args;
428390
PyObject* result;
429391
int numcols;
430-
int statement_type;
431392
PyObject* descriptor;
432393
PyObject* second_argument = NULL;
394+
sqlite_int64 lastrowid;
433395

434396
if (!check_cursor(self)) {
435397
goto error;
@@ -510,7 +472,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
510472
/* reset description and rowcount */
511473
Py_INCREF(Py_None);
512474
Py_SETREF(self->description, Py_None);
513-
self->rowcount = -1L;
475+
self->rowcount = 0L;
514476

515477
func_args = PyTuple_New(1);
516478
if (!func_args) {
@@ -549,43 +511,19 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
549511
pysqlite_statement_reset(self->statement);
550512
pysqlite_statement_mark_dirty(self->statement);
551513

552-
statement_type = detect_statement_type(operation_cstr);
553-
if (self->connection->begin_statement) {
554-
switch (statement_type) {
555-
case STATEMENT_UPDATE:
556-
case STATEMENT_DELETE:
557-
case STATEMENT_INSERT:
558-
case STATEMENT_REPLACE:
559-
if (!self->connection->inTransaction) {
560-
result = _pysqlite_connection_begin(self->connection);
561-
if (!result) {
562-
goto error;
563-
}
564-
Py_DECREF(result);
565-
}
566-
break;
567-
case STATEMENT_OTHER:
568-
/* it's a DDL statement or something similar
569-
- we better COMMIT first so it works for all cases */
570-
if (self->connection->inTransaction) {
571-
result = pysqlite_connection_commit(self->connection, NULL);
572-
if (!result) {
573-
goto error;
574-
}
575-
Py_DECREF(result);
576-
}
577-
break;
578-
case STATEMENT_SELECT:
579-
if (multiple) {
580-
PyErr_SetString(pysqlite_ProgrammingError,
581-
"You cannot execute SELECT statements in executemany().");
582-
goto error;
583-
}
584-
break;
514+
/* For backwards compatibility reasons, do not start a transaction if a
515+
DDL statement is encountered. If anybody wants transactional DDL,
516+
they can issue a BEGIN statement manually. */
517+
if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) {
518+
if (sqlite3_get_autocommit(self->connection->db)) {
519+
result = _pysqlite_connection_begin(self->connection);
520+
if (!result) {
521+
goto error;
522+
}
523+
Py_DECREF(result);
585524
}
586525
}
587526

588-
589527
while (1) {
590528
parameters = PyIter_Next(parameters_iter);
591529
if (!parameters) {
@@ -671,6 +609,20 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
671609
}
672610
}
673611

612+
if (!sqlite3_stmt_readonly(self->statement->st)) {
613+
self->rowcount += (long)sqlite3_changes(self->connection->db);
614+
} else {
615+
self->rowcount= -1L;
616+
}
617+
618+
if (!multiple) {
619+
Py_DECREF(self->lastrowid);
620+
Py_BEGIN_ALLOW_THREADS
621+
lastrowid = sqlite3_last_insert_rowid(self->connection->db);
622+
Py_END_ALLOW_THREADS
623+
self->lastrowid = _pysqlite_long_from_int64(lastrowid);
624+
}
625+
674626
if (rc == SQLITE_ROW) {
675627
if (multiple) {
676628
PyErr_SetString(pysqlite_ProgrammingError, "executemany() can only execute DML statements.");
@@ -685,31 +637,6 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
685637
Py_CLEAR(self->statement);
686638
}
687639

688-
switch (statement_type) {
689-
case STATEMENT_UPDATE:
690-
case STATEMENT_DELETE:
691-
case STATEMENT_INSERT:
692-
case STATEMENT_REPLACE:
693-
if (self->rowcount == -1L) {
694-
self->rowcount = 0L;
695-
}
696-
self->rowcount += (long)sqlite3_changes(self->connection->db);
697-
}
698-
699-
Py_DECREF(self->lastrowid);
700-
if (!multiple &&
701-
/* REPLACE is an alias for INSERT OR REPLACE */
702-
(statement_type == STATEMENT_INSERT || statement_type == STATEMENT_REPLACE)) {
703-
sqlite_int64 lastrowid;
704-
Py_BEGIN_ALLOW_THREADS
705-
lastrowid = sqlite3_last_insert_rowid(self->connection->db);
706-
Py_END_ALLOW_THREADS
707-
self->lastrowid = _pysqlite_long_from_int64(lastrowid);
708-
} else {
709-
Py_INCREF(Py_None);
710-
self->lastrowid = Py_None;
711-
}
712-
713640
if (multiple) {
714641
pysqlite_statement_reset(self->statement);
715642
}

Modules/_sqlite/cursor.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,6 @@ typedef struct
5151
PyObject* in_weakreflist; /* List of weak references */
5252
} pysqlite_Cursor;
5353

54-
typedef enum {
55-
STATEMENT_INVALID, STATEMENT_INSERT, STATEMENT_DELETE,
56-
STATEMENT_UPDATE, STATEMENT_REPLACE, STATEMENT_SELECT,
57-
STATEMENT_OTHER
58-
} pysqlite_StatementKind;
59-
6054
extern PyTypeObject pysqlite_CursorType;
6155

6256
PyObject* pysqlite_cursor_execute(pysqlite_Cursor* self, PyObject* args);

Modules/_sqlite/statement.c

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con
5454
int rc;
5555
const char* sql_cstr;
5656
Py_ssize_t sql_cstr_len;
57+
const char* p;
5758

5859
self->st = NULL;
5960
self->in_use = 0;
@@ -72,6 +73,23 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con
7273
Py_INCREF(sql);
7374
self->sql = sql;
7475

76+
/* determine if the statement is a DDL statement */
77+
self->is_ddl = 0;
78+
for (p = sql_cstr; *p != 0; p++) {
79+
switch (*p) {
80+
case ' ':
81+
case '\r':
82+
case '\n':
83+
case '\t':
84+
continue;
85+
}
86+
87+
self->is_ddl = (PyOS_strnicmp(p, "create ", 7) == 0)
88+
|| (PyOS_strnicmp(p, "drop ", 5) == 0)
89+
|| (PyOS_strnicmp(p, "reindex ", 8) == 0);
90+
break;
91+
}
92+
7593
Py_BEGIN_ALLOW_THREADS
7694
rc = sqlite3_prepare(connection->db,
7795
sql_cstr,

Modules/_sqlite/statement.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ typedef struct
3838
sqlite3_stmt* st;
3939
PyObject* sql;
4040
int in_use;
41+
int is_ddl;
4142
PyObject* in_weakreflist; /* List of weak references */
4243
} pysqlite_Statement;
4344

0 commit comments

Comments
 (0)