Adds Sqlite3.release/2 for prepared statements (#155)

* Adds exqlite_release
* Manually call release on the prepared statements
  The garbage collector will come back through later and release the
  opaque nif resource. This will at least reclaim a large portion of the
  memory taken up by the prepared statement.
* Update documentation and changelog
* Adds tests for Sqlite3.release/2

Author: warmwaffles

CHANGELOG.md

@@ -1,7 +1,8 @@
 # Changelog
 
 ## [Unreleased](unreleased)
-
+### Fixed
+- Fixed perceived memory leak for prepared statements not being cleaned up in a timely manner. This would be an issue for systems under a heavy load. [#155](https://github.com/elixir-sqlite/exqlite/pull/155)
 
 ## [0.6.2] - 2021-08-25
 ### Changed

README.md

@@ -74,6 +74,15 @@ The `Exqlite.Sqlite3` module usage is fairly straight forward.
 
 # No more results
 :done = Exqlite.Sqlite3.step(conn, statement)
+
+# Release the statement.
+#
+# It is recommended you release the statement after using it to reclaim the memory
+# asap, instead of letting the garbage collector eventually releasing the statement.
+#
+# If you are operating at a high load issuing thousands of statements, it would be
+# possible to run out of memory or cause a lot of pressure on memory.
+:ok = Exqlite.Sqlite3.release(conn, statement)
 ```
 
 

c_src/sqlite3_nif.c

@@ -712,6 +712,34 @@ exqlite_deserialize(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     return make_atom(env, "ok");
 }
 
+static ERL_NIF_TERM
+exqlite_release(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    assert(env);
+
+    statement_t* statement = NULL;
+    connection_t* conn     = NULL;
+
+    if (argc != 2) {
+        return enif_make_badarg(env);
+    }
+
+    if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) {
+        return make_error_tuple(env, "invalid_connection");
+    }
+
+    if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) {
+        return make_error_tuple(env, "invalid_statement");
+    }
+
+    if (statement->statement) {
+        sqlite3_finalize(statement->statement);
+        statement->statement = NULL;
+    }
+
+    return make_atom(env, "ok");
+}
+
 static void
 connection_type_destructor(ErlNifEnv* env, void* arg)
 {
@@ -788,6 +816,7 @@ static ErlNifFunc nif_funcs[] = {
   {"transaction_status", 1, exqlite_transaction_status, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"serialize", 2, exqlite_serialize, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"deserialize", 3, exqlite_deserialize, ERL_NIF_DIRTY_JOB_IO_BOUND},
+  {"release", 2, exqlite_release, ERL_NIF_DIRTY_JOB_IO_BOUND},
 };
 
 ERL_NIF_INIT(Elixir.Exqlite.Sqlite3NIF, nif_funcs, on_load, NULL, NULL, NULL)

lib/exqlite/connection.ex

@@ -259,7 +259,8 @@ defmodule Exqlite.Connection do
   This callback is called in the client process.
   """
   @impl true
-  def handle_close(_query, _opts, state) do
+  def handle_close(query, _opts, state) do
+    Sqlite3.release(state.db, query.ref)
     {:ok, nil, state}
   end
 
@@ -274,10 +275,8 @@ defmodule Exqlite.Connection do
   end
 
   @impl true
-  def handle_deallocate(%Query{} = _query, _cursor, _opts, state) do
-    # We actually don't need to do anything about the cursor. Since it is a
-    # prepared statement, it will be garbage collected by erlang when it loses
-    # references.
+  def handle_deallocate(%Query{} = query, _cursor, _opts, state) do
+    Sqlite3.release(state.db, query.ref)
     {:ok, nil, state}
   end
 

lib/exqlite/sqlite3.ex

@@ -166,6 +166,24 @@ defmodule Exqlite.Sqlite3 do
     Sqlite3NIF.deserialize(conn, String.to_charlist(database), serialized)
   end
 
+  def release(_conn, nil), do: :ok
+
+  @doc """
+  Once finished with the prepared statement, call this to release the underlying
+  resources.
+
+  This should be called whenever you are done operating with the prepared statement. If
+  the system has a high load the garbage collector may not clean up the prepared
+  statements in a timely manner and causing higher than normal levels of memory
+  pressure.
+
+  If you are operating on limited memory capacity systems, definitely call this.
+  """
+  @spec release(db(), statement()) :: :ok | {:error, reason()}
+  def release(conn, statement) do
+    Sqlite3NIF.release(conn, statement)
+  end
+
   defp convert(%Date{} = val), do: Date.to_iso8601(val)
   defp convert(%Time{} = val), do: Time.to_iso8601(val)
   defp convert(%NaiveDateTime{} = val), do: NaiveDateTime.to_iso8601(val)

lib/exqlite/sqlite3_nif.ex

@@ -56,5 +56,8 @@ defmodule Exqlite.Sqlite3NIF do
   @spec deserialize(db(), String.Chars.t(), binary()) :: :ok | {:error, reason()}
   def deserialize(_conn, _database, _serialized), do: :erlang.nif_error(:not_loaded)
 
+  @spec release(db(), statement()) :: :ok | {:error, reason()}
+  def release(_conn, _statement), do: :erlang.nif_error(:not_loaded)
+
   # TODO: add statement inspection tooling https://sqlite.org/c3ref/expanded_sql.html
 end

test/exqlite/connection_test.exs

@@ -188,4 +188,22 @@ defmodule Exqlite.ConnectionTest do
       assert {:ok, conn} == Connection.ping(conn)
     end
   end
+
+  describe ".handle_close/3" do
+    test "releases the underlying prepared statement" do
+      {:ok, conn} = Connection.connect(database: :memory)
+
+      {:ok, query, _result, conn} =
+        %Query{statement: "create table users (id integer primary key, name text)"}
+        |> Connection.handle_execute([], [], conn)
+
+      assert {:ok, nil, conn} == Connection.handle_close(query, [], conn)
+
+      {:ok, query, conn} =
+        %Query{statement: "select * from users where id < ?"}
+        |> Connection.handle_prepare([], conn)
+
+      assert {:ok, nil, conn} == Connection.handle_close(query, [], conn)
+    end
+  end
 end

test/exqlite/sqlite3_test.exs

@@ -96,6 +96,34 @@ defmodule Exqlite.Sqlite3Test do
     end
   end
 
+  describe ".release/2" do
+    test "double releasing a statement" do
+      {:ok, conn} = Sqlite3.open(":memory:")
+
+      :ok =
+        Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)")
+
+      {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)")
+      :ok = Sqlite3.release(conn, statement)
+      :ok = Sqlite3.release(conn, statement)
+    end
+
+    test "releasing a statement" do
+      {:ok, conn} = Sqlite3.open(":memory:")
+
+      :ok =
+        Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)")
+
+      {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)")
+      :ok = Sqlite3.release(conn, statement)
+    end
+
+    test "releasing a nil statement" do
+      {:ok, conn} = Sqlite3.open(":memory:")
+      :ok = Sqlite3.release(conn, nil)
+    end
+  end
+
   describe ".bind/3" do
     test "binding values to a valid sql statement" do
       {:ok, conn} = Sqlite3.open(":memory:")