From cfa27508547b69d805da08f954f9cb70ff189d62 Mon Sep 17 00:00:00 2001 From: Anthony Tuininga Date: Tue, 18 May 2021 16:56:04 -0600 Subject: [PATCH] Add support for pool reconfiguration. --- doc/src/api_manual/module.rst | 9 +- doc/src/api_manual/session_pool.rst | 67 ++++++++++++ doc/src/release_notes.rst | 36 +++--- doc/src/user_guide/connection_handling.rst | 36 ++++++ src/cxoSessionPool.c | 121 ++++++++++++++++++++- test/test_2400_session_pool.py | 69 ++++++++++++ 6 files changed, 321 insertions(+), 17 deletions(-) diff --git a/doc/src/api_manual/module.rst b/doc/src/api_manual/module.rst index daad2fa..dac1a2f 100644 --- a/doc/src/api_manual/module.rst +++ b/doc/src/api_manual/module.rst @@ -320,9 +320,12 @@ Module Interface id=GUID-B853A020-752F-494A-8D88-D0396EF57177>`__ for more information. The max_sessions_per_shard parameter is expected to be an integer, if - specified, and sets the maximum number of sessions in the pool that can be - used for any given shard in a sharded database. This value is ignored if - the Oracle client library version is less than 18.3. + specified. Setting this greater than zero specifies the maximum number of + sessions in the pool that can be used for any given shard in a sharded + database. This lets connections in the pool be balanced across the shards. + A value of zero will not set any maximum number of sessions for each shard. + This value is ignored if the Oracle client library version is less than + 18.3. The soda_metadata_cache parameter is expected to be a boolean expresion which indicates whether or not to enable the SODA metatata cache. This diff --git a/doc/src/api_manual/session_pool.rst b/doc/src/api_manual/session_pool.rst index 83e2f3e..5ef8222 100644 --- a/doc/src/api_manual/session_pool.rst +++ b/doc/src/api_manual/session_pool.rst @@ -118,6 +118,19 @@ SessionPool Object .. versionadded:: 5.3 +.. attribute:: SessionPool.max_sessions_per_shard + + This read-write attribute returns the number of sessions that can be created + per shard in the pool. Setting this attribute greater than zero specifies + the maximum number of sessions in the pool that can be used for any given + shard in a sharded database. This lets connections in the pool be balanced + across the shards. A value of zero will not set any maximum number of + sessions for each shard. This attribute is only available in Oracle Client + 18.3 and higher. + + .. versionadded:: 8.2 + + .. attribute:: SessionPool.min This read-only attribute returns the number of sessions with which the @@ -154,6 +167,60 @@ SessionPool Object .. versionadded:: 8.2 +.. method:: SessionPool.reconfigure([min, max, increment, getmode, timeout, \ + wait_timeout, max_lifetime_session, max_sessions_per_shard, \ + soda_metadata_cache, stmtcachesize, ping_interval]) + + Reconfigures various parameters of a connection pool. The pool size can be + altered with ``reconfigure()`` by passing values for + :data:`~SessionPool.min`, :data:`~SessionPool.max` or + :data:`~SessionPool.increment`. The :data:`~SessionPool.getmode`, + :data:`~SessionPool.timeout`, :data:`~SessionPool.wait_timeout`, + :data:`~SessionPool.max_lifetime_session`, + :data:`~SessionPool.max_sessions_per_shard`, + :data:`~SessionPool.soda_metadata_cache`, :data:`~SessionPool.stmtcachesize` + and :data:`~SessionPool.ping_interval` attributes can be set directly or + with ``reconfigure()``. + + All parameters are optional. Unspecified parameters will leave those pool + attributes unchanged. The parameters are processed in two stages. After any + size change has been processed, reconfiguration on the other parameters is + done sequentially. If an error such as an invalid value occurs when changing + one attribute, then an exception will be generated but any already changed + attributes will retain their new values. + + During reconfiguration of a pool's size, the behavior of + :meth:`SessionPool.acquire()` depends on the ``getmode`` in effect when + ``acquire()`` is called: + + * With mode :data:`~cx_Oracle.SPOOL_ATTRVAL_FORCEGET`, an ``acquire()`` call + will wait until the pool has been reconfigured. + + * With mode :data:`~cx_Oracle.SPOOL_ATTRVAL_TIMEDWAIT`, an ``acquire()`` call + will try to acquire a connection in the time specified by + :data:`~SessionPool.wait_timeout` and return an error if the time taken + exceeds that value. + + * With mode :data:`~cx_Oracle.SPOOL_ATTRVAL_WAIT`, an ``acquire()`` call + will wait until after the pool has been reconfigured and a connection is + available. + + * With mode :data:`~cx_Oracle.SPOOL_ATTRVAL_NOWAIT`, if the number of busy + connections is less than the pool size, ``acquire()`` will return a new + connection after pool reconfiguration is complete. + + Closing connections with :meth:`SessionPool.release()` or + :meth:`Connection.close()` will wait until any pool size reconfiguration is + complete. + + Closing the connection pool with :meth:`SessionPool.close()` will wait until + reconfiguration is complete. + + See :ref:`Connection Pool Reconfiguration `. + + .. versionadded:: 8.2 + + .. method:: SessionPool.release(connection, tag=None) Release the connection back to the pool now, rather than whenever __del__ diff --git a/doc/src/release_notes.rst b/doc/src/release_notes.rst index ecbf2ab..11037ac 100644 --- a/doc/src/release_notes.rst +++ b/doc/src/release_notes.rst @@ -15,23 +15,33 @@ Version 8.2 (TBD) version-4-2-tbd>`__. #) Threaded mode is now always enabled when creating connection pools with :meth:`cx_Oracle.SessionPool()`. Any `threaded` parameter value is ignored. +#) Added :meth:`SessionPool.reconfigure()` to support pool reconfiguration. + This method provides the ability to change properties such as the size of + existing pools instead of having to restart the application or create a new + pool. +#) Added parameter `max_sessions_per_shard` to :meth:`cx_Oracle.SessionPool()` + to allow configuration of the maximum number of sessions per shard in the + pool. In addition, the attribute + :data:`SessionPool.max_sessions_per_shard` was added in order to permit + making adjustments after the pool has been created. They are usable when + using Oracle Client version 18.4 and higher. #) Added parameter `stmtcachesize` to :meth:`cx_Oracle.connect()` and :meth:`cx_Oracle.SessionPool()` in order to permit specifying the size of the statement cache during the creation of pools and standalone connections. -#) Added parameter `ping_interval` to :meth:`cx_Oracle.SessionPool()` to specify - the ping interval when acquiring pooled connections. In addition, the - attribute :data:`SessionPool.ping_interval` was added in order to permit - making adjustments after the pool has been created. In previous cx_Oracle - releases a fixed ping interval of 60 seconds was used. -#) Added parameter `soda_metadata_cache` to :meth:`cx_Oracle.SessionPool()` for - :ref:`SODA metadata cache ` support. In addition, the - attribute :data:`SessionPool.soda_metadata_cache` was added in order to +#) Added parameter `ping_interval` to :meth:`cx_Oracle.SessionPool()` to + specify the ping interval when acquiring pooled connections. In addition, + the attribute :data:`SessionPool.ping_interval` was added in order to + permit making adjustments after the pool has been created. In previous + cx_Oracle releases a fixed ping interval of 60 seconds was used. +#) Added parameter `soda_metadata_cache` to :meth:`cx_Oracle.SessionPool()` + for :ref:`SODA metadata cache ` support. In addition, + the attribute :data:`SessionPool.soda_metadata_cache` was added in order to permit making adjustments after the pool has been created. This feature significantly improves the performance of methods - :meth:`SodaDatabase.createCollection()` (when not specifying a value for the - metadata parameter) and :meth:`SodaDatabase.openCollection()`. Caching is - available when using Oracle Client version 19.11 and higher. + :meth:`SodaDatabase.createCollection()` (when not specifying a value for + the metadata parameter) and :meth:`SodaDatabase.openCollection()`. Caching + is available when using Oracle Client version 19.11 and higher. #) Added support for supplying hints to SODA operations. A new non-terminal method :meth:`~SodaOperation.hint()` was added and a `hint` parameter was added to the methods :meth:`SodaCollection.insertOneAndGet()`, @@ -56,8 +66,8 @@ Version 8.2 (TBD) #) The distributed transaction handle assosciated with the connection is now cleared on commit or rollback (`issue 530 `__). -#) Added a check to ensure that when setting variables or object attributes, the - type of the temporary LOB must match the expected type. +#) Added a check to ensure that when setting variables or object attributes, + the type of the temporary LOB must match the expected type. #) A small number of parameter, method, and attribute names were updated to follow the PEP 8 style guide. This brings better consistency to the cx_Oracle API. The old names are still usable but may be removed in a diff --git a/doc/src/user_guide/connection_handling.rst b/doc/src/user_guide/connection_handling.rst index 0942ce1..ad14220 100644 --- a/doc/src/user_guide/connection_handling.rst +++ b/doc/src/user_guide/connection_handling.rst @@ -407,6 +407,42 @@ or user profile `IDLE_TIME do not expire idle sessions, since this will require connections be recreated, which will impact performance and scalability. +.. _poolreconfiguration: + +Connection Pool Reconfiguration +------------------------------- + +Some pool settings can be changed dynamically with +:meth:`SessionPool.reconfigure()`. This allows the pool size and other +attributes to be changed during application runtime without needing to restart +the pool or application. + +For example a pool's size can be changed like: + +.. code-block:: python + + pool.reconfigure(min=10, max=10, increment=0) + +After any size change has been processed, reconfiguration on the other +parameters is done sequentially. If an error such as an invalid value occurs +when changing one attribute, then an exception will be generated but any already +changed attributes will retain their new values. + +During reconfiguration of a pool's size, the behavior of +:meth:`SessionPool.acquire()` depends on the ``getmode`` in effect when +``acquire()`` is called, see :meth:`SessionPool.reconfigure()`. Closing +connections or closing the pool will wait until after pool reconfiguration is +complete. + +Calling ``reconfigure()`` is the only way to change a pool's ``min``, ``max`` +and ``increment`` values. Other attributes such as +:data:`~SessionPool.wait_timeout` can also be passed to ``reconfigure()`` or +they can be set directly: + +.. code-block:: python + + pool.wait_timeout = 1000 + .. _sessioncallback: Session CallBacks for Setting Pooled Connection State diff --git a/src/cxoSessionPool.c b/src/cxoSessionPool.c index 3d38af6..917592d 100644 --- a/src/cxoSessionPool.c +++ b/src/cxoSessionPool.c @@ -1,5 +1,5 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. // // Portions Copyright 2007-2015, Anthony Tuininga. All rights reserved. // @@ -14,6 +14,11 @@ #include "cxoModule.h" +// forward declarations +int cxoSessionPool_reconfigureHelper(cxoSessionPool *pool, + const char *attrName, PyObject *value); + + //----------------------------------------------------------------------------- // cxoSessionPool_new() // Create a new session pool object. @@ -338,6 +343,93 @@ static PyObject *cxoSessionPool_drop(cxoSessionPool *pool, PyObject *args) } +//----------------------------------------------------------------------------- +// cxoSessionPool_reconfigure() +// Reconfigure properties of the session pool. +//----------------------------------------------------------------------------- +static PyObject *cxoSessionPool_reconfigure(cxoSessionPool *pool, + PyObject *args, PyObject *keywordArgs) +{ + PyObject *timeout, *waitTimeout, *maxLifetimeSession, *maxSessionsPerShard; + PyObject *sodaMetadataCache, *stmtcachesize, *pingInterval, *getMode; + uint32_t minSessions, maxSessions, sessionIncrement; + + // define keyword arguments + static char *keywordList[] = { "min", "max", "increment", "getmode", + "timeout", "wait_timeout", "max_lifetime_session", + "max_sessions_per_shard", "soda_metadata_cache", "stmtcachesize", + "ping_interval", NULL }; + + // set up default values + minSessions = pool->minSessions; + maxSessions = pool->maxSessions; + sessionIncrement = pool->sessionIncrement; + timeout = waitTimeout = maxLifetimeSession = maxSessionsPerShard = NULL; + sodaMetadataCache = stmtcachesize = pingInterval = getMode = NULL; + + // parse arguments and keywords + if (!PyArg_ParseTupleAndKeywords(args, keywordArgs, "|iiiOOOOOOOO", + keywordList, &minSessions, &maxSessions, &sessionIncrement, + &getMode, &timeout, &waitTimeout, &maxLifetimeSession, + &maxSessionsPerShard, &sodaMetadataCache, &stmtcachesize, + &pingInterval)) + return NULL; + + // perform reconfiguration of the pool itself if needed + if (minSessions != pool->minSessions || maxSessions != pool->maxSessions || + sessionIncrement != pool->sessionIncrement) { + if (dpiPool_reconfigure(pool->handle, minSessions, maxSessions, + sessionIncrement) < 0) + return cxoError_raiseAndReturnNull(); + pool->minSessions = minSessions; + pool->maxSessions = maxSessions; + pool->sessionIncrement = sessionIncrement; + } + + // adjust attributes + if (cxoSessionPool_reconfigureHelper(pool, "getmode", getMode) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "timeout", timeout) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "wait_timeout", + waitTimeout) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "max_lifetime_session", + maxLifetimeSession) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "max_sessions_per_shard", + maxSessionsPerShard) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "soda_metadata_cache", + sodaMetadataCache) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "stmtcachesize", + stmtcachesize) < 0) + return NULL; + if (cxoSessionPool_reconfigureHelper(pool, "ping_interval", + pingInterval) < 0) + return NULL; + + Py_RETURN_NONE; +} + + +//----------------------------------------------------------------------------- +// cxoSessionPool_reconfigureHelpe() +// Helper function that calls the setter for the session pool's property, +// after first checking that a value was supplied and not None. +//----------------------------------------------------------------------------- +int cxoSessionPool_reconfigureHelper(cxoSessionPool *pool, + const char *attrName, PyObject *value) +{ + if (value != NULL && value != Py_None) { + if (PyObject_SetAttrString((PyObject*) pool, attrName, value) < 0) + return cxoError_raiseAndReturnInt(); + } + return 0; +} + + //----------------------------------------------------------------------------- // cxoSessionPool_release() // Release a connection back to the session pool. @@ -455,6 +547,17 @@ static PyObject *cxoSessionPool_getMaxLifetimeSession(cxoSessionPool *pool, } +//----------------------------------------------------------------------------- +// cxoSessionPool_getMaxSessionsPerShard() +// Return the maximum sessions per shard in the session pool. +//----------------------------------------------------------------------------- +static PyObject *cxoSessionPool_getMaxSessionsPerShard(cxoSessionPool *pool, + void *unused) +{ + return cxoSessionPool_getAttribute(pool, dpiPool_getMaxSessionsPerShard); +} + + //----------------------------------------------------------------------------- // cxoSessionPool_getOpenCount() // Return the number of open connections in the session pool. @@ -559,6 +662,18 @@ static int cxoSessionPool_setMaxLifetimeSession(cxoSessionPool *pool, } +//----------------------------------------------------------------------------- +// cxoSessionPool_setMaxSessionsPerShard() +// Set the maximum lifetime for connections in the session pool. +//----------------------------------------------------------------------------- +static int cxoSessionPool_setMaxSessionsPerShard(cxoSessionPool *pool, + PyObject *value, void *unused) +{ + return cxoSessionPool_setAttribute(pool, value, + dpiPool_setMaxSessionsPerShard); +} + + //----------------------------------------------------------------------------- // cxoSessionPool_setPingInterval() // Set the value of the OCI attribute. @@ -649,6 +764,8 @@ static PyMethodDef cxoMethods[] = { { "close", (PyCFunction) cxoSessionPool_close, METH_VARARGS | METH_KEYWORDS }, { "drop", (PyCFunction) cxoSessionPool_drop, METH_VARARGS }, + { "reconfigure", (PyCFunction) cxoSessionPool_reconfigure, + METH_VARARGS | METH_KEYWORDS }, { "release", (PyCFunction) cxoSessionPool_release, METH_VARARGS | METH_KEYWORDS }, { NULL } @@ -684,6 +801,8 @@ static PyGetSetDef cxoCalcMembers[] = { (setter) cxoSessionPool_setGetMode, 0, 0 }, { "max_lifetime_session", (getter) cxoSessionPool_getMaxLifetimeSession, (setter) cxoSessionPool_setMaxLifetimeSession, 0, 0 }, + { "max_sessions_per_shard", (getter) cxoSessionPool_getMaxSessionsPerShard, + (setter) cxoSessionPool_setMaxSessionsPerShard, 0, 0 }, { "ping_interval", (getter) cxoSessionPool_getPingInterval, (setter) cxoSessionPool_setPingInterval, 0, 0 }, { "soda_metadata_cache", (getter) cxoSessionPool_getSodaMetadataCache, diff --git a/test/test_2400_session_pool.py b/test/test_2400_session_pool.py index eb9ae25..d4fac1a 100644 --- a/test/test_2400_session_pool.py +++ b/test/test_2400_session_pool.py @@ -339,5 +339,74 @@ class TestCase(test_env.BaseTestCase): self.assertEqual(pool.opened, 2, "opened (2)") pool.release(connection_3) + def test_2415_reconfigure_pool(self): + "2415 - test to ensure reconfigure() updates pool properties" + pool = test_env.get_pool(min=1, max=2, increment=1, + getmode=oracledb.SPOOL_ATTRVAL_WAIT) + self.assertEqual(pool.min, 1, "min (1)") + self.assertEqual(pool.max, 2, "max (2)") + self.assertEqual(pool.increment, 1, "increment (1)") + self.assertEqual(pool.getmode, oracledb.SPOOL_ATTRVAL_WAIT, + "getmode differs") + self.assertEqual(pool.timeout, 0, "timeout (0)") + self.assertEqual(pool.wait_timeout, 5000, "wait_timeout (5000)") + self.assertEqual(pool.max_lifetime_session, 0, + "max_lifetime_sessionmeout (0)") + self.assertEqual(pool.max_sessions_per_shard, 0, + "max_sessions_per_shard (0)") + self.assertEqual(pool.stmtcachesize, 20, "stmtcachesize (20)") + self.assertEqual(pool.ping_interval, 60, "ping_interval (60)") + + pool.reconfigure(min=2, max=5, increment=2, timeout=30, + getmode=oracledb.SPOOL_ATTRVAL_TIMEDWAIT, + wait_timeout=3000, max_lifetime_session=20, + max_sessions_per_shard=2, stmtcachesize=30, + ping_interval=30) + self.assertEqual(pool.min, 2, "min (2)") + self.assertEqual(pool.max, 5, "max (5)") + self.assertEqual(pool.increment, 2, "increment (2)") + self.assertEqual(pool.getmode, oracledb.SPOOL_ATTRVAL_TIMEDWAIT, + "getmode differs") + self.assertEqual(pool.timeout, 30, "timeout (30)") + self.assertEqual(pool.wait_timeout, 3000, "wait_timeout (3000)") + self.assertEqual(pool.max_lifetime_session, 20, + "max_lifetime_sessionmeout (20)") + self.assertEqual(pool.max_sessions_per_shard, 2, + "max_sessions_per_shard (2)") + self.assertEqual(pool.stmtcachesize, 30, "stmtcachesize (30)") + self.assertEqual(pool.ping_interval, 30, "ping_interval (30)") + + def test_2416_reconfigure_pool_with_missing_params(self): + "2416 - test to ensure reconfigure uses initial values if unspecified" + pool = test_env.get_pool(min=1, max=2, increment=1) + self.assertEqual(pool.min, 1, "min (1)") + self.assertEqual(pool.max, 2, "max (2)") + self.assertEqual(pool.increment, 1, "increment (1)") + self.assertEqual(pool.getmode, oracledb.SPOOL_ATTRVAL_NOWAIT, + "getmode differs") + self.assertEqual(pool.timeout, 0, "timeout (0)") + self.assertEqual(pool.wait_timeout, 5000, "wait_timeout (5000)") + self.assertEqual(pool.max_lifetime_session, 0, + "max_lifetime_sessionmeout (0)") + self.assertEqual(pool.max_sessions_per_shard, 0, + "max_sessions_per_shard (0)") + self.assertEqual(pool.stmtcachesize, 20, "stmtcachesize (20)") + self.assertEqual(pool.ping_interval, 60, "ping_interval (60)") + + pool.reconfigure(min=2, max=5, increment=2) + self.assertEqual(pool.min, 2, "min (2)") + self.assertEqual(pool.max, 5, "max (5)") + self.assertEqual(pool.increment, 2, "increment (2)") + self.assertEqual(pool.getmode, oracledb.SPOOL_ATTRVAL_NOWAIT, + "getmode differs") + self.assertEqual(pool.timeout, 0, "timeout (0)") + self.assertEqual(pool.wait_timeout, 5000, "wait_timeout (5000)") + self.assertEqual(pool.max_lifetime_session, 0, + "max_lifetime_sessionmeout (0)") + self.assertEqual(pool.max_sessions_per_shard, 0, + "max_sessions_per_shard (0)") + self.assertEqual(pool.stmtcachesize, 20, "stmtcachesize (20)") + self.assertEqual(pool.ping_interval, 60, "ping_interval (60)") + if __name__ == "__main__": test_env.run_test_cases()