Skip to content

Commit f239ca1

Browse files
dyemanovdyemanov
authored
Surface internal optimization modes (all rows vs first rows) at the SQL and configuration levels (#7405)
* Surface internal optimization modes (all rows vs first rows) at the SQL and configuration levels. * Add session-level control over the optimization strategy * More informative name as suggested by Adriano * Cost-based approach for the first-rows optimization mode
1 parent 5b14baa commit f239ca1

22 files changed

+225
-89
lines changed

‎builds/install/misc/firebird.conf

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,21 @@
469469
#InlineSortThreshold = 1000
470470

471471

472+
# ----------------------------
473+
# Defines whether queries should be optimized to retrieve the first records
474+
# as soon as possible rather than returning the whole dataset as soon as possible.
475+
# By default retrieval of all rows is implied by the optimizer.
476+
#
477+
# Can be overridden at the session level using the SET OPTIMIZE statement
478+
# or at the SQL statement level by using the OPTIMIZE FOR clause.
479+
#
480+
# Per-database configurable.
481+
#
482+
# Type: boolean
483+
#
484+
#OptimizeForFirstRows = false
485+
486+
472487
# ============================
473488
# Plugin settings
474489
# ============================

‎src/common/classes/Nullable.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,21 @@ template <typename T> class BaseNullable
7070
return (!specified && !o.specified) || (specified == o.specified && value == o.value);
7171
}
7272

73+
bool operator !=(const BaseNullable<T>& o) const
74+
{
75+
return !(*this == o);
76+
}
77+
7378
bool operator ==(const T& o) const
7479
{
7580
return specified && value == o;
7681
}
7782

83+
bool operator !=(const T& o) const
84+
{
85+
return !(*this == o);
86+
}
87+
7888
void operator =(const T& v)
7989
{
8090
this->value = v;

‎src/common/config/config.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ enum ConfigKey
192192
KEY_MAX_STATEMENT_CACHE_SIZE,
193193
KEY_PARALLEL_WORKERS,
194194
KEY_MAX_PARALLEL_WORKERS,
195+
KEY_OPTIMIZE_FOR_FIRST_ROWS,
195196
MAX_CONFIG_KEY // keep it last
196197
};
197198

@@ -310,7 +311,8 @@ constexpr ConfigEntry entries[MAX_CONFIG_KEY] =
310311
{TYPE_STRING, "TempTableDirectory", false, ""},
311312
{TYPE_INTEGER, "MaxStatementCacheSize", false, 2 * 1048576}, // bytes
312313
{TYPE_INTEGER, "ParallelWorkers", true, 1},
313-
{TYPE_INTEGER, "MaxParallelWorkers", true, 1}
314+
{TYPE_INTEGER, "MaxParallelWorkers", true, 1},
315+
{TYPE_BOOLEAN, "OptimizeForFirstRows", false, false}
314316
};
315317

316318

@@ -638,6 +640,8 @@ class Config : public RefCounted, public GlobalStorage
638640
CONFIG_GET_GLOBAL_INT(getParallelWorkers, KEY_PARALLEL_WORKERS);
639641

640642
CONFIG_GET_GLOBAL_INT(getMaxParallelWorkers, KEY_MAX_PARALLEL_WORKERS);
643+
644+
CONFIG_GET_PER_DB_BOOL(getOptimizeForFirstRows, KEY_OPTIMIZE_FOR_FIRST_ROWS);
641645
};
642646

643647
// Implementation of interface to access master configuration file

‎src/common/keywords.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,7 @@ static const TOK tokens[] =
339339
{TOK_ON, "ON", false},
340340
{TOK_ONLY, "ONLY", false},
341341
{TOK_OPEN, "OPEN", false},
342+
{TOK_OPTIMIZE, "OPTIMIZE", true},
342343
{TOK_OPTION, "OPTION", true},
343344
{TOK_OR, "OR", false},
344345
{TOK_ORDER, "ORDER", false},

‎src/dsql/BoolNodes.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1124,7 +1124,7 @@ BoolExprNode* ComparativeBoolNode::createRseNode(DsqlCompilerScratch* dsqlScratc
11241124
const DsqlContextStack::iterator baseDT(dsqlScratch->derivedContext);
11251125
const DsqlContextStack::iterator baseUnion(dsqlScratch->unionContext);
11261126

1127-
RseNode* rse = PASS1_rse(dsqlScratch, select_expr, false, false);
1127+
RseNode* rse = PASS1_rse(dsqlScratch, select_expr);
11281128
rse->flags |= RseNode::FLAG_DSQL_COMPARATIVE;
11291129

11301130
// Create a conjunct to be injected.
@@ -1709,7 +1709,7 @@ DmlNode* RseBoolNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch*
17091709
node->rse->flags |= RseNode::FLAG_SUB_QUERY;
17101710

17111711
if (blrOp == blr_any || blrOp == blr_exists) // maybe for blr_unique as well?
1712-
node->rse->flags |= RseNode::FLAG_OPT_FIRST_ROWS;
1712+
node->rse->firstRows = true;
17131713

17141714
if (csb->csb_currentForNode && csb->csb_currentForNode->parBlrBeginCnt <= 1)
17151715
node->ownSavepoint = false;
@@ -1744,7 +1744,7 @@ BoolExprNode* RseBoolNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
17441744
const DsqlContextStack::iterator base(*dsqlScratch->context);
17451745

17461746
RseBoolNode* node = FB_NEW_POOL(dsqlScratch->getPool()) RseBoolNode(dsqlScratch->getPool(), blrOp,
1747-
PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse), false, false));
1747+
PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse)));
17481748

17491749
// Finish off by cleaning up contexts
17501750
dsqlScratch->context->clear(base);

‎src/dsql/DdlNodes.epp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8683,7 +8683,7 @@ void CreateAlterViewNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScra
86838683

86848684
dsqlScratch->resetContextStack();
86858685
++dsqlScratch->contextNumber;
8686-
RseNode* rse = PASS1_rse(dsqlScratch, selectExpr, false, false);
8686+
RseNode* rse = PASS1_rse(dsqlScratch, selectExpr);
86878687

86888688
dsqlScratch->getBlrData().clear();
86898689
dsqlScratch->appendUChar(dsqlScratch->isVersion4() ? blr_version4 : blr_version5);

‎src/dsql/ExprNodes.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11208,7 +11208,7 @@ ValueExprNode* SubQueryNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
1120811208

1120911209
const DsqlContextStack::iterator base(*dsqlScratch->context);
1121011210

11211-
RseNode* rse = PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse), false, false);
11211+
RseNode* rse = PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse));
1121211212

1121311213
SubQueryNode* node = FB_NEW_POOL(dsqlScratch->getPool()) SubQueryNode(dsqlScratch->getPool(), blrOp, rse,
1121411214
rse->dsqlSelectList->items[0], NullNode::instance());

‎src/dsql/StmtNodes.cpp

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
#include "../dsql/gen_proto.h"
6464
#include "../dsql/make_proto.h"
6565
#include "../dsql/pass1_proto.h"
66+
#include "../dsql/DsqlStatementCache.h"
6667

6768
using namespace Firebird;
6869
using namespace Jrd;
@@ -1238,7 +1239,7 @@ DeclareCursorNode* DeclareCursorNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
12381239
dt->querySpec = dsqlSelect->dsqlExpr;
12391240
dt->alias = dsqlName.c_str();
12401241

1241-
rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect->dsqlWithLock, dsqlSelect->dsqlSkipLocked);
1242+
rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect);
12421243

12431244
// Assign number and store in the dsqlScratch stack.
12441245
cursorNumber = dsqlScratch->cursorNumber++;
@@ -4931,7 +4932,7 @@ ForNode* ForNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
49314932
dt->querySpec = dsqlSelect->dsqlExpr;
49324933
dt->alias = dsqlCursor->dsqlName.c_str();
49334934

4934-
node->rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect->dsqlWithLock, dsqlSelect->dsqlSkipLocked);
4935+
node->rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect);
49354936

49364937
dsqlCursor->rse = node->rse;
49374938
dsqlCursor->cursorNumber = dsqlScratch->cursorNumber++;
@@ -7528,7 +7529,7 @@ StmtNode* StoreNode::internalDsqlPass(DsqlCompilerScratch* dsqlScratch,
75287529
if (dsqlRse && dsqlScratch->isPsql() && dsqlReturning)
75297530
selExpr->dsqlFlags |= RecordSourceNode::DFLAG_SINGLETON;
75307531

7531-
RseNode* rse = PASS1_rse(dsqlScratch, selExpr, false, false);
7532+
RseNode* rse = PASS1_rse(dsqlScratch, selExpr);
75327533
node->dsqlRse = rse;
75337534
values = rse->dsqlSelectList;
75347535
needSavePoint = false;
@@ -8206,9 +8207,10 @@ SelectNode* SelectNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
82068207
{
82078208
SelectNode* node = FB_NEW_POOL(dsqlScratch->getPool()) SelectNode(dsqlScratch->getPool());
82088209
node->dsqlForUpdate = dsqlForUpdate;
8210+
node->dsqlOptimizeForFirstRows = dsqlOptimizeForFirstRows;
82098211

82108212
const DsqlContextStack::iterator base(*dsqlScratch->context);
8211-
node->dsqlRse = PASS1_rse(dsqlScratch, dsqlExpr, dsqlWithLock, dsqlSkipLocked);
8213+
node->dsqlRse = PASS1_rse(dsqlScratch, dsqlExpr, this);
82128214
dsqlScratch->context->clear(base);
82138215

82148216
if (dsqlForUpdate)
@@ -9170,6 +9172,25 @@ void SetSessionNode::execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** /*
91709172
//--------------------
91719173

91729174

9175+
void SetOptimizeNode::execute(thread_db* tdbb, DsqlRequest* /*request*/, jrd_tra** /*traHandle*/) const
9176+
{
9177+
const auto attachment = tdbb->getAttachment();
9178+
9179+
if (attachment->att_opt_first_rows != optimizeMode)
9180+
{
9181+
attachment->att_opt_first_rows = optimizeMode;
9182+
9183+
// Clear the local compiled statements cache to allow queries
9184+
// to be re-optimized accordingly to the new rules
9185+
9186+
attachment->att_dsql_instance->dbb_statement_cache->purge(tdbb, false);
9187+
}
9188+
}
9189+
9190+
9191+
//--------------------
9192+
9193+
91739194
void SetTimeZoneNode::execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** /*traHandle*/) const
91749195
{
91759196
Attachment* const attachment = tdbb->getAttachment();

‎src/dsql/StmtNodes.h

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1344,6 +1344,7 @@ class SelectNode final : public TypedNode<StmtNode, StmtNode::TYPE_SELECT>
13441344
bool dsqlForUpdate = false;
13451345
bool dsqlWithLock = false;
13461346
bool dsqlSkipLocked = false;
1347+
TriState dsqlOptimizeForFirstRows;
13471348
};
13481349

13491350

@@ -1840,6 +1841,37 @@ class SetBindNode : public SessionManagementNode
18401841
};
18411842

18421843

1844+
class SetOptimizeNode : public SessionManagementNode
1845+
{
1846+
public:
1847+
explicit SetOptimizeNode(MemoryPool& pool)
1848+
: SessionManagementNode(pool)
1849+
{
1850+
}
1851+
1852+
SetOptimizeNode(MemoryPool& pool, bool mode)
1853+
: SessionManagementNode(pool),
1854+
optimizeMode(mode)
1855+
{
1856+
}
1857+
1858+
public:
1859+
virtual Firebird::string internalPrint(NodePrinter& printer) const
1860+
{
1861+
SessionManagementNode::internalPrint(printer);
1862+
1863+
NODE_PRINT(printer, optimizeMode);
1864+
1865+
return "SetOptimizeNode";
1866+
}
1867+
1868+
virtual void execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** traHandle) const;
1869+
1870+
public:
1871+
TriState optimizeMode;
1872+
};
1873+
1874+
18431875
class SetTimeZoneNode : public SessionManagementNode
18441876
{
18451877
public:

‎src/dsql/gen.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,12 @@ void GEN_rse(DsqlCompilerScratch* dsqlScratch, RseNode* rse)
605605
gen_plan(dsqlScratch, rse->rse_plan);
606606
}
607607

608+
if (rse->firstRows.isAssigned())
609+
{
610+
dsqlScratch->appendUChar(blr_optimize);
611+
dsqlScratch->appendUChar(rse->firstRows.value);
612+
}
613+
608614
dsqlScratch->appendUChar(blr_end);
609615
}
610616

‎src/dsql/parse-conflicts.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
68 shift/reduce conflicts, 19 reduce/reduce conflicts.
1+
69 shift/reduce conflicts, 21 reduce/reduce conflicts.

‎src/dsql/parse.y

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -689,6 +689,7 @@ using namespace Firebird;
689689
// tokens added for Firebird 5.0
690690

691691
%token <metaNamePtr> LOCKED
692+
%token <metaNamePtr> OPTIMIZE
692693
%token <metaNamePtr> QUARTER
693694
%token <metaNamePtr> TARGET
694695
%token <metaNamePtr> TIMEZONE_NAME
@@ -918,6 +919,7 @@ mng_statement
918919
| session_reset { $$ = $1; }
919920
| set_time_zone { $$ = $1; }
920921
| set_bind { $$ = $1; }
922+
| set_optimize { $$ = $1; }
921923
;
922924

923925

@@ -5481,6 +5483,14 @@ decfloat_trap($setDecFloatTrapsNode)
54815483
{ $setDecFloatTrapsNode->trap($1); }
54825484
;
54835485

5486+
%type <mngNode> set_optimize
5487+
set_optimize
5488+
: SET OPTIMIZE optimize_mode
5489+
{ $$ = newNode<SetOptimizeNode>($3); }
5490+
| SET OPTIMIZE TO DEFAULT
5491+
{ $$ = newNode<SetOptimizeNode>(); }
5492+
;
5493+
54845494
%type <setSessionNode> session_statement
54855495
session_statement
54865496
: SET SESSION IDLE TIMEOUT long_integer timepart_sesion_idle_tout
@@ -5764,13 +5774,14 @@ ddl_desc
57645774

57655775
%type <selectNode> select
57665776
select
5767-
: select_expr for_update_clause lock_clause
5777+
: select_expr for_update_clause lock_clause optimize_clause
57685778
{
57695779
SelectNode* node = newNode<SelectNode>();
57705780
node->dsqlExpr = $1;
57715781
node->dsqlForUpdate = $2;
57725782
node->dsqlWithLock = $3.first;
57735783
node->dsqlSkipLocked = $3.second;
5784+
node->dsqlOptimizeForFirstRows = $4;
57745785
$$ = node;
57755786
}
57765787
;
@@ -5799,6 +5810,22 @@ skip_locked_clause_opt
57995810
| SKIP LOCKED { $$ = true; }
58005811
;
58015812

5813+
%type <nullableBoolVal> optimize_clause
5814+
optimize_clause
5815+
: OPTIMIZE optimize_mode
5816+
{ $$ = Nullable<bool>::val($2); }
5817+
| // nothing
5818+
{ $$ = Nullable<bool>::empty(); }
5819+
;
5820+
5821+
%type <boolVal> optimize_mode
5822+
optimize_mode
5823+
: FOR FIRST ROWS
5824+
{ $$ = true; }
5825+
| FOR ALL ROWS
5826+
{ $$ = false; }
5827+
;
5828+
58025829

58035830
// SELECT expression
58045831

@@ -9186,6 +9213,7 @@ non_reserved_word
91869213
| BLOB_APPEND
91879214
// added in FB 5.0
91889215
| LOCKED
9216+
| OPTIMIZE
91899217
| QUARTER
91909218
| TARGET
91919219
| TIMEZONE_NAME

‎src/dsql/pass1.cpp

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -578,15 +578,23 @@ dsql_ctx* PASS1_make_context(DsqlCompilerScratch* dsqlScratch, RecordSourceNode*
578578

579579
// Compile a record selection expression, bumping up the statement scope level everytime an rse is
580580
// seen. The scope level controls parsing of aliases.
581-
RseNode* PASS1_rse(DsqlCompilerScratch* dsqlScratch, SelectExprNode* input, bool updateLock, bool skipLocked)
581+
RseNode* PASS1_rse(DsqlCompilerScratch* dsqlScratch,
582+
SelectExprNode* input,
583+
const SelectNode* select)
582584
{
583585
DEV_BLKCHK(dsqlScratch, dsql_type_req);
584586
DEV_BLKCHK(input, dsql_type_nod);
585587

588+
const bool updateLock = select ? select->dsqlWithLock : false;
589+
const bool skipLocked = select ? select->dsqlSkipLocked : false;
590+
586591
dsqlScratch->scopeLevel++;
587592
RseNode* node = pass1_rse(dsqlScratch, input, NULL, NULL, updateLock, skipLocked, 0);
588593
dsqlScratch->scopeLevel--;
589594

595+
if (select)
596+
node->firstRows = select->dsqlOptimizeForFirstRows;
597+
590598
return node;
591599
}
592600

@@ -975,7 +983,7 @@ void PASS1_expand_contexts(DsqlContextStack& contexts, dsql_ctx* context)
975983

976984
// Process derived table which is part of a from clause.
977985
RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* input,
978-
const char* cte_alias, bool updateLock, bool skipLocked)
986+
const char* cte_alias, const SelectNode* select)
979987
{
980988
DEV_BLKCHK(dsqlScratch, dsql_type_req);
981989

@@ -1076,7 +1084,7 @@ RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* i
10761084
rse = pass1_union(dsqlScratch, unionExpr, NULL, NULL, false, false, 0);
10771085
}
10781086
else
1079-
rse = PASS1_rse(dsqlScratch, input, updateLock, skipLocked);
1087+
rse = PASS1_rse(dsqlScratch, input, select);
10801088

10811089
// Finish off by cleaning up contexts and put them into derivedContext
10821090
// so create view (ddl) can deal with it.
@@ -1237,7 +1245,7 @@ RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* i
12371245
dsqlScratch->currCteAlias ? *dsqlScratch->currCteAlias : NULL;
12381246
dsqlScratch->resetCTEAlias(alias);
12391247

1240-
rse = PASS1_rse(dsqlScratch, input, updateLock, skipLocked);
1248+
rse = PASS1_rse(dsqlScratch, input, select);
12411249

12421250
if (saveCteAlias)
12431251
dsqlScratch->resetCTEAlias(*saveCteAlias);

0 commit comments

Comments
 (0)