000001 /*
000002 ** 2001 September 15
000003 **
000004 ** The author disclaims copyright to this source code. In place of
000005 ** a legal notice, here is a blessing:
000006 **
000007 ** May you do good and not evil.
000008 ** May you find forgiveness for yourself and forgive others.
000009 ** May you share freely, never taking more than you give.
000010 **
000011 *************************************************************************
000012 ** This module contains C code that generates VDBE code used to process
000013 ** the WHERE clause of SQL statements. This module is responsible for
000014 ** generating the code that loops through a table looking for applicable
000015 ** rows. Indices are selected and used to speed the search when doing
000016 ** so is applicable. Because this module is responsible for selecting
000017 ** indices, you might also think of this module as the "query optimizer".
000018 */
000019 #include "sqliteInt.h"
000020 #include "whereInt.h"
000021
000022 /*
000023 ** Extra information appended to the end of sqlite3_index_info but not
000024 ** visible to the xBestIndex function, at least not directly. The
000025 ** sqlite3_vtab_collation() interface knows how to reach it, however.
000026 **
000027 ** This object is not an API and can be changed from one release to the
000028 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
000029 ** agree on the structure, all will be well.
000030 */
000031 typedef struct HiddenIndexInfo HiddenIndexInfo;
000032 struct HiddenIndexInfo {
000033 WhereClause *pWC; /* The Where clause being analyzed */
000034 Parse *pParse; /* The parsing context */
000035 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
000036 u32 mIn; /* Mask of terms that are <col> IN (...) */
000037 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */
000038 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
000039 ** because extra space is allocated to hold up
000040 ** to nTerm such values */
000041 };
000042
000043 /* Forward declaration of methods */
000044 static int whereLoopResize(sqlite3*, WhereLoop*, int);
000045
000046 /*
000047 ** Return the estimated number of output rows from a WHERE clause
000048 */
000049 LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
000050 return pWInfo->nRowOut;
000051 }
000052
000053 /*
000054 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
000055 ** WHERE clause returns outputs for DISTINCT processing.
000056 */
000057 int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
000058 return pWInfo->eDistinct;
000059 }
000060
000061 /*
000062 ** Return the number of ORDER BY terms that are satisfied by the
000063 ** WHERE clause. A return of 0 means that the output must be
000064 ** completely sorted. A return equal to the number of ORDER BY
000065 ** terms means that no sorting is needed at all. A return that
000066 ** is positive but less than the number of ORDER BY terms means that
000067 ** block sorting is required.
000068 */
000069 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
000070 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
000071 }
000072
000073 /*
000074 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known
000075 ** to emit rows in increasing order, and if the last row emitted by the
000076 ** inner-most loop did not fit within the sorter, then we can skip all
000077 ** subsequent rows for the current iteration of the inner loop (because they
000078 ** will not fit in the sorter either) and continue with the second inner
000079 ** loop - the loop immediately outside the inner-most.
000080 **
000081 ** When a row does not fit in the sorter (because the sorter already
000082 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
000083 ** label returned by this function.
000084 **
000085 ** If the ORDER BY LIMIT optimization applies, the jump destination should
000086 ** be the continuation for the second-inner-most loop. If the ORDER BY
000087 ** LIMIT optimization does not apply, then the jump destination should
000088 ** be the continuation for the inner-most loop.
000089 **
000090 ** It is always safe for this routine to return the continuation of the
000091 ** inner-most loop, in the sense that a correct answer will result.
000092 ** Returning the continuation the second inner loop is an optimization
000093 ** that might make the code run a little faster, but should not change
000094 ** the final answer.
000095 */
000096 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
000097 WhereLevel *pInner;
000098 if( !pWInfo->bOrderedInnerLoop ){
000099 /* The ORDER BY LIMIT optimization does not apply. Jump to the
000100 ** continuation of the inner-most loop. */
000101 return pWInfo->iContinue;
000102 }
000103 pInner = &pWInfo->a[pWInfo->nLevel-1];
000104 assert( pInner->addrNxt!=0 );
000105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt;
000106 }
000107
000108 /*
000109 ** While generating code for the min/max optimization, after handling
000110 ** the aggregate-step call to min() or max(), check to see if any
000111 ** additional looping is required. If the output order is such that
000112 ** we are certain that the correct answer has already been found, then
000113 ** code an OP_Goto to by pass subsequent processing.
000114 **
000115 ** Any extra OP_Goto that is coded here is an optimization. The
000116 ** correct answer should be obtained regardless. This OP_Goto just
000117 ** makes the answer appear faster.
000118 */
000119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){
000120 WhereLevel *pInner;
000121 int i;
000122 if( !pWInfo->bOrderedInnerLoop ) return;
000123 if( pWInfo->nOBSat==0 ) return;
000124 for(i=pWInfo->nLevel-1; i>=0; i--){
000125 pInner = &pWInfo->a[i];
000126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){
000127 sqlite3VdbeGoto(v, pInner->addrNxt);
000128 return;
000129 }
000130 }
000131 sqlite3VdbeGoto(v, pWInfo->iBreak);
000132 }
000133
000134 /*
000135 ** Return the VDBE address or label to jump to in order to continue
000136 ** immediately with the next row of a WHERE clause.
000137 */
000138 int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
000139 assert( pWInfo->iContinue!=0 );
000140 return pWInfo->iContinue;
000141 }
000142
000143 /*
000144 ** Return the VDBE address or label to jump to in order to break
000145 ** out of a WHERE loop.
000146 */
000147 int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
000148 return pWInfo->iBreak;
000149 }
000150
000151 /*
000152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
000153 ** operate directly on the rowids returned by a WHERE clause. Return
000154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
000155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
000156 ** optimization can be used on multiple
000157 **
000158 ** If the ONEPASS optimization is used (if this routine returns true)
000159 ** then also write the indices of open cursors used by ONEPASS
000160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
000161 ** table and iaCur[1] gets the cursor used by an auxiliary index.
000162 ** Either value may be -1, indicating that cursor is not used.
000163 ** Any cursors returned will have been opened for writing.
000164 **
000165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
000166 ** unable to use the ONEPASS optimization.
000167 */
000168 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
000169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
000170 #ifdef WHERETRACE_ENABLED
000171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
000172 sqlite3DebugPrintf("%s cursors: %d %d\n",
000173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
000174 aiCur[0], aiCur[1]);
000175 }
000176 #endif
000177 return pWInfo->eOnePass;
000178 }
000179
000180 /*
000181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
000182 ** the data cursor to the row selected by the index cursor.
000183 */
000184 int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){
000185 return pWInfo->bDeferredSeek;
000186 }
000187
000188 /*
000189 ** Move the content of pSrc into pDest
000190 */
000191 static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
000192 pDest->n = pSrc->n;
000193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
000194 }
000195
000196 /*
000197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
000198 **
000199 ** The new entry might overwrite an existing entry, or it might be
000200 ** appended, or it might be discarded. Do whatever is the right thing
000201 ** so that pSet keeps the N_OR_COST best entries seen so far.
000202 */
000203 static int whereOrInsert(
000204 WhereOrSet *pSet, /* The WhereOrSet to be updated */
000205 Bitmask prereq, /* Prerequisites of the new entry */
000206 LogEst rRun, /* Run-cost of the new entry */
000207 LogEst nOut /* Number of outputs for the new entry */
000208 ){
000209 u16 i;
000210 WhereOrCost *p;
000211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
000212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
000213 goto whereOrInsert_done;
000214 }
000215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
000216 return 0;
000217 }
000218 }
000219 if( pSet->n<N_OR_COST ){
000220 p = &pSet->a[pSet->n++];
000221 p->nOut = nOut;
000222 }else{
000223 p = pSet->a;
000224 for(i=1; i<pSet->n; i++){
000225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
000226 }
000227 if( p->rRun<=rRun ) return 0;
000228 }
000229 whereOrInsert_done:
000230 p->prereq = prereq;
000231 p->rRun = rRun;
000232 if( p->nOut>nOut ) p->nOut = nOut;
000233 return 1;
000234 }
000235
000236 /*
000237 ** Return the bitmask for the given cursor number. Return 0 if
000238 ** iCursor is not in the set.
000239 */
000240 Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
000241 int i;
000242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
000243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 );
000244 assert( iCursor>=-1 );
000245 if( pMaskSet->ix[0]==iCursor ){
000246 return 1;
000247 }
000248 for(i=1; i<pMaskSet->n; i++){
000249 if( pMaskSet->ix[i]==iCursor ){
000250 return MASKBIT(i);
000251 }
000252 }
000253 return 0;
000254 }
000255
000256 /* Allocate memory that is automatically freed when pWInfo is freed.
000257 */
000258 void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){
000259 WhereMemBlock *pBlock;
000260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock));
000261 if( pBlock ){
000262 pBlock->pNext = pWInfo->pMemToFree;
000263 pBlock->sz = nByte;
000264 pWInfo->pMemToFree = pBlock;
000265 pBlock++;
000266 }
000267 return (void*)pBlock;
000268 }
000269 void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){
000270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte);
000271 if( pNew && pOld ){
000272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld;
000273 pOldBlk--;
000274 assert( pOldBlk->sz<nByte );
000275 memcpy(pNew, pOld, pOldBlk->sz);
000276 }
000277 return pNew;
000278 }
000279
000280 /*
000281 ** Create a new mask for cursor iCursor.
000282 **
000283 ** There is one cursor per table in the FROM clause. The number of
000284 ** tables in the FROM clause is limited by a test early in the
000285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
000286 ** array will never overflow.
000287 */
000288 static void createMask(WhereMaskSet *pMaskSet, int iCursor){
000289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
000290 pMaskSet->ix[pMaskSet->n++] = iCursor;
000291 }
000292
000293 /*
000294 ** If the right-hand branch of the expression is a TK_COLUMN, then return
000295 ** a pointer to the right-hand branch. Otherwise, return NULL.
000296 */
000297 static Expr *whereRightSubexprIsColumn(Expr *p){
000298 p = sqlite3ExprSkipCollateAndLikely(p->pRight);
000299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){
000300 return p;
000301 }
000302 return 0;
000303 }
000304
000305 /*
000306 ** Term pTerm is guaranteed to be a WO_IN term. It may be a component term
000307 ** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)".
000308 ** This function checks to see if the term is compatible with an index
000309 ** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so,
000310 ** it returns a pointer to the name of the collation sequence (e.g. "BINARY"
000311 ** or "NOCASE") used by the comparison in pTerm. If it is not compatible
000312 ** with affinity idxaff, NULL is returned.
000313 */
000314 static SQLITE_NOINLINE const char *indexInAffinityOk(
000315 Parse *pParse,
000316 WhereTerm *pTerm,
000317 u8 idxaff
000318 ){
000319 Expr *pX = pTerm->pExpr;
000320 Expr inexpr;
000321
000322 assert( pTerm->eOperator & WO_IN );
000323
000324 if( sqlite3ExprIsVector(pX->pLeft) ){
000325 int iField = pTerm->u.x.iField - 1;
000326 inexpr.flags = 0;
000327 inexpr.op = TK_EQ;
000328 inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr;
000329 assert( ExprUseXSelect(pX) );
000330 inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr;
000331 pX = &inexpr;
000332 }
000333
000334 if( sqlite3IndexAffinityOk(pX, idxaff) ){
000335 CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX);
000336 return pRet ? pRet->zName : sqlite3StrBINARY;
000337 }
000338 return 0;
000339 }
000340
000341 /*
000342 ** Advance to the next WhereTerm that matches according to the criteria
000343 ** established when the pScan object was initialized by whereScanInit().
000344 ** Return NULL if there are no more matching WhereTerms.
000345 */
000346 static WhereTerm *whereScanNext(WhereScan *pScan){
000347 int iCur; /* The cursor on the LHS of the term */
000348 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
000349 Expr *pX; /* An expression being tested */
000350 WhereClause *pWC; /* Shorthand for pScan->pWC */
000351 WhereTerm *pTerm; /* The term being tested */
000352 int k = pScan->k; /* Where to start scanning */
000353
000354 assert( pScan->iEquiv<=pScan->nEquiv );
000355 pWC = pScan->pWC;
000356 while(1){
000357 iColumn = pScan->aiColumn[pScan->iEquiv-1];
000358 iCur = pScan->aiCur[pScan->iEquiv-1];
000359 assert( pWC!=0 );
000360 assert( iCur>=0 );
000361 do{
000362 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
000363 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 );
000364 if( pTerm->leftCursor==iCur
000365 && pTerm->u.x.leftColumn==iColumn
000366 && (iColumn!=XN_EXPR
000367 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
000368 pScan->pIdxExpr,iCur)==0)
000369 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON))
000370 ){
000371 if( (pTerm->eOperator & WO_EQUIV)!=0
000372 && pScan->nEquiv<ArraySize(pScan->aiCur)
000373 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
000374 ){
000375 int j;
000376 for(j=0; j<pScan->nEquiv; j++){
000377 if( pScan->aiCur[j]==pX->iTable
000378 && pScan->aiColumn[j]==pX->iColumn ){
000379 break;
000380 }
000381 }
000382 if( j==pScan->nEquiv ){
000383 pScan->aiCur[j] = pX->iTable;
000384 pScan->aiColumn[j] = pX->iColumn;
000385 pScan->nEquiv++;
000386 }
000387 }
000388 if( (pTerm->eOperator & pScan->opMask)!=0 ){
000389 /* Verify the affinity and collating sequence match */
000390 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
000391 const char *zCollName;
000392 Parse *pParse = pWC->pWInfo->pParse;
000393 pX = pTerm->pExpr;
000394
000395 if( (pTerm->eOperator & WO_IN) ){
000396 zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff);
000397 if( !zCollName ) continue;
000398 }else{
000399 CollSeq *pColl;
000400 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
000401 continue;
000402 }
000403 assert(pX->pLeft);
000404 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
000405 zCollName = pColl ? pColl->zName : sqlite3StrBINARY;
000406 }
000407
000408 if( sqlite3StrICmp(zCollName, pScan->zCollName) ){
000409 continue;
000410 }
000411 }
000412 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
000413 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0))
000414 && pX->op==TK_COLUMN
000415 && pX->iTable==pScan->aiCur[0]
000416 && pX->iColumn==pScan->aiColumn[0]
000417 ){
000418 testcase( pTerm->eOperator & WO_IS );
000419 continue;
000420 }
000421 pScan->pWC = pWC;
000422 pScan->k = k+1;
000423 #ifdef WHERETRACE_ENABLED
000424 if( sqlite3WhereTrace & 0x20000 ){
000425 int ii;
000426 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
000427 pTerm, pScan->nEquiv);
000428 for(ii=0; ii<pScan->nEquiv; ii++){
000429 sqlite3DebugPrintf(" {%d:%d}",
000430 pScan->aiCur[ii], pScan->aiColumn[ii]);
000431 }
000432 sqlite3DebugPrintf("\n");
000433 }
000434 #endif
000435 return pTerm;
000436 }
000437 }
000438 }
000439 pWC = pWC->pOuter;
000440 k = 0;
000441 }while( pWC!=0 );
000442 if( pScan->iEquiv>=pScan->nEquiv ) break;
000443 pWC = pScan->pOrigWC;
000444 k = 0;
000445 pScan->iEquiv++;
000446 }
000447 return 0;
000448 }
000449
000450 /*
000451 ** This is whereScanInit() for the case of an index on an expression.
000452 ** It is factored out into a separate tail-recursion subroutine so that
000453 ** the normal whereScanInit() routine, which is a high-runner, does not
000454 ** need to push registers onto the stack as part of its prologue.
000455 */
000456 static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){
000457 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr);
000458 return whereScanNext(pScan);
000459 }
000460
000461 /*
000462 ** Initialize a WHERE clause scanner object. Return a pointer to the
000463 ** first match. Return NULL if there are no matches.
000464 **
000465 ** The scanner will be searching the WHERE clause pWC. It will look
000466 ** for terms of the form "X <op> <expr>" where X is column iColumn of table
000467 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
000468 ** must be one of the indexes of table iCur.
000469 **
000470 ** The <op> must be one of the operators described by opMask.
000471 **
000472 ** If the search is for X and the WHERE clause contains terms of the
000473 ** form X=Y then this routine might also return terms of the form
000474 ** "Y <op> <expr>". The number of levels of transitivity is limited,
000475 ** but is enough to handle most commonly occurring SQL statements.
000476 **
000477 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with
000478 ** index pIdx.
000479 */
000480 static WhereTerm *whereScanInit(
000481 WhereScan *pScan, /* The WhereScan object being initialized */
000482 WhereClause *pWC, /* The WHERE clause to be scanned */
000483 int iCur, /* Cursor to scan for */
000484 int iColumn, /* Column to scan for */
000485 u32 opMask, /* Operator(s) to scan for */
000486 Index *pIdx /* Must be compatible with this index */
000487 ){
000488 pScan->pOrigWC = pWC;
000489 pScan->pWC = pWC;
000490 pScan->pIdxExpr = 0;
000491 pScan->idxaff = 0;
000492 pScan->zCollName = 0;
000493 pScan->opMask = opMask;
000494 pScan->k = 0;
000495 pScan->aiCur[0] = iCur;
000496 pScan->nEquiv = 1;
000497 pScan->iEquiv = 1;
000498 if( pIdx ){
000499 int j = iColumn;
000500 iColumn = pIdx->aiColumn[j];
000501 if( iColumn==pIdx->pTable->iPKey ){
000502 iColumn = XN_ROWID;
000503 }else if( iColumn>=0 ){
000504 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
000505 pScan->zCollName = pIdx->azColl[j];
000506 }else if( iColumn==XN_EXPR ){
000507 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
000508 pScan->zCollName = pIdx->azColl[j];
000509 pScan->aiColumn[0] = XN_EXPR;
000510 return whereScanInitIndexExpr(pScan);
000511 }
000512 }else if( iColumn==XN_EXPR ){
000513 return 0;
000514 }
000515 pScan->aiColumn[0] = iColumn;
000516 return whereScanNext(pScan);
000517 }
000518
000519 /*
000520 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
000521 ** where X is a reference to the iColumn of table iCur or of index pIdx
000522 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
000523 ** the op parameter. Return a pointer to the term. Return 0 if not found.
000524 **
000525 ** If pIdx!=0 then it must be one of the indexes of table iCur.
000526 ** Search for terms matching the iColumn-th column of pIdx
000527 ** rather than the iColumn-th column of table iCur.
000528 **
000529 ** The term returned might by Y=<expr> if there is another constraint in
000530 ** the WHERE clause that specifies that X=Y. Any such constraints will be
000531 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
000532 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
000533 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
000534 ** other equivalent values. Hence a search for X will return <expr> if X=A1
000535 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
000536 **
000537 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
000538 ** then try for the one with no dependencies on <expr> - in other words where
000539 ** <expr> is a constant expression of some kind. Only return entries of
000540 ** the form "X <op> Y" where Y is a column in another table if no terms of
000541 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
000542 ** exist, try to return a term that does not use WO_EQUIV.
000543 */
000544 WhereTerm *sqlite3WhereFindTerm(
000545 WhereClause *pWC, /* The WHERE clause to be searched */
000546 int iCur, /* Cursor number of LHS */
000547 int iColumn, /* Column number of LHS */
000548 Bitmask notReady, /* RHS must not overlap with this mask */
000549 u32 op, /* Mask of WO_xx values describing operator */
000550 Index *pIdx /* Must be compatible with this index, if not NULL */
000551 ){
000552 WhereTerm *pResult = 0;
000553 WhereTerm *p;
000554 WhereScan scan;
000555
000556 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
000557 op &= WO_EQ|WO_IS;
000558 while( p ){
000559 if( (p->prereqRight & notReady)==0 ){
000560 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
000561 testcase( p->eOperator & WO_IS );
000562 return p;
000563 }
000564 if( pResult==0 ) pResult = p;
000565 }
000566 p = whereScanNext(&scan);
000567 }
000568 return pResult;
000569 }
000570
000571 /*
000572 ** This function searches pList for an entry that matches the iCol-th column
000573 ** of index pIdx.
000574 **
000575 ** If such an expression is found, its index in pList->a[] is returned. If
000576 ** no expression is found, -1 is returned.
000577 */
000578 static int findIndexCol(
000579 Parse *pParse, /* Parse context */
000580 ExprList *pList, /* Expression list to search */
000581 int iBase, /* Cursor for table associated with pIdx */
000582 Index *pIdx, /* Index to match column of */
000583 int iCol /* Column of index to match */
000584 ){
000585 int i;
000586 const char *zColl = pIdx->azColl[iCol];
000587
000588 for(i=0; i<pList->nExpr; i++){
000589 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
000590 if( ALWAYS(p!=0)
000591 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
000592 && p->iColumn==pIdx->aiColumn[iCol]
000593 && p->iTable==iBase
000594 ){
000595 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
000596 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
000597 return i;
000598 }
000599 }
000600 }
000601
000602 return -1;
000603 }
000604
000605 /*
000606 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL
000607 */
000608 static int indexColumnNotNull(Index *pIdx, int iCol){
000609 int j;
000610 assert( pIdx!=0 );
000611 assert( iCol>=0 && iCol<pIdx->nColumn );
000612 j = pIdx->aiColumn[iCol];
000613 if( j>=0 ){
000614 return pIdx->pTable->aCol[j].notNull;
000615 }else if( j==(-1) ){
000616 return 1;
000617 }else{
000618 assert( j==(-2) );
000619 return 0; /* Assume an indexed expression can always yield a NULL */
000620
000621 }
000622 }
000623
000624 /*
000625 ** Return true if the DISTINCT expression-list passed as the third argument
000626 ** is redundant.
000627 **
000628 ** A DISTINCT list is redundant if any subset of the columns in the
000629 ** DISTINCT list are collectively unique and individually non-null.
000630 */
000631 static int isDistinctRedundant(
000632 Parse *pParse, /* Parsing context */
000633 SrcList *pTabList, /* The FROM clause */
000634 WhereClause *pWC, /* The WHERE clause */
000635 ExprList *pDistinct /* The result set that needs to be DISTINCT */
000636 ){
000637 Table *pTab;
000638 Index *pIdx;
000639 int i;
000640 int iBase;
000641
000642 /* If there is more than one table or sub-select in the FROM clause of
000643 ** this query, then it will not be possible to show that the DISTINCT
000644 ** clause is redundant. */
000645 if( pTabList->nSrc!=1 ) return 0;
000646 iBase = pTabList->a[0].iCursor;
000647 pTab = pTabList->a[0].pSTab;
000648
000649 /* If any of the expressions is an IPK column on table iBase, then return
000650 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
000651 ** current SELECT is a correlated sub-query.
000652 */
000653 for(i=0; i<pDistinct->nExpr; i++){
000654 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
000655 if( NEVER(p==0) ) continue;
000656 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
000657 if( p->iTable==iBase && p->iColumn<0 ) return 1;
000658 }
000659
000660 /* Loop through all indices on the table, checking each to see if it makes
000661 ** the DISTINCT qualifier redundant. It does so if:
000662 **
000663 ** 1. The index is itself UNIQUE, and
000664 **
000665 ** 2. All of the columns in the index are either part of the pDistinct
000666 ** list, or else the WHERE clause contains a term of the form "col=X",
000667 ** where X is a constant value. The collation sequences of the
000668 ** comparison and select-list expressions must match those of the index.
000669 **
000670 ** 3. All of those index columns for which the WHERE clause does not
000671 ** contain a "col=X" term are subject to a NOT NULL constraint.
000672 */
000673 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
000674 if( !IsUniqueIndex(pIdx) ) continue;
000675 if( pIdx->pPartIdxWhere ) continue;
000676 for(i=0; i<pIdx->nKeyCol; i++){
000677 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
000678 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
000679 if( indexColumnNotNull(pIdx, i)==0 ) break;
000680 }
000681 }
000682 if( i==pIdx->nKeyCol ){
000683 /* This index implies that the DISTINCT qualifier is redundant. */
000684 return 1;
000685 }
000686 }
000687
000688 return 0;
000689 }
000690
000691
000692 /*
000693 ** Estimate the logarithm of the input value to base 2.
000694 */
000695 static LogEst estLog(LogEst N){
000696 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
000697 }
000698
000699 /*
000700 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
000701 **
000702 ** This routine runs over generated VDBE code and translates OP_Column
000703 ** opcodes into OP_Copy when the table is being accessed via co-routine
000704 ** instead of via table lookup.
000705 **
000706 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
000707 ** cursor iTabCur are transformed into OP_Sequence opcode for the
000708 ** iAutoidxCur cursor, in order to generate unique rowids for the
000709 ** automatic index being generated.
000710 */
000711 static void translateColumnToCopy(
000712 Parse *pParse, /* Parsing context */
000713 int iStart, /* Translate from this opcode to the end */
000714 int iTabCur, /* OP_Column/OP_Rowid references to this table */
000715 int iRegister, /* The first column is in this register */
000716 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */
000717 ){
000718 Vdbe *v = pParse->pVdbe;
000719 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
000720 int iEnd = sqlite3VdbeCurrentAddr(v);
000721 if( pParse->db->mallocFailed ) return;
000722 #ifdef SQLITE_DEBUG
000723 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000724 printf("CHECKING for column-to-copy on cursor %d for %d..%d\n",
000725 iTabCur, iStart, iEnd);
000726 }
000727 #endif
000728 for(; iStart<iEnd; iStart++, pOp++){
000729 if( pOp->p1!=iTabCur ) continue;
000730 if( pOp->opcode==OP_Column ){
000731 #ifdef SQLITE_DEBUG
000732 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000733 printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart);
000734 }
000735 #endif
000736 pOp->opcode = OP_Copy;
000737 pOp->p1 = pOp->p2 + iRegister;
000738 pOp->p2 = pOp->p3;
000739 pOp->p3 = 0;
000740 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */
000741 }else if( pOp->opcode==OP_Rowid ){
000742 #ifdef SQLITE_DEBUG
000743 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000744 printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart);
000745 }
000746 #endif
000747 pOp->opcode = OP_Sequence;
000748 pOp->p1 = iAutoidxCur;
000749 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
000750 if( iAutoidxCur==0 ){
000751 pOp->opcode = OP_Null;
000752 pOp->p3 = 0;
000753 }
000754 #endif
000755 }
000756 }
000757 }
000758
000759 /*
000760 ** Two routines for printing the content of an sqlite3_index_info
000761 ** structure. Used for testing and debugging only. If neither
000762 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
000763 ** are no-ops.
000764 */
000765 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
000766 static void whereTraceIndexInfoInputs(
000767 sqlite3_index_info *p, /* The IndexInfo object */
000768 Table *pTab /* The TABLE that is the virtual table */
000769 ){
000770 int i;
000771 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000772 sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName);
000773 for(i=0; i<p->nConstraint; i++){
000774 sqlite3DebugPrintf(
000775 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
000776 i,
000777 p->aConstraint[i].iColumn,
000778 p->aConstraint[i].iTermOffset,
000779 p->aConstraint[i].op,
000780 p->aConstraint[i].usable,
000781 sqlite3_vtab_collation(p,i));
000782 }
000783 for(i=0; i<p->nOrderBy; i++){
000784 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
000785 i,
000786 p->aOrderBy[i].iColumn,
000787 p->aOrderBy[i].desc);
000788 }
000789 }
000790 static void whereTraceIndexInfoOutputs(
000791 sqlite3_index_info *p, /* The IndexInfo object */
000792 Table *pTab /* The TABLE that is the virtual table */
000793 ){
000794 int i;
000795 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000796 sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName);
000797 for(i=0; i<p->nConstraint; i++){
000798 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
000799 i,
000800 p->aConstraintUsage[i].argvIndex,
000801 p->aConstraintUsage[i].omit);
000802 }
000803 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
000804 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
000805 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
000806 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
000807 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
000808 }
000809 #else
000810 #define whereTraceIndexInfoInputs(A,B)
000811 #define whereTraceIndexInfoOutputs(A,B)
000812 #endif
000813
000814 /*
000815 ** We know that pSrc is an operand of an outer join. Return true if
000816 ** pTerm is a constraint that is compatible with that join.
000817 **
000818 ** pTerm must be EP_OuterON if pSrc is the right operand of an
000819 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
000820 ** is the left operand of a RIGHT join.
000821 **
000822 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
000823 ** for an example of a WHERE clause constraints that may not be used on
000824 ** the right table of a RIGHT JOIN because the constraint implies a
000825 ** not-NULL condition on the left table of the RIGHT JOIN.
000826 */
000827 static int constraintCompatibleWithOuterJoin(
000828 const WhereTerm *pTerm, /* WHERE clause term to check */
000829 const SrcItem *pSrc /* Table we are trying to access */
000830 ){
000831 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
000832 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
000833 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
000834 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
000835 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
000836 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
000837 || pTerm->pExpr->w.iJoin != pSrc->iCursor
000838 ){
000839 return 0;
000840 }
000841 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
000842 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
000843 ){
000844 return 0;
000845 }
000846 return 1;
000847 }
000848
000849 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000850 /*
000851 ** Return true if column iCol of table pTab seem like it might be a
000852 ** good column to use as part of a query-time index.
000853 **
000854 ** Current algorithm (subject to improvement!):
000855 **
000856 ** 1. If iCol is already the left-most column of some other index,
000857 ** then return false.
000858 **
000859 ** 2. If iCol is part of an existing index that has an aiRowLogEst of
000860 ** more than 20, then return false.
000861 **
000862 ** 3. If no disqualifying conditions above are found, return true.
000863 */
000864 static SQLITE_NOINLINE int columnIsGoodIndexCandidate(
000865 const Table *pTab,
000866 int iCol
000867 ){
000868 const Index *pIdx;
000869 for(pIdx = pTab->pIndex; pIdx!=0; pIdx=pIdx->pNext){
000870 int j;
000871 for(j=0; j<pIdx->nKeyCol; j++){
000872 if( pIdx->aiColumn[j]==iCol ){
000873 if( j==0 ) return 0;
000874 if( pIdx->hasStat1 && pIdx->aiRowLogEst[j+1]>20 ) return 0;
000875 break;
000876 }
000877 }
000878 }
000879 return 1;
000880 }
000881 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
000882
000883
000884
000885 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000886 /*
000887 ** Return TRUE if the WHERE clause term pTerm is of a form where it
000888 ** could be used with an index to access pSrc, assuming an appropriate
000889 ** index existed.
000890 */
000891 static int termCanDriveIndex(
000892 const WhereTerm *pTerm, /* WHERE clause term to check */
000893 const SrcItem *pSrc, /* Table we are trying to access */
000894 const Bitmask notReady /* Tables in outer loops of the join */
000895 ){
000896 char aff;
000897 int leftCol;
000898
000899 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
000900 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
000901 assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
000902 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
000903 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
000904 ){
000905 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
000906 }
000907 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
000908 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000909 leftCol = pTerm->u.x.leftColumn;
000910 if( leftCol<0 ) return 0;
000911 aff = pSrc->pSTab->aCol[leftCol].affinity;
000912 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
000913 testcase( pTerm->pExpr->op==TK_IS );
000914 return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol);
000915 }
000916 #endif
000917
000918
000919 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000920
000921 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000922 /*
000923 ** Argument pIdx represents an automatic index that the current statement
000924 ** will create and populate. Add an OP_Explain with text of the form:
000925 **
000926 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
000927 **
000928 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
000929 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
000930 ** values with. In order to avoid breaking legacy code and test cases,
000931 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
000932 */
000933 static void explainAutomaticIndex(
000934 Parse *pParse,
000935 Index *pIdx, /* Automatic index to explain */
000936 int bPartial, /* True if pIdx is a partial index */
000937 int *pAddrExplain /* OUT: Address of OP_Explain */
000938 ){
000939 if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){
000940 Table *pTab = pIdx->pTable;
000941 const char *zSep = "";
000942 char *zText = 0;
000943 int ii = 0;
000944 sqlite3_str *pStr = sqlite3_str_new(pParse->db);
000945 sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
000946 assert( pIdx->nColumn>1 );
000947 assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID );
000948 for(ii=0; ii<(pIdx->nColumn-1); ii++){
000949 const char *zName = 0;
000950 int iCol = pIdx->aiColumn[ii];
000951
000952 zName = pTab->aCol[iCol].zCnName;
000953 sqlite3_str_appendf(pStr, "%s%s", zSep, zName);
000954 zSep = ", ";
000955 }
000956 zText = sqlite3_str_finish(pStr);
000957 if( zText==0 ){
000958 sqlite3OomFault(pParse->db);
000959 }else{
000960 *pAddrExplain = sqlite3VdbeExplain(
000961 pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "")
000962 );
000963 sqlite3_free(zText);
000964 }
000965 }
000966 }
000967 #else
000968 # define explainAutomaticIndex(a,b,c,d)
000969 #endif
000970
000971 /*
000972 ** Generate code to construct the Index object for an automatic index
000973 ** and to set up the WhereLevel object pLevel so that the code generator
000974 ** makes use of the automatic index.
000975 */
000976 static SQLITE_NOINLINE void constructAutomaticIndex(
000977 Parse *pParse, /* The parsing context */
000978 WhereClause *pWC, /* The WHERE clause */
000979 const Bitmask notReady, /* Mask of cursors that are not available */
000980 WhereLevel *pLevel /* Write new index here */
000981 ){
000982 int nKeyCol; /* Number of columns in the constructed index */
000983 WhereTerm *pTerm; /* A single term of the WHERE clause */
000984 WhereTerm *pWCEnd; /* End of pWC->a[] */
000985 Index *pIdx; /* Object describing the transient index */
000986 Vdbe *v; /* Prepared statement under construction */
000987 int addrInit; /* Address of the initialization bypass jump */
000988 Table *pTable; /* The table being indexed */
000989 int addrTop; /* Top of the index fill loop */
000990 int regRecord; /* Register holding an index record */
000991 int n; /* Column counter */
000992 int i; /* Loop counter */
000993 int mxBitCol; /* Maximum column in pSrc->colUsed */
000994 CollSeq *pColl; /* Collating sequence to on a column */
000995 WhereLoop *pLoop; /* The Loop object */
000996 char *zNotUsed; /* Extra space on the end of pIdx */
000997 Bitmask idxCols; /* Bitmap of columns used for indexing */
000998 Bitmask extraCols; /* Bitmap of additional columns */
000999 u8 sentWarning = 0; /* True if a warning has been issued */
001000 u8 useBloomFilter = 0; /* True to also add a Bloom filter */
001001 Expr *pPartial = 0; /* Partial Index Expression */
001002 int iContinue = 0; /* Jump here to skip excluded rows */
001003 SrcList *pTabList; /* The complete FROM clause */
001004 SrcItem *pSrc; /* The FROM clause term to get the next index */
001005 int addrCounter = 0; /* Address where integer counter is initialized */
001006 int regBase; /* Array of registers where record is assembled */
001007 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
001008 int addrExp = 0; /* Address of OP_Explain */
001009 #endif
001010
001011 /* Generate code to skip over the creation and initialization of the
001012 ** transient index on 2nd and subsequent iterations of the loop. */
001013 v = pParse->pVdbe;
001014 assert( v!=0 );
001015 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001016
001017 /* Count the number of columns that will be added to the index
001018 ** and used to match WHERE clause constraints */
001019 nKeyCol = 0;
001020 pTabList = pWC->pWInfo->pTabList;
001021 pSrc = &pTabList->a[pLevel->iFrom];
001022 pTable = pSrc->pSTab;
001023 pWCEnd = &pWC->a[pWC->nTerm];
001024 pLoop = pLevel->pWLoop;
001025 idxCols = 0;
001026 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
001027 Expr *pExpr = pTerm->pExpr;
001028 /* Make the automatic index a partial index if there are terms in the
001029 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
001030 ** rows of the target table (pSrc) that can be used. */
001031 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001032 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0)
001033 ){
001034 pPartial = sqlite3ExprAnd(pParse, pPartial,
001035 sqlite3ExprDup(pParse->db, pExpr, 0));
001036 }
001037 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
001038 int iCol;
001039 Bitmask cMask;
001040 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001041 iCol = pTerm->u.x.leftColumn;
001042 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001043 testcase( iCol==BMS );
001044 testcase( iCol==BMS-1 );
001045 if( !sentWarning ){
001046 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
001047 "automatic index on %s(%s)", pTable->zName,
001048 pTable->aCol[iCol].zCnName);
001049 sentWarning = 1;
001050 }
001051 if( (idxCols & cMask)==0 ){
001052 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
001053 goto end_auto_index_create;
001054 }
001055 pLoop->aLTerm[nKeyCol++] = pTerm;
001056 idxCols |= cMask;
001057 }
001058 }
001059 }
001060 assert( nKeyCol>0 || pParse->db->mallocFailed );
001061 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
001062 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
001063 | WHERE_AUTO_INDEX;
001064
001065 /* Count the number of additional columns needed to create a
001066 ** covering index. A "covering index" is an index that contains all
001067 ** columns that are needed by the query. With a covering index, the
001068 ** original table never needs to be accessed. Automatic indices must
001069 ** be a covering index because the index will not be updated if the
001070 ** original table changes and the index and table cannot both be used
001071 ** if they go out of sync.
001072 */
001073 if( IsView(pTable) ){
001074 extraCols = ALLBITS & ~idxCols;
001075 }else{
001076 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
001077 }
001078 mxBitCol = MIN(BMS-1,pTable->nCol);
001079 testcase( pTable->nCol==BMS-1 );
001080 testcase( pTable->nCol==BMS-2 );
001081 for(i=0; i<mxBitCol; i++){
001082 if( extraCols & MASKBIT(i) ) nKeyCol++;
001083 }
001084 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001085 nKeyCol += pTable->nCol - BMS + 1;
001086 }
001087
001088 /* Construct the Index object to describe this index */
001089 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
001090 if( pIdx==0 ) goto end_auto_index_create;
001091 pLoop->u.btree.pIndex = pIdx;
001092 pIdx->zName = "auto-index";
001093 pIdx->pTable = pTable;
001094 n = 0;
001095 idxCols = 0;
001096 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
001097 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
001098 int iCol;
001099 Bitmask cMask;
001100 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001101 iCol = pTerm->u.x.leftColumn;
001102 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001103 testcase( iCol==BMS-1 );
001104 testcase( iCol==BMS );
001105 if( (idxCols & cMask)==0 ){
001106 Expr *pX = pTerm->pExpr;
001107 idxCols |= cMask;
001108 pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
001109 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
001110 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
001111 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
001112 n++;
001113 if( ALWAYS(pX->pLeft!=0)
001114 && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT
001115 ){
001116 /* TUNING: only use a Bloom filter on an automatic index
001117 ** if one or more key columns has the ability to hold numeric
001118 ** values, since strings all have the same hash in the Bloom
001119 ** filter implementation and hence a Bloom filter on a text column
001120 ** is not usually helpful. */
001121 useBloomFilter = 1;
001122 }
001123 }
001124 }
001125 }
001126 assert( (u32)n==pLoop->u.btree.nEq );
001127
001128 /* Add additional columns needed to make the automatic index into
001129 ** a covering index */
001130 for(i=0; i<mxBitCol; i++){
001131 if( extraCols & MASKBIT(i) ){
001132 pIdx->aiColumn[n] = i;
001133 pIdx->azColl[n] = sqlite3StrBINARY;
001134 n++;
001135 }
001136 }
001137 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001138 for(i=BMS-1; i<pTable->nCol; i++){
001139 pIdx->aiColumn[n] = i;
001140 pIdx->azColl[n] = sqlite3StrBINARY;
001141 n++;
001142 }
001143 }
001144 assert( n==nKeyCol );
001145 pIdx->aiColumn[n] = XN_ROWID;
001146 pIdx->azColl[n] = sqlite3StrBINARY;
001147
001148 /* Create the automatic index */
001149 explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
001150 assert( pLevel->iIdxCur>=0 );
001151 pLevel->iIdxCur = pParse->nTab++;
001152 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
001153 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
001154 VdbeComment((v, "for %s", pTable->zName));
001155 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){
001156 sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel);
001157 pLevel->regFilter = ++pParse->nMem;
001158 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
001159 }
001160
001161 /* Fill the automatic index with content */
001162 assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
001163 if( pSrc->fg.viaCoroutine ){
001164 int regYield;
001165 Subquery *pSubq;
001166 assert( pSrc->fg.isSubquery );
001167 pSubq = pSrc->u4.pSubq;
001168 assert( pSubq!=0 );
001169 regYield = pSubq->regReturn;
001170 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
001171 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub);
001172 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
001173 VdbeCoverage(v);
001174 VdbeComment((v, "next row of %s", pSrc->pSTab->zName));
001175 }else{
001176 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
001177 }
001178 if( pPartial ){
001179 iContinue = sqlite3VdbeMakeLabel(pParse);
001180 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
001181 pLoop->wsFlags |= WHERE_PARTIALIDX;
001182 }
001183 regRecord = sqlite3GetTempReg(pParse);
001184 regBase = sqlite3GenerateIndexKey(
001185 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
001186 );
001187 if( pLevel->regFilter ){
001188 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
001189 regBase, pLoop->u.btree.nEq);
001190 }
001191 sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v));
001192 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
001193 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
001194 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
001195 if( pSrc->fg.viaCoroutine ){
001196 assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 );
001197 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
001198 testcase( pParse->db->mallocFailed );
001199 assert( pLevel->iIdxCur>0 );
001200 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
001201 pSrc->u4.pSubq->regResult, pLevel->iIdxCur);
001202 sqlite3VdbeGoto(v, addrTop);
001203 pSrc->fg.viaCoroutine = 0;
001204 }else{
001205 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
001206 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
001207 }
001208 sqlite3VdbeJumpHere(v, addrTop);
001209 sqlite3ReleaseTempReg(pParse, regRecord);
001210
001211 /* Jump here when skipping the initialization */
001212 sqlite3VdbeJumpHere(v, addrInit);
001213 sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
001214
001215 end_auto_index_create:
001216 sqlite3ExprDelete(pParse->db, pPartial);
001217 }
001218 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
001219
001220 /*
001221 ** Generate bytecode that will initialize a Bloom filter that is appropriate
001222 ** for pLevel.
001223 **
001224 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
001225 ** flag set, initialize a Bloomfilter for them as well. Except don't do
001226 ** this recursive initialization if the SQLITE_BloomPulldown optimization has
001227 ** been turned off.
001228 **
001229 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
001230 ** from the loop, but the regFilter value is set to a register that implements
001231 ** the Bloom filter. When regFilter is positive, the
001232 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
001233 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that
001234 ** no matching rows exist.
001235 **
001236 ** This routine may only be called if it has previously been determined that
001237 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
001238 ** is set.
001239 */
001240 static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
001241 WhereInfo *pWInfo, /* The WHERE clause */
001242 int iLevel, /* Index in pWInfo->a[] that is pLevel */
001243 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
001244 Bitmask notReady /* Loops that are not ready */
001245 ){
001246 int addrOnce; /* Address of opening OP_Once */
001247 int addrTop; /* Address of OP_Rewind */
001248 int addrCont; /* Jump here to skip a row */
001249 const WhereTerm *pTerm; /* For looping over WHERE clause terms */
001250 const WhereTerm *pWCEnd; /* Last WHERE clause term */
001251 Parse *pParse = pWInfo->pParse; /* Parsing context */
001252 Vdbe *v = pParse->pVdbe; /* VDBE under construction */
001253 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
001254 int iCur; /* Cursor for table getting the filter */
001255 IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */
001256 IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */
001257
001258 saved_pIdxEpr = pParse->pIdxEpr;
001259 saved_pIdxPartExpr = pParse->pIdxPartExpr;
001260 pParse->pIdxEpr = 0;
001261 pParse->pIdxPartExpr = 0;
001262
001263 assert( pLoop!=0 );
001264 assert( v!=0 );
001265 assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
001266 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 );
001267
001268 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001269 do{
001270 const SrcList *pTabList;
001271 const SrcItem *pItem;
001272 const Table *pTab;
001273 u64 sz;
001274 int iSrc;
001275 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
001276 addrCont = sqlite3VdbeMakeLabel(pParse);
001277 iCur = pLevel->iTabCur;
001278 pLevel->regFilter = ++pParse->nMem;
001279
001280 /* The Bloom filter is a Blob held in a register. Initialize it
001281 ** to zero-filled blob of at least 80K bits, but maybe more if the
001282 ** estimated size of the table is larger. We could actually
001283 ** measure the size of the table at run-time using OP_Count with
001284 ** P3==1 and use that value to initialize the blob. But that makes
001285 ** testing complicated. By basing the blob size on the value in the
001286 ** sqlite_stat1 table, testing is much easier.
001287 */
001288 pTabList = pWInfo->pTabList;
001289 iSrc = pLevel->iFrom;
001290 pItem = &pTabList->a[iSrc];
001291 assert( pItem!=0 );
001292 pTab = pItem->pSTab;
001293 assert( pTab!=0 );
001294 sz = sqlite3LogEstToInt(pTab->nRowLogEst);
001295 if( sz<10000 ){
001296 sz = 10000;
001297 }else if( sz>10000000 ){
001298 sz = 10000000;
001299 }
001300 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
001301
001302 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
001303 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
001304 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
001305 Expr *pExpr = pTerm->pExpr;
001306 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001307 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0)
001308 ){
001309 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
001310 }
001311 }
001312 if( pLoop->wsFlags & WHERE_IPK ){
001313 int r1 = sqlite3GetTempReg(pParse);
001314 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
001315 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
001316 sqlite3ReleaseTempReg(pParse, r1);
001317 }else{
001318 Index *pIdx = pLoop->u.btree.pIndex;
001319 int n = pLoop->u.btree.nEq;
001320 int r1 = sqlite3GetTempRange(pParse, n);
001321 int jj;
001322 for(jj=0; jj<n; jj++){
001323 assert( pIdx->pTable==pItem->pSTab );
001324 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj);
001325 }
001326 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
001327 sqlite3ReleaseTempRange(pParse, r1, n);
001328 }
001329 sqlite3VdbeResolveLabel(v, addrCont);
001330 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
001331 VdbeCoverage(v);
001332 sqlite3VdbeJumpHere(v, addrTop);
001333 pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
001334 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
001335 while( ++iLevel < pWInfo->nLevel ){
001336 const SrcItem *pTabItem;
001337 pLevel = &pWInfo->a[iLevel];
001338 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
001339 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue;
001340 pLoop = pLevel->pWLoop;
001341 if( NEVER(pLoop==0) ) continue;
001342 if( pLoop->prereq & notReady ) continue;
001343 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN))
001344 ==WHERE_BLOOMFILTER
001345 ){
001346 /* This is a candidate for bloom-filter pull-down (early evaluation).
001347 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
001348 ** not able to do early evaluation of bloom filters that make use of
001349 ** the IN operator */
001350 break;
001351 }
001352 }
001353 }while( iLevel < pWInfo->nLevel );
001354 sqlite3VdbeJumpHere(v, addrOnce);
001355 pParse->pIdxEpr = saved_pIdxEpr;
001356 pParse->pIdxPartExpr = saved_pIdxPartExpr;
001357 }
001358
001359
001360 #ifndef SQLITE_OMIT_VIRTUALTABLE
001361 /*
001362 ** Return term iTerm of the WhereClause passed as the first argument. Terms
001363 ** are numbered from 0 upwards, starting with the terms in pWC->a[], then
001364 ** those in pWC->pOuter->a[] (if any), and so on.
001365 */
001366 static WhereTerm *termFromWhereClause(WhereClause *pWC, int iTerm){
001367 WhereClause *p;
001368 for(p=pWC; p; p=p->pOuter){
001369 if( iTerm<p->nTerm ) return &p->a[iTerm];
001370 iTerm -= p->nTerm;
001371 }
001372 return 0;
001373 }
001374
001375 /*
001376 ** Allocate and populate an sqlite3_index_info structure. It is the
001377 ** responsibility of the caller to eventually release the structure
001378 ** by passing the pointer returned by this function to freeIndexInfo().
001379 */
001380 static sqlite3_index_info *allocateIndexInfo(
001381 WhereInfo *pWInfo, /* The WHERE clause */
001382 WhereClause *pWC, /* The WHERE clause being analyzed */
001383 Bitmask mUnusable, /* Ignore terms with these prereqs */
001384 SrcItem *pSrc, /* The FROM clause term that is the vtab */
001385 u16 *pmNoOmit /* Mask of terms not to omit */
001386 ){
001387 int i, j;
001388 int nTerm;
001389 Parse *pParse = pWInfo->pParse;
001390 struct sqlite3_index_constraint *pIdxCons;
001391 struct sqlite3_index_orderby *pIdxOrderBy;
001392 struct sqlite3_index_constraint_usage *pUsage;
001393 struct HiddenIndexInfo *pHidden;
001394 WhereTerm *pTerm;
001395 int nOrderBy;
001396 sqlite3_index_info *pIdxInfo;
001397 u16 mNoOmit = 0;
001398 const Table *pTab;
001399 int eDistinct = 0;
001400 ExprList *pOrderBy = pWInfo->pOrderBy;
001401 WhereClause *p;
001402
001403 assert( pSrc!=0 );
001404 pTab = pSrc->pSTab;
001405 assert( pTab!=0 );
001406 assert( IsVirtual(pTab) );
001407
001408 /* Find all WHERE clause constraints referring to this virtual table.
001409 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
001410 ** terms found.
001411 */
001412 for(p=pWC, nTerm=0; p; p=p->pOuter){
001413 for(i=0, pTerm=p->a; i<p->nTerm; i++, pTerm++){
001414 pTerm->wtFlags &= ~TERM_OK;
001415 if( pTerm->leftCursor != pSrc->iCursor ) continue;
001416 if( pTerm->prereqRight & mUnusable ) continue;
001417 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
001418 testcase( pTerm->eOperator & WO_IN );
001419 testcase( pTerm->eOperator & WO_ISNULL );
001420 testcase( pTerm->eOperator & WO_IS );
001421 testcase( pTerm->eOperator & WO_ALL );
001422 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
001423 if( pTerm->wtFlags & TERM_VNULL ) continue;
001424
001425 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001426 assert( pTerm->u.x.leftColumn>=XN_ROWID );
001427 assert( pTerm->u.x.leftColumn<pTab->nCol );
001428 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
001429 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
001430 ){
001431 continue;
001432 }
001433 nTerm++;
001434 pTerm->wtFlags |= TERM_OK;
001435 }
001436 }
001437
001438 /* If the ORDER BY clause contains only columns in the current
001439 ** virtual table then allocate space for the aOrderBy part of
001440 ** the sqlite3_index_info structure.
001441 */
001442 nOrderBy = 0;
001443 if( pOrderBy ){
001444 int n = pOrderBy->nExpr;
001445 for(i=0; i<n; i++){
001446 Expr *pExpr = pOrderBy->a[i].pExpr;
001447 Expr *pE2;
001448
001449 /* Skip over constant terms in the ORDER BY clause */
001450 if( sqlite3ExprIsConstant(0, pExpr) ){
001451 continue;
001452 }
001453
001454 /* Virtual tables are unable to deal with NULLS FIRST */
001455 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break;
001456
001457 /* First case - a direct column references without a COLLATE operator */
001458 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){
001459 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol );
001460 continue;
001461 }
001462
001463 /* 2nd case - a column reference with a COLLATE operator. Only match
001464 ** of the COLLATE operator matches the collation of the column. */
001465 if( pExpr->op==TK_COLLATE
001466 && (pE2 = pExpr->pLeft)->op==TK_COLUMN
001467 && pE2->iTable==pSrc->iCursor
001468 ){
001469 const char *zColl; /* The collating sequence name */
001470 assert( !ExprHasProperty(pExpr, EP_IntValue) );
001471 assert( pExpr->u.zToken!=0 );
001472 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol );
001473 pExpr->iColumn = pE2->iColumn;
001474 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */
001475 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]);
001476 if( zColl==0 ) zColl = sqlite3StrBINARY;
001477 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue;
001478 }
001479
001480 /* No matches cause a break out of the loop */
001481 break;
001482 }
001483 if( i==n ){
001484 nOrderBy = n;
001485 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){
001486 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
001487 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
001488 eDistinct = 1;
001489 }
001490 }
001491 }
001492
001493 /* Allocate the sqlite3_index_info structure
001494 */
001495 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
001496 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
001497 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
001498 + sizeof(sqlite3_value*)*nTerm );
001499 if( pIdxInfo==0 ){
001500 sqlite3ErrorMsg(pParse, "out of memory");
001501 return 0;
001502 }
001503 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
001504 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm];
001505 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
001506 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
001507 pIdxInfo->aConstraint = pIdxCons;
001508 pIdxInfo->aOrderBy = pIdxOrderBy;
001509 pIdxInfo->aConstraintUsage = pUsage;
001510 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
001511 if( HasRowid(pTab)==0 ){
001512 /* Ensure that all bits associated with PK columns are set. This is to
001513 ** ensure they are available for cases like RIGHT joins or OR loops. */
001514 Index *pPk = sqlite3PrimaryKeyIndex((Table*)pTab);
001515 assert( pPk!=0 );
001516 for(i=0; i<pPk->nKeyCol; i++){
001517 int iCol = pPk->aiColumn[i];
001518 assert( iCol>=0 );
001519 if( iCol>=BMS-1 ) iCol = BMS-1;
001520 pIdxInfo->colUsed |= MASKBIT(iCol);
001521 }
001522 }
001523 pHidden->pWC = pWC;
001524 pHidden->pParse = pParse;
001525 pHidden->eDistinct = eDistinct;
001526 pHidden->mIn = 0;
001527 for(p=pWC, i=j=0; p; p=p->pOuter){
001528 int nLast = i+p->nTerm;;
001529 for(pTerm=p->a; i<nLast; i++, pTerm++){
001530 u16 op;
001531 if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
001532 pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
001533 pIdxCons[j].iTermOffset = i;
001534 op = pTerm->eOperator & WO_ALL;
001535 if( op==WO_IN ){
001536 if( (pTerm->wtFlags & TERM_SLICE)==0 ){
001537 pHidden->mIn |= SMASKBIT32(j);
001538 }
001539 op = WO_EQ;
001540 }
001541 if( op==WO_AUX ){
001542 pIdxCons[j].op = pTerm->eMatchOp;
001543 }else if( op & (WO_ISNULL|WO_IS) ){
001544 if( op==WO_ISNULL ){
001545 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
001546 }else{
001547 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
001548 }
001549 }else{
001550 pIdxCons[j].op = (u8)op;
001551 /* The direct assignment in the previous line is possible only because
001552 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
001553 ** following asserts verify this fact. */
001554 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
001555 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
001556 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
001557 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
001558 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
001559 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
001560
001561 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
001562 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
001563 ){
001564 testcase( j!=i );
001565 if( j<16 ) mNoOmit |= (1 << j);
001566 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
001567 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
001568 }
001569 }
001570
001571 j++;
001572 }
001573 }
001574 assert( j==nTerm );
001575 pIdxInfo->nConstraint = j;
001576 for(i=j=0; i<nOrderBy; i++){
001577 Expr *pExpr = pOrderBy->a[i].pExpr;
001578 if( sqlite3ExprIsConstant(0, pExpr) ) continue;
001579 assert( pExpr->op==TK_COLUMN
001580 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
001581 && pExpr->iColumn==pExpr->pLeft->iColumn) );
001582 pIdxOrderBy[j].iColumn = pExpr->iColumn;
001583 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC;
001584 j++;
001585 }
001586 pIdxInfo->nOrderBy = j;
001587
001588 *pmNoOmit = mNoOmit;
001589 return pIdxInfo;
001590 }
001591
001592 /*
001593 ** Free and zero the sqlite3_index_info.idxStr value if needed.
001594 */
001595 static void freeIdxStr(sqlite3_index_info *pIdxInfo){
001596 if( pIdxInfo->needToFreeIdxStr ){
001597 sqlite3_free(pIdxInfo->idxStr);
001598 pIdxInfo->idxStr = 0;
001599 pIdxInfo->needToFreeIdxStr = 0;
001600 }
001601 }
001602
001603 /*
001604 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
001605 ** and possibly modified by xBestIndex methods.
001606 */
001607 static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
001608 HiddenIndexInfo *pHidden;
001609 int i;
001610 assert( pIdxInfo!=0 );
001611 pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
001612 assert( pHidden->pParse!=0 );
001613 assert( pHidden->pParse->db==db );
001614 for(i=0; i<pIdxInfo->nConstraint; i++){
001615 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
001616 pHidden->aRhs[i] = 0;
001617 }
001618 freeIdxStr(pIdxInfo);
001619 sqlite3DbFree(db, pIdxInfo);
001620 }
001621
001622 /*
001623 ** The table object reference passed as the second argument to this function
001624 ** must represent a virtual table. This function invokes the xBestIndex()
001625 ** method of the virtual table with the sqlite3_index_info object that
001626 ** comes in as the 3rd argument to this function.
001627 **
001628 ** If an error occurs, pParse is populated with an error message and an
001629 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
001630 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
001631 ** the current configuration of "unusable" flags in sqlite3_index_info can
001632 ** not result in a valid plan.
001633 **
001634 ** Whether or not an error is returned, it is the responsibility of the
001635 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
001636 ** that this is required.
001637 */
001638 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
001639 int rc;
001640 sqlite3_vtab *pVtab;
001641
001642 assert( IsVirtual(pTab) );
001643 pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
001644 whereTraceIndexInfoInputs(p, pTab);
001645 pParse->db->nSchemaLock++;
001646 rc = pVtab->pModule->xBestIndex(pVtab, p);
001647 pParse->db->nSchemaLock--;
001648 whereTraceIndexInfoOutputs(p, pTab);
001649
001650 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
001651 if( rc==SQLITE_NOMEM ){
001652 sqlite3OomFault(pParse->db);
001653 }else if( !pVtab->zErrMsg ){
001654 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
001655 }else{
001656 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
001657 }
001658 }
001659 if( pTab->u.vtab.p->bAllSchemas ){
001660 sqlite3VtabUsesAllSchemas(pParse);
001661 }
001662 sqlite3_free(pVtab->zErrMsg);
001663 pVtab->zErrMsg = 0;
001664 return rc;
001665 }
001666 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
001667
001668 #ifdef SQLITE_ENABLE_STAT4
001669 /*
001670 ** Estimate the location of a particular key among all keys in an
001671 ** index. Store the results in aStat as follows:
001672 **
001673 ** aStat[0] Est. number of rows less than pRec
001674 ** aStat[1] Est. number of rows equal to pRec
001675 **
001676 ** Return the index of the sample that is the smallest sample that
001677 ** is greater than or equal to pRec. Note that this index is not an index
001678 ** into the aSample[] array - it is an index into a virtual set of samples
001679 ** based on the contents of aSample[] and the number of fields in record
001680 ** pRec.
001681 */
001682 static int whereKeyStats(
001683 Parse *pParse, /* Database connection */
001684 Index *pIdx, /* Index to consider domain of */
001685 UnpackedRecord *pRec, /* Vector of values to consider */
001686 int roundUp, /* Round up if true. Round down if false */
001687 tRowcnt *aStat /* OUT: stats written here */
001688 ){
001689 IndexSample *aSample = pIdx->aSample;
001690 int iCol; /* Index of required stats in anEq[] etc. */
001691 int i; /* Index of first sample >= pRec */
001692 int iSample; /* Smallest sample larger than or equal to pRec */
001693 int iMin = 0; /* Smallest sample not yet tested */
001694 int iTest; /* Next sample to test */
001695 int res; /* Result of comparison operation */
001696 int nField; /* Number of fields in pRec */
001697 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
001698
001699 #ifndef SQLITE_DEBUG
001700 UNUSED_PARAMETER( pParse );
001701 #endif
001702 assert( pRec!=0 );
001703 assert( pIdx->nSample>0 );
001704 assert( pRec->nField>0 );
001705
001706
001707 /* Do a binary search to find the first sample greater than or equal
001708 ** to pRec. If pRec contains a single field, the set of samples to search
001709 ** is simply the aSample[] array. If the samples in aSample[] contain more
001710 ** than one fields, all fields following the first are ignored.
001711 **
001712 ** If pRec contains N fields, where N is more than one, then as well as the
001713 ** samples in aSample[] (truncated to N fields), the search also has to
001714 ** consider prefixes of those samples. For example, if the set of samples
001715 ** in aSample is:
001716 **
001717 ** aSample[0] = (a, 5)
001718 ** aSample[1] = (a, 10)
001719 ** aSample[2] = (b, 5)
001720 ** aSample[3] = (c, 100)
001721 ** aSample[4] = (c, 105)
001722 **
001723 ** Then the search space should ideally be the samples above and the
001724 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
001725 ** the code actually searches this set:
001726 **
001727 ** 0: (a)
001728 ** 1: (a, 5)
001729 ** 2: (a, 10)
001730 ** 3: (a, 10)
001731 ** 4: (b)
001732 ** 5: (b, 5)
001733 ** 6: (c)
001734 ** 7: (c, 100)
001735 ** 8: (c, 105)
001736 ** 9: (c, 105)
001737 **
001738 ** For each sample in the aSample[] array, N samples are present in the
001739 ** effective sample array. In the above, samples 0 and 1 are based on
001740 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
001741 **
001742 ** Often, sample i of each block of N effective samples has (i+1) fields.
001743 ** Except, each sample may be extended to ensure that it is greater than or
001744 ** equal to the previous sample in the array. For example, in the above,
001745 ** sample 2 is the first sample of a block of N samples, so at first it
001746 ** appears that it should be 1 field in size. However, that would make it
001747 ** smaller than sample 1, so the binary search would not work. As a result,
001748 ** it is extended to two fields. The duplicates that this creates do not
001749 ** cause any problems.
001750 */
001751 if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
001752 nField = pIdx->nKeyCol;
001753 }else{
001754 nField = pIdx->nColumn;
001755 }
001756 nField = MIN(pRec->nField, nField);
001757 iCol = 0;
001758 iSample = pIdx->nSample * nField;
001759 do{
001760 int iSamp; /* Index in aSample[] of test sample */
001761 int n; /* Number of fields in test sample */
001762
001763 iTest = (iMin+iSample)/2;
001764 iSamp = iTest / nField;
001765 if( iSamp>0 ){
001766 /* The proposed effective sample is a prefix of sample aSample[iSamp].
001767 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
001768 ** fields that is greater than the previous effective sample. */
001769 for(n=(iTest % nField) + 1; n<nField; n++){
001770 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
001771 }
001772 }else{
001773 n = iTest + 1;
001774 }
001775
001776 pRec->nField = n;
001777 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
001778 if( res<0 ){
001779 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
001780 iMin = iTest+1;
001781 }else if( res==0 && n<nField ){
001782 iLower = aSample[iSamp].anLt[n-1];
001783 iMin = iTest+1;
001784 res = -1;
001785 }else{
001786 iSample = iTest;
001787 iCol = n-1;
001788 }
001789 }while( res && iMin<iSample );
001790 i = iSample / nField;
001791
001792 #ifdef SQLITE_DEBUG
001793 /* The following assert statements check that the binary search code
001794 ** above found the right answer. This block serves no purpose other
001795 ** than to invoke the asserts. */
001796 if( pParse->db->mallocFailed==0 ){
001797 if( res==0 ){
001798 /* If (res==0) is true, then pRec must be equal to sample i. */
001799 assert( i<pIdx->nSample );
001800 assert( iCol==nField-1 );
001801 pRec->nField = nField;
001802 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
001803 || pParse->db->mallocFailed
001804 );
001805 }else{
001806 /* Unless i==pIdx->nSample, indicating that pRec is larger than
001807 ** all samples in the aSample[] array, pRec must be smaller than the
001808 ** (iCol+1) field prefix of sample i. */
001809 assert( i<=pIdx->nSample && i>=0 );
001810 pRec->nField = iCol+1;
001811 assert( i==pIdx->nSample
001812 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
001813 || pParse->db->mallocFailed );
001814
001815 /* if i==0 and iCol==0, then record pRec is smaller than all samples
001816 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
001817 ** be greater than or equal to the (iCol) field prefix of sample i.
001818 ** If (i>0), then pRec must also be greater than sample (i-1). */
001819 if( iCol>0 ){
001820 pRec->nField = iCol;
001821 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
001822 || pParse->db->mallocFailed || CORRUPT_DB );
001823 }
001824 if( i>0 ){
001825 pRec->nField = nField;
001826 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
001827 || pParse->db->mallocFailed || CORRUPT_DB );
001828 }
001829 }
001830 }
001831 #endif /* ifdef SQLITE_DEBUG */
001832
001833 if( res==0 ){
001834 /* Record pRec is equal to sample i */
001835 assert( iCol==nField-1 );
001836 aStat[0] = aSample[i].anLt[iCol];
001837 aStat[1] = aSample[i].anEq[iCol];
001838 }else{
001839 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
001840 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
001841 ** is larger than all samples in the array. */
001842 tRowcnt iUpper, iGap;
001843 if( i>=pIdx->nSample ){
001844 iUpper = pIdx->nRowEst0;
001845 }else{
001846 iUpper = aSample[i].anLt[iCol];
001847 }
001848
001849 if( iLower>=iUpper ){
001850 iGap = 0;
001851 }else{
001852 iGap = iUpper - iLower;
001853 }
001854 if( roundUp ){
001855 iGap = (iGap*2)/3;
001856 }else{
001857 iGap = iGap/3;
001858 }
001859 aStat[0] = iLower + iGap;
001860 aStat[1] = pIdx->aAvgEq[nField-1];
001861 }
001862
001863 /* Restore the pRec->nField value before returning. */
001864 pRec->nField = nField;
001865 return i;
001866 }
001867 #endif /* SQLITE_ENABLE_STAT4 */
001868
001869 /*
001870 ** If it is not NULL, pTerm is a term that provides an upper or lower
001871 ** bound on a range scan. Without considering pTerm, it is estimated
001872 ** that the scan will visit nNew rows. This function returns the number
001873 ** estimated to be visited after taking pTerm into account.
001874 **
001875 ** If the user explicitly specified a likelihood() value for this term,
001876 ** then the return value is the likelihood multiplied by the number of
001877 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
001878 ** has a likelihood of 0.50, and any other term a likelihood of 0.25.
001879 */
001880 static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
001881 LogEst nRet = nNew;
001882 if( pTerm ){
001883 if( pTerm->truthProb<=0 ){
001884 nRet += pTerm->truthProb;
001885 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
001886 nRet -= 20; assert( 20==sqlite3LogEst(4) );
001887 }
001888 }
001889 return nRet;
001890 }
001891
001892
001893 #ifdef SQLITE_ENABLE_STAT4
001894 /*
001895 ** Return the affinity for a single column of an index.
001896 */
001897 char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
001898 assert( iCol>=0 && iCol<pIdx->nColumn );
001899 if( !pIdx->zColAff ){
001900 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
001901 }
001902 assert( pIdx->zColAff[iCol]!=0 );
001903 return pIdx->zColAff[iCol];
001904 }
001905 #endif
001906
001907
001908 #ifdef SQLITE_ENABLE_STAT4
001909 /*
001910 ** This function is called to estimate the number of rows visited by a
001911 ** range-scan on a skip-scan index. For example:
001912 **
001913 ** CREATE INDEX i1 ON t1(a, b, c);
001914 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
001915 **
001916 ** Value pLoop->nOut is currently set to the estimated number of rows
001917 ** visited for scanning (a=? AND b=?). This function reduces that estimate
001918 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
001919 ** on the stat4 data for the index. this scan will be performed multiple
001920 ** times (once for each (a,b) combination that matches a=?) is dealt with
001921 ** by the caller.
001922 **
001923 ** It does this by scanning through all stat4 samples, comparing values
001924 ** extracted from pLower and pUpper with the corresponding column in each
001925 ** sample. If L and U are the number of samples found to be less than or
001926 ** equal to the values extracted from pLower and pUpper respectively, and
001927 ** N is the total number of samples, the pLoop->nOut value is adjusted
001928 ** as follows:
001929 **
001930 ** nOut = nOut * ( min(U - L, 1) / N )
001931 **
001932 ** If pLower is NULL, or a value cannot be extracted from the term, L is
001933 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
001934 ** U is set to N.
001935 **
001936 ** Normally, this function sets *pbDone to 1 before returning. However,
001937 ** if no value can be extracted from either pLower or pUpper (and so the
001938 ** estimate of the number of rows delivered remains unchanged), *pbDone
001939 ** is left as is.
001940 **
001941 ** If an error occurs, an SQLite error code is returned. Otherwise,
001942 ** SQLITE_OK.
001943 */
001944 static int whereRangeSkipScanEst(
001945 Parse *pParse, /* Parsing & code generating context */
001946 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001947 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001948 WhereLoop *pLoop, /* Update the .nOut value of this loop */
001949 int *pbDone /* Set to true if at least one expr. value extracted */
001950 ){
001951 Index *p = pLoop->u.btree.pIndex;
001952 int nEq = pLoop->u.btree.nEq;
001953 sqlite3 *db = pParse->db;
001954 int nLower = -1;
001955 int nUpper = p->nSample+1;
001956 int rc = SQLITE_OK;
001957 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
001958 CollSeq *pColl;
001959
001960 sqlite3_value *p1 = 0; /* Value extracted from pLower */
001961 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
001962 sqlite3_value *pVal = 0; /* Value extracted from record */
001963
001964 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
001965 if( pLower ){
001966 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
001967 nLower = 0;
001968 }
001969 if( pUpper && rc==SQLITE_OK ){
001970 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
001971 nUpper = p2 ? 0 : p->nSample;
001972 }
001973
001974 if( p1 || p2 ){
001975 int i;
001976 int nDiff;
001977 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
001978 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
001979 if( rc==SQLITE_OK && p1 ){
001980 int res = sqlite3MemCompare(p1, pVal, pColl);
001981 if( res>=0 ) nLower++;
001982 }
001983 if( rc==SQLITE_OK && p2 ){
001984 int res = sqlite3MemCompare(p2, pVal, pColl);
001985 if( res>=0 ) nUpper++;
001986 }
001987 }
001988 nDiff = (nUpper - nLower);
001989 if( nDiff<=0 ) nDiff = 1;
001990
001991 /* If there is both an upper and lower bound specified, and the
001992 ** comparisons indicate that they are close together, use the fallback
001993 ** method (assume that the scan visits 1/64 of the rows) for estimating
001994 ** the number of rows visited. Otherwise, estimate the number of rows
001995 ** using the method described in the header comment for this function. */
001996 if( nDiff!=1 || pUpper==0 || pLower==0 ){
001997 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
001998 pLoop->nOut -= nAdjust;
001999 *pbDone = 1;
002000 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
002001 nLower, nUpper, nAdjust*-1, pLoop->nOut));
002002 }
002003
002004 }else{
002005 assert( *pbDone==0 );
002006 }
002007
002008 sqlite3ValueFree(p1);
002009 sqlite3ValueFree(p2);
002010 sqlite3ValueFree(pVal);
002011
002012 return rc;
002013 }
002014 #endif /* SQLITE_ENABLE_STAT4 */
002015
002016 /*
002017 ** This function is used to estimate the number of rows that will be visited
002018 ** by scanning an index for a range of values. The range may have an upper
002019 ** bound, a lower bound, or both. The WHERE clause terms that set the upper
002020 ** and lower bounds are represented by pLower and pUpper respectively. For
002021 ** example, assuming that index p is on t1(a):
002022 **
002023 ** ... FROM t1 WHERE a > ? AND a < ? ...
002024 ** |_____| |_____|
002025 ** | |
002026 ** pLower pUpper
002027 **
002028 ** If either of the upper or lower bound is not present, then NULL is passed in
002029 ** place of the corresponding WhereTerm.
002030 **
002031 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
002032 ** column subject to the range constraint. Or, equivalently, the number of
002033 ** equality constraints optimized by the proposed index scan. For example,
002034 ** assuming index p is on t1(a, b), and the SQL query is:
002035 **
002036 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
002037 **
002038 ** then nEq is set to 1 (as the range restricted column, b, is the second
002039 ** left-most column of the index). Or, if the query is:
002040 **
002041 ** ... FROM t1 WHERE a > ? AND a < ? ...
002042 **
002043 ** then nEq is set to 0.
002044 **
002045 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
002046 ** number of rows that the index scan is expected to visit without
002047 ** considering the range constraints. If nEq is 0, then *pnOut is the number of
002048 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
002049 ** to account for the range constraints pLower and pUpper.
002050 **
002051 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
002052 ** used, a single range inequality reduces the search space by a factor of 4.
002053 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
002054 ** rows visited by a factor of 64.
002055 */
002056 static int whereRangeScanEst(
002057 Parse *pParse, /* Parsing & code generating context */
002058 WhereLoopBuilder *pBuilder,
002059 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
002060 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
002061 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
002062 ){
002063 int rc = SQLITE_OK;
002064 int nOut = pLoop->nOut;
002065 LogEst nNew;
002066
002067 #ifdef SQLITE_ENABLE_STAT4
002068 Index *p = pLoop->u.btree.pIndex;
002069 int nEq = pLoop->u.btree.nEq;
002070
002071 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol)
002072 && OptimizationEnabled(pParse->db, SQLITE_Stat4)
002073 ){
002074 if( nEq==pBuilder->nRecValid ){
002075 UnpackedRecord *pRec = pBuilder->pRec;
002076 tRowcnt a[2];
002077 int nBtm = pLoop->u.btree.nBtm;
002078 int nTop = pLoop->u.btree.nTop;
002079
002080 /* Variable iLower will be set to the estimate of the number of rows in
002081 ** the index that are less than the lower bound of the range query. The
002082 ** lower bound being the concatenation of $P and $L, where $P is the
002083 ** key-prefix formed by the nEq values matched against the nEq left-most
002084 ** columns of the index, and $L is the value in pLower.
002085 **
002086 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
002087 ** is not a simple variable or literal value), the lower bound of the
002088 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
002089 ** if $L is available, whereKeyStats() is called for both ($P) and
002090 ** ($P:$L) and the larger of the two returned values is used.
002091 **
002092 ** Similarly, iUpper is to be set to the estimate of the number of rows
002093 ** less than the upper bound of the range query. Where the upper bound
002094 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
002095 ** of iUpper are requested of whereKeyStats() and the smaller used.
002096 **
002097 ** The number of rows between the two bounds is then just iUpper-iLower.
002098 */
002099 tRowcnt iLower; /* Rows less than the lower bound */
002100 tRowcnt iUpper; /* Rows less than the upper bound */
002101 int iLwrIdx = -2; /* aSample[] for the lower bound */
002102 int iUprIdx = -1; /* aSample[] for the upper bound */
002103
002104 if( pRec ){
002105 testcase( pRec->nField!=pBuilder->nRecValid );
002106 pRec->nField = pBuilder->nRecValid;
002107 }
002108 /* Determine iLower and iUpper using ($P) only. */
002109 if( nEq==0 ){
002110 iLower = 0;
002111 iUpper = p->nRowEst0;
002112 }else{
002113 /* Note: this call could be optimized away - since the same values must
002114 ** have been requested when testing key $P in whereEqualScanEst(). */
002115 whereKeyStats(pParse, p, pRec, 0, a);
002116 iLower = a[0];
002117 iUpper = a[0] + a[1];
002118 }
002119
002120 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
002121 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
002122 assert( p->aSortOrder!=0 );
002123 if( p->aSortOrder[nEq] ){
002124 /* The roles of pLower and pUpper are swapped for a DESC index */
002125 SWAP(WhereTerm*, pLower, pUpper);
002126 SWAP(int, nBtm, nTop);
002127 }
002128
002129 /* If possible, improve on the iLower estimate using ($P:$L). */
002130 if( pLower ){
002131 int n; /* Values extracted from pExpr */
002132 Expr *pExpr = pLower->pExpr->pRight;
002133 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
002134 if( rc==SQLITE_OK && n ){
002135 tRowcnt iNew;
002136 u16 mask = WO_GT|WO_LE;
002137 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002138 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
002139 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
002140 if( iNew>iLower ) iLower = iNew;
002141 nOut--;
002142 pLower = 0;
002143 }
002144 }
002145
002146 /* If possible, improve on the iUpper estimate using ($P:$U). */
002147 if( pUpper ){
002148 int n; /* Values extracted from pExpr */
002149 Expr *pExpr = pUpper->pExpr->pRight;
002150 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
002151 if( rc==SQLITE_OK && n ){
002152 tRowcnt iNew;
002153 u16 mask = WO_GT|WO_LE;
002154 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002155 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
002156 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
002157 if( iNew<iUpper ) iUpper = iNew;
002158 nOut--;
002159 pUpper = 0;
002160 }
002161 }
002162
002163 pBuilder->pRec = pRec;
002164 if( rc==SQLITE_OK ){
002165 if( iUpper>iLower ){
002166 nNew = sqlite3LogEst(iUpper - iLower);
002167 /* TUNING: If both iUpper and iLower are derived from the same
002168 ** sample, then assume they are 4x more selective. This brings
002169 ** the estimated selectivity more in line with what it would be
002170 ** if estimated without the use of STAT4 tables. */
002171 if( iLwrIdx==iUprIdx ){ nNew -= 20; }
002172 assert( 20==sqlite3LogEst(4) );
002173 }else{
002174 nNew = 10; assert( 10==sqlite3LogEst(2) );
002175 }
002176 if( nNew<nOut ){
002177 nOut = nNew;
002178 }
002179 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n",
002180 (u32)iLower, (u32)iUpper, nOut));
002181 }
002182 }else{
002183 int bDone = 0;
002184 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
002185 if( bDone ) return rc;
002186 }
002187 }
002188 #else
002189 UNUSED_PARAMETER(pParse);
002190 UNUSED_PARAMETER(pBuilder);
002191 assert( pLower || pUpper );
002192 #endif
002193 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
002194 nNew = whereRangeAdjust(pLower, nOut);
002195 nNew = whereRangeAdjust(pUpper, nNew);
002196
002197 /* TUNING: If there is both an upper and lower limit and neither limit
002198 ** has an application-defined likelihood(), assume the range is
002199 ** reduced by an additional 75%. This means that, by default, an open-ended
002200 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
002201 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
002202 ** match 1/64 of the index. */
002203 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
002204 nNew -= 20;
002205 }
002206
002207 nOut -= (pLower!=0) + (pUpper!=0);
002208 if( nNew<10 ) nNew = 10;
002209 if( nNew<nOut ) nOut = nNew;
002210 #if defined(WHERETRACE_ENABLED)
002211 if( pLoop->nOut>nOut ){
002212 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n",
002213 pLoop->nOut, nOut));
002214 }
002215 #endif
002216 pLoop->nOut = (LogEst)nOut;
002217 return rc;
002218 }
002219
002220 #ifdef SQLITE_ENABLE_STAT4
002221 /*
002222 ** Estimate the number of rows that will be returned based on
002223 ** an equality constraint x=VALUE and where that VALUE occurs in
002224 ** the histogram data. This only works when x is the left-most
002225 ** column of an index and sqlite_stat4 histogram data is available
002226 ** for that index. When pExpr==NULL that means the constraint is
002227 ** "x IS NULL" instead of "x=VALUE".
002228 **
002229 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002230 ** If unable to make an estimate, leave *pnRow unchanged and return
002231 ** non-zero.
002232 **
002233 ** This routine can fail if it is unable to load a collating sequence
002234 ** required for string comparison, or if unable to allocate memory
002235 ** for a UTF conversion required for comparison. The error is stored
002236 ** in the pParse structure.
002237 */
002238 static int whereEqualScanEst(
002239 Parse *pParse, /* Parsing & code generating context */
002240 WhereLoopBuilder *pBuilder,
002241 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
002242 tRowcnt *pnRow /* Write the revised row estimate here */
002243 ){
002244 Index *p = pBuilder->pNew->u.btree.pIndex;
002245 int nEq = pBuilder->pNew->u.btree.nEq;
002246 UnpackedRecord *pRec = pBuilder->pRec;
002247 int rc; /* Subfunction return code */
002248 tRowcnt a[2]; /* Statistics */
002249 int bOk;
002250
002251 assert( nEq>=1 );
002252 assert( nEq<=p->nColumn );
002253 assert( p->aSample!=0 );
002254 assert( p->nSample>0 );
002255 assert( pBuilder->nRecValid<nEq );
002256
002257 /* If values are not available for all fields of the index to the left
002258 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
002259 if( pBuilder->nRecValid<(nEq-1) ){
002260 return SQLITE_NOTFOUND;
002261 }
002262
002263 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
002264 ** below would return the same value. */
002265 if( nEq>=p->nColumn ){
002266 *pnRow = 1;
002267 return SQLITE_OK;
002268 }
002269
002270 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
002271 pBuilder->pRec = pRec;
002272 if( rc!=SQLITE_OK ) return rc;
002273 if( bOk==0 ) return SQLITE_NOTFOUND;
002274 pBuilder->nRecValid = nEq;
002275
002276 whereKeyStats(pParse, p, pRec, 0, a);
002277 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
002278 p->zName, nEq-1, (int)a[1]));
002279 *pnRow = a[1];
002280
002281 return rc;
002282 }
002283 #endif /* SQLITE_ENABLE_STAT4 */
002284
002285 #ifdef SQLITE_ENABLE_STAT4
002286 /*
002287 ** Estimate the number of rows that will be returned based on
002288 ** an IN constraint where the right-hand side of the IN operator
002289 ** is a list of values. Example:
002290 **
002291 ** WHERE x IN (1,2,3,4)
002292 **
002293 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002294 ** If unable to make an estimate, leave *pnRow unchanged and return
002295 ** non-zero.
002296 **
002297 ** This routine can fail if it is unable to load a collating sequence
002298 ** required for string comparison, or if unable to allocate memory
002299 ** for a UTF conversion required for comparison. The error is stored
002300 ** in the pParse structure.
002301 */
002302 static int whereInScanEst(
002303 Parse *pParse, /* Parsing & code generating context */
002304 WhereLoopBuilder *pBuilder,
002305 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
002306 tRowcnt *pnRow /* Write the revised row estimate here */
002307 ){
002308 Index *p = pBuilder->pNew->u.btree.pIndex;
002309 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
002310 int nRecValid = pBuilder->nRecValid;
002311 int rc = SQLITE_OK; /* Subfunction return code */
002312 tRowcnt nEst; /* Number of rows for a single term */
002313 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
002314 int i; /* Loop counter */
002315
002316 assert( p->aSample!=0 );
002317 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
002318 nEst = nRow0;
002319 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
002320 nRowEst += nEst;
002321 pBuilder->nRecValid = nRecValid;
002322 }
002323
002324 if( rc==SQLITE_OK ){
002325 if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0;
002326 *pnRow = nRowEst;
002327 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst));
002328 }
002329 assert( pBuilder->nRecValid==nRecValid );
002330 return rc;
002331 }
002332 #endif /* SQLITE_ENABLE_STAT4 */
002333
002334
002335 #ifdef WHERETRACE_ENABLED
002336 /*
002337 ** Print the content of a WhereTerm object
002338 */
002339 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
002340 if( pTerm==0 ){
002341 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
002342 }else{
002343 char zType[8];
002344 char zLeft[50];
002345 memcpy(zType, "....", 5);
002346 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
002347 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
002348 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L';
002349 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C';
002350 if( pTerm->eOperator & WO_SINGLE ){
002351 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
002352 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
002353 pTerm->leftCursor, pTerm->u.x.leftColumn);
002354 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
002355 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
002356 pTerm->u.pOrInfo->indexable);
002357 }else{
002358 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
002359 }
002360 sqlite3DebugPrintf(
002361 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
002362 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
002363 /* The 0x10000 .wheretrace flag causes extra information to be
002364 ** shown about each Term */
002365 if( sqlite3WhereTrace & 0x10000 ){
002366 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
002367 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
002368 }
002369 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){
002370 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
002371 }
002372 if( pTerm->iParent>=0 ){
002373 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
002374 }
002375 sqlite3DebugPrintf("\n");
002376 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
002377 }
002378 }
002379 #endif
002380
002381 #ifdef WHERETRACE_ENABLED
002382 /*
002383 ** Show the complete content of a WhereClause
002384 */
002385 void sqlite3WhereClausePrint(WhereClause *pWC){
002386 int i;
002387 for(i=0; i<pWC->nTerm; i++){
002388 sqlite3WhereTermPrint(&pWC->a[i], i);
002389 }
002390 }
002391 #endif
002392
002393 #ifdef WHERETRACE_ENABLED
002394 /*
002395 ** Print a WhereLoop object for debugging purposes
002396 **
002397 ** Format example:
002398 **
002399 ** .--- Position in WHERE clause rSetup, rRun, nOut ---.
002400 ** | |
002401 ** | .--- selfMask nTerm ------. |
002402 ** | | | |
002403 ** | | .-- prereq Idx wsFlags----. | |
002404 ** | | | Name | | |
002405 ** | | | __|__ nEq ---. ___|__ | __|__
002406 ** | / \ / \ / \ | / \ / \ / \
002407 ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31
002408 */
002409 void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){
002410 if( pWC ){
002411 WhereInfo *pWInfo = pWC->pWInfo;
002412 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
002413 SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
002414 Table *pTab = pItem->pSTab;
002415 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
002416 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
002417 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
002418 sqlite3DebugPrintf(" %12s",
002419 pItem->zAlias ? pItem->zAlias : pTab->zName);
002420 }else{
002421 sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d",
002422 p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab);
002423 }
002424 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002425 const char *zName;
002426 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
002427 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
002428 int i = sqlite3Strlen30(zName) - 1;
002429 while( zName[i]!='_' ) i--;
002430 zName += i;
002431 }
002432 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
002433 }else{
002434 sqlite3DebugPrintf("%20s","");
002435 }
002436 }else{
002437 char *z;
002438 if( p->u.vtab.idxStr ){
002439 z = sqlite3_mprintf("(%d,\"%s\",%#x)",
002440 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
002441 }else{
002442 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
002443 }
002444 sqlite3DebugPrintf(" %-19s", z);
002445 sqlite3_free(z);
002446 }
002447 if( p->wsFlags & WHERE_SKIPSCAN ){
002448 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
002449 }else{
002450 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
002451 }
002452 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
002453 if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){
002454 int i;
002455 for(i=0; i<p->nLTerm; i++){
002456 sqlite3WhereTermPrint(p->aLTerm[i], i);
002457 }
002458 }
002459 }
002460 void sqlite3ShowWhereLoop(const WhereLoop *p){
002461 if( p ) sqlite3WhereLoopPrint(p, 0);
002462 }
002463 void sqlite3ShowWhereLoopList(const WhereLoop *p){
002464 while( p ){
002465 sqlite3ShowWhereLoop(p);
002466 p = p->pNextLoop;
002467 }
002468 }
002469 #endif
002470
002471 /*
002472 ** Convert bulk memory into a valid WhereLoop that can be passed
002473 ** to whereLoopClear harmlessly.
002474 */
002475 static void whereLoopInit(WhereLoop *p){
002476 p->aLTerm = p->aLTermSpace;
002477 p->nLTerm = 0;
002478 p->nLSlot = ArraySize(p->aLTermSpace);
002479 p->wsFlags = 0;
002480 }
002481
002482 /*
002483 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
002484 */
002485 static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
002486 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
002487 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
002488 sqlite3_free(p->u.vtab.idxStr);
002489 p->u.vtab.needFree = 0;
002490 p->u.vtab.idxStr = 0;
002491 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
002492 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
002493 sqlite3DbFreeNN(db, p->u.btree.pIndex);
002494 p->u.btree.pIndex = 0;
002495 }
002496 }
002497 }
002498
002499 /*
002500 ** Deallocate internal memory used by a WhereLoop object. Leave the
002501 ** object in an initialized state, as if it had been newly allocated.
002502 */
002503 static void whereLoopClear(sqlite3 *db, WhereLoop *p){
002504 if( p->aLTerm!=p->aLTermSpace ){
002505 sqlite3DbFreeNN(db, p->aLTerm);
002506 p->aLTerm = p->aLTermSpace;
002507 p->nLSlot = ArraySize(p->aLTermSpace);
002508 }
002509 whereLoopClearUnion(db, p);
002510 p->nLTerm = 0;
002511 p->wsFlags = 0;
002512 }
002513
002514 /*
002515 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
002516 */
002517 static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
002518 WhereTerm **paNew;
002519 if( p->nLSlot>=n ) return SQLITE_OK;
002520 n = (n+7)&~7;
002521 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
002522 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
002523 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
002524 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
002525 p->aLTerm = paNew;
002526 p->nLSlot = n;
002527 return SQLITE_OK;
002528 }
002529
002530 /*
002531 ** Transfer content from the second pLoop into the first.
002532 */
002533 static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
002534 whereLoopClearUnion(db, pTo);
002535 if( pFrom->nLTerm > pTo->nLSlot
002536 && whereLoopResize(db, pTo, pFrom->nLTerm)
002537 ){
002538 memset(pTo, 0, WHERE_LOOP_XFER_SZ);
002539 return SQLITE_NOMEM_BKPT;
002540 }
002541 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
002542 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
002543 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
002544 pFrom->u.vtab.needFree = 0;
002545 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
002546 pFrom->u.btree.pIndex = 0;
002547 }
002548 return SQLITE_OK;
002549 }
002550
002551 /*
002552 ** Delete a WhereLoop object
002553 */
002554 static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
002555 assert( db!=0 );
002556 whereLoopClear(db, p);
002557 sqlite3DbNNFreeNN(db, p);
002558 }
002559
002560 /*
002561 ** Free a WhereInfo structure
002562 */
002563 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
002564 assert( pWInfo!=0 );
002565 assert( db!=0 );
002566 sqlite3WhereClauseClear(&pWInfo->sWC);
002567 while( pWInfo->pLoops ){
002568 WhereLoop *p = pWInfo->pLoops;
002569 pWInfo->pLoops = p->pNextLoop;
002570 whereLoopDelete(db, p);
002571 }
002572 while( pWInfo->pMemToFree ){
002573 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
002574 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
002575 pWInfo->pMemToFree = pNext;
002576 }
002577 sqlite3DbNNFreeNN(db, pWInfo);
002578 }
002579
002580 /*
002581 ** Return TRUE if X is a proper subset of Y but is of equal or less cost.
002582 ** In other words, return true if all constraints of X are also part of Y
002583 ** and Y has additional constraints that might speed the search that X lacks
002584 ** but the cost of running X is not more than the cost of running Y.
002585 **
002586 ** In other words, return true if the cost relationship between X and Y
002587 ** is inverted and needs to be adjusted.
002588 **
002589 ** Case 1:
002590 **
002591 ** (1a) X and Y use the same index.
002592 ** (1b) X has fewer == terms than Y
002593 ** (1c) Neither X nor Y use skip-scan
002594 ** (1d) X does not have a a greater cost than Y
002595 **
002596 ** Case 2:
002597 **
002598 ** (2a) X has the same or lower cost, or returns the same or fewer rows,
002599 ** than Y.
002600 ** (2b) X uses fewer WHERE clause terms than Y
002601 ** (2c) Every WHERE clause term used by X is also used by Y
002602 ** (2d) X skips at least as many columns as Y
002603 ** (2e) If X is a covering index, than Y is too
002604 */
002605 static int whereLoopCheaperProperSubset(
002606 const WhereLoop *pX, /* First WhereLoop to compare */
002607 const WhereLoop *pY /* Compare against this WhereLoop */
002608 ){
002609 int i, j;
002610 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */
002611 assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 );
002612 assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 );
002613 if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */
002614 && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */
002615 && pX->nSkip==0 && pY->nSkip==0 /* (1c) */
002616 ){
002617 return 1; /* Case 1 is true */
002618 }
002619 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
002620 return 0; /* (2b) */
002621 }
002622 if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */
002623 for(i=pX->nLTerm-1; i>=0; i--){
002624 if( pX->aLTerm[i]==0 ) continue;
002625 for(j=pY->nLTerm-1; j>=0; j--){
002626 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
002627 }
002628 if( j<0 ) return 0; /* (2c) */
002629 }
002630 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
002631 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
002632 return 0; /* (2e) */
002633 }
002634 return 1; /* Case 2 is true */
002635 }
002636
002637 /*
002638 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate
002639 ** upwards or downwards so that:
002640 **
002641 ** (1) pTemplate costs less than any other WhereLoops that are a proper
002642 ** subset of pTemplate
002643 **
002644 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate
002645 ** is a proper subset.
002646 **
002647 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
002648 ** WHERE clause terms than Y and that every WHERE clause term used by X is
002649 ** also used by Y.
002650 */
002651 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
002652 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
002653 for(; p; p=p->pNextLoop){
002654 if( p->iTab!=pTemplate->iTab ) continue;
002655 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
002656 if( whereLoopCheaperProperSubset(p, pTemplate) ){
002657 /* Adjust pTemplate cost downward so that it is cheaper than its
002658 ** subset p. */
002659 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002660 pTemplate->rRun, pTemplate->nOut,
002661 MIN(p->rRun, pTemplate->rRun),
002662 MIN(p->nOut - 1, pTemplate->nOut)));
002663 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
002664 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut);
002665 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
002666 /* Adjust pTemplate cost upward so that it is costlier than p since
002667 ** pTemplate is a proper subset of p */
002668 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002669 pTemplate->rRun, pTemplate->nOut,
002670 MAX(p->rRun, pTemplate->rRun),
002671 MAX(p->nOut + 1, pTemplate->nOut)));
002672 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
002673 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut);
002674 }
002675 }
002676 }
002677
002678 /*
002679 ** Search the list of WhereLoops in *ppPrev looking for one that can be
002680 ** replaced by pTemplate.
002681 **
002682 ** Return NULL if pTemplate does not belong on the WhereLoop list.
002683 ** In other words if pTemplate ought to be dropped from further consideration.
002684 **
002685 ** If pX is a WhereLoop that pTemplate can replace, then return the
002686 ** link that points to pX.
002687 **
002688 ** If pTemplate cannot replace any existing element of the list but needs
002689 ** to be added to the list as a new entry, then return a pointer to the
002690 ** tail of the list.
002691 */
002692 static WhereLoop **whereLoopFindLesser(
002693 WhereLoop **ppPrev,
002694 const WhereLoop *pTemplate
002695 ){
002696 WhereLoop *p;
002697 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
002698 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
002699 /* If either the iTab or iSortIdx values for two WhereLoop are different
002700 ** then those WhereLoops need to be considered separately. Neither is
002701 ** a candidate to replace the other. */
002702 continue;
002703 }
002704 /* In the current implementation, the rSetup value is either zero
002705 ** or the cost of building an automatic index (NlogN) and the NlogN
002706 ** is the same for compatible WhereLoops. */
002707 assert( p->rSetup==0 || pTemplate->rSetup==0
002708 || p->rSetup==pTemplate->rSetup );
002709
002710 /* whereLoopAddBtree() always generates and inserts the automatic index
002711 ** case first. Hence compatible candidate WhereLoops never have a larger
002712 ** rSetup. Call this SETUP-INVARIANT */
002713 assert( p->rSetup>=pTemplate->rSetup );
002714
002715 /* Any loop using an application-defined index (or PRIMARY KEY or
002716 ** UNIQUE constraint) with one or more == constraints is better
002717 ** than an automatic index. Unless it is a skip-scan. */
002718 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
002719 && (pTemplate->nSkip)==0
002720 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
002721 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
002722 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
002723 ){
002724 break;
002725 }
002726
002727 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
002728 ** discarded. WhereLoop p is better if:
002729 ** (1) p has no more dependencies than pTemplate, and
002730 ** (2) p has an equal or lower cost than pTemplate
002731 */
002732 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
002733 && p->rSetup<=pTemplate->rSetup /* (2a) */
002734 && p->rRun<=pTemplate->rRun /* (2b) */
002735 && p->nOut<=pTemplate->nOut /* (2c) */
002736 ){
002737 return 0; /* Discard pTemplate */
002738 }
002739
002740 /* If pTemplate is always better than p, then cause p to be overwritten
002741 ** with pTemplate. pTemplate is better than p if:
002742 ** (1) pTemplate has no more dependencies than p, and
002743 ** (2) pTemplate has an equal or lower cost than p.
002744 */
002745 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
002746 && p->rRun>=pTemplate->rRun /* (2a) */
002747 && p->nOut>=pTemplate->nOut /* (2b) */
002748 ){
002749 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
002750 break; /* Cause p to be overwritten by pTemplate */
002751 }
002752 }
002753 return ppPrev;
002754 }
002755
002756 /*
002757 ** Insert or replace a WhereLoop entry using the template supplied.
002758 **
002759 ** An existing WhereLoop entry might be overwritten if the new template
002760 ** is better and has fewer dependencies. Or the template will be ignored
002761 ** and no insert will occur if an existing WhereLoop is faster and has
002762 ** fewer dependencies than the template. Otherwise a new WhereLoop is
002763 ** added based on the template.
002764 **
002765 ** If pBuilder->pOrSet is not NULL then we care about only the
002766 ** prerequisites and rRun and nOut costs of the N best loops. That
002767 ** information is gathered in the pBuilder->pOrSet object. This special
002768 ** processing mode is used only for OR clause processing.
002769 **
002770 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
002771 ** still might overwrite similar loops with the new template if the
002772 ** new template is better. Loops may be overwritten if the following
002773 ** conditions are met:
002774 **
002775 ** (1) They have the same iTab.
002776 ** (2) They have the same iSortIdx.
002777 ** (3) The template has same or fewer dependencies than the current loop
002778 ** (4) The template has the same or lower cost than the current loop
002779 */
002780 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
002781 WhereLoop **ppPrev, *p;
002782 WhereInfo *pWInfo = pBuilder->pWInfo;
002783 sqlite3 *db = pWInfo->pParse->db;
002784 int rc;
002785
002786 /* Stop the search once we hit the query planner search limit */
002787 if( pBuilder->iPlanLimit==0 ){
002788 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
002789 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0;
002790 return SQLITE_DONE;
002791 }
002792 pBuilder->iPlanLimit--;
002793
002794 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
002795
002796 /* If pBuilder->pOrSet is defined, then only keep track of the costs
002797 ** and prereqs.
002798 */
002799 if( pBuilder->pOrSet!=0 ){
002800 if( pTemplate->nLTerm ){
002801 #if WHERETRACE_ENABLED
002802 u16 n = pBuilder->pOrSet->n;
002803 int x =
002804 #endif
002805 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
002806 pTemplate->nOut);
002807 #if WHERETRACE_ENABLED /* 0x8 */
002808 if( sqlite3WhereTrace & 0x8 ){
002809 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
002810 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002811 }
002812 #endif
002813 }
002814 return SQLITE_OK;
002815 }
002816
002817 /* Look for an existing WhereLoop to replace with pTemplate
002818 */
002819 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
002820
002821 if( ppPrev==0 ){
002822 /* There already exists a WhereLoop on the list that is better
002823 ** than pTemplate, so just ignore pTemplate */
002824 #if WHERETRACE_ENABLED /* 0x8 */
002825 if( sqlite3WhereTrace & 0x8 ){
002826 sqlite3DebugPrintf(" skip: ");
002827 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002828 }
002829 #endif
002830 return SQLITE_OK;
002831 }else{
002832 p = *ppPrev;
002833 }
002834
002835 /* If we reach this point it means that either p[] should be overwritten
002836 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
002837 ** WhereLoop and insert it.
002838 */
002839 #if WHERETRACE_ENABLED /* 0x8 */
002840 if( sqlite3WhereTrace & 0x8 ){
002841 if( p!=0 ){
002842 sqlite3DebugPrintf("replace: ");
002843 sqlite3WhereLoopPrint(p, pBuilder->pWC);
002844 sqlite3DebugPrintf(" with: ");
002845 }else{
002846 sqlite3DebugPrintf(" add: ");
002847 }
002848 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002849 }
002850 #endif
002851 if( p==0 ){
002852 /* Allocate a new WhereLoop to add to the end of the list */
002853 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
002854 if( p==0 ) return SQLITE_NOMEM_BKPT;
002855 whereLoopInit(p);
002856 p->pNextLoop = 0;
002857 }else{
002858 /* We will be overwriting WhereLoop p[]. But before we do, first
002859 ** go through the rest of the list and delete any other entries besides
002860 ** p[] that are also supplanted by pTemplate */
002861 WhereLoop **ppTail = &p->pNextLoop;
002862 WhereLoop *pToDel;
002863 while( *ppTail ){
002864 ppTail = whereLoopFindLesser(ppTail, pTemplate);
002865 if( ppTail==0 ) break;
002866 pToDel = *ppTail;
002867 if( pToDel==0 ) break;
002868 *ppTail = pToDel->pNextLoop;
002869 #if WHERETRACE_ENABLED /* 0x8 */
002870 if( sqlite3WhereTrace & 0x8 ){
002871 sqlite3DebugPrintf(" delete: ");
002872 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC);
002873 }
002874 #endif
002875 whereLoopDelete(db, pToDel);
002876 }
002877 }
002878 rc = whereLoopXfer(db, p, pTemplate);
002879 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002880 Index *pIndex = p->u.btree.pIndex;
002881 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){
002882 p->u.btree.pIndex = 0;
002883 }
002884 }
002885 return rc;
002886 }
002887
002888 /*
002889 ** Adjust the WhereLoop.nOut value downward to account for terms of the
002890 ** WHERE clause that reference the loop but which are not used by an
002891 ** index.
002892 *
002893 ** For every WHERE clause term that is not used by the index
002894 ** and which has a truth probability assigned by one of the likelihood(),
002895 ** likely(), or unlikely() SQL functions, reduce the estimated number
002896 ** of output rows by the probability specified.
002897 **
002898 ** TUNING: For every WHERE clause term that is not used by the index
002899 ** and which does not have an assigned truth probability, heuristics
002900 ** described below are used to try to estimate the truth probability.
002901 ** TODO --> Perhaps this is something that could be improved by better
002902 ** table statistics.
002903 **
002904 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
002905 ** value corresponds to -1 in LogEst notation, so this means decrement
002906 ** the WhereLoop.nOut field for every such WHERE clause term.
002907 **
002908 ** Heuristic 2: If there exists one or more WHERE clause terms of the
002909 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
002910 ** final output row estimate is no greater than 1/4 of the total number
002911 ** of rows in the table. In other words, assume that x==EXPR will filter
002912 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
002913 ** "x" column is boolean or else -1 or 0 or 1 is a common default value
002914 ** on the "x" column and so in that case only cap the output row estimate
002915 ** at 1/2 instead of 1/4.
002916 */
002917 static void whereLoopOutputAdjust(
002918 WhereClause *pWC, /* The WHERE clause */
002919 WhereLoop *pLoop, /* The loop to adjust downward */
002920 LogEst nRow /* Number of rows in the entire table */
002921 ){
002922 WhereTerm *pTerm, *pX;
002923 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
002924 int i, j;
002925 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
002926
002927 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
002928 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){
002929 assert( pTerm!=0 );
002930 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
002931 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
002932 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
002933 for(j=pLoop->nLTerm-1; j>=0; j--){
002934 pX = pLoop->aLTerm[j];
002935 if( pX==0 ) continue;
002936 if( pX==pTerm ) break;
002937 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
002938 }
002939 if( j<0 ){
002940 sqlite3ProgressCheck(pWC->pWInfo->pParse);
002941 if( pLoop->maskSelf==pTerm->prereqAll ){
002942 /* If there are extra terms in the WHERE clause not used by an index
002943 ** that depend only on the table being scanned, and that will tend to
002944 ** cause many rows to be omitted, then mark that table as
002945 ** "self-culling".
002946 **
002947 ** 2022-03-24: Self-culling only applies if either the extra terms
002948 ** are straight comparison operators that are non-true with NULL
002949 ** operand, or if the loop is not an OUTER JOIN.
002950 */
002951 if( (pTerm->eOperator & 0x3f)!=0
002952 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype
002953 & (JT_LEFT|JT_LTORJ))==0
002954 ){
002955 pLoop->wsFlags |= WHERE_SELFCULL;
002956 }
002957 }
002958 if( pTerm->truthProb<=0 ){
002959 /* If a truth probability is specified using the likelihood() hints,
002960 ** then use the probability provided by the application. */
002961 pLoop->nOut += pTerm->truthProb;
002962 }else{
002963 /* In the absence of explicit truth probabilities, use heuristics to
002964 ** guess a reasonable truth probability. */
002965 pLoop->nOut--;
002966 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
002967 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */
002968 ){
002969 Expr *pRight = pTerm->pExpr->pRight;
002970 int k = 0;
002971 testcase( pTerm->pExpr->op==TK_IS );
002972 if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){
002973 k = 10;
002974 }else{
002975 k = 20;
002976 }
002977 if( iReduce<k ){
002978 pTerm->wtFlags |= TERM_HEURTRUTH;
002979 iReduce = k;
002980 }
002981 }
002982 }
002983 }
002984 }
002985 if( pLoop->nOut > nRow-iReduce ){
002986 pLoop->nOut = nRow - iReduce;
002987 }
002988 }
002989
002990 /*
002991 ** Term pTerm is a vector range comparison operation. The first comparison
002992 ** in the vector can be optimized using column nEq of the index. This
002993 ** function returns the total number of vector elements that can be used
002994 ** as part of the range comparison.
002995 **
002996 ** For example, if the query is:
002997 **
002998 ** WHERE a = ? AND (b, c, d) > (?, ?, ?)
002999 **
003000 ** and the index:
003001 **
003002 ** CREATE INDEX ... ON (a, b, c, d, e)
003003 **
003004 ** then this function would be invoked with nEq=1. The value returned in
003005 ** this case is 3.
003006 */
003007 static int whereRangeVectorLen(
003008 Parse *pParse, /* Parsing context */
003009 int iCur, /* Cursor open on pIdx */
003010 Index *pIdx, /* The index to be used for a inequality constraint */
003011 int nEq, /* Number of prior equality constraints on same index */
003012 WhereTerm *pTerm /* The vector inequality constraint */
003013 ){
003014 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
003015 int i;
003016
003017 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
003018 for(i=1; i<nCmp; i++){
003019 /* Test if comparison i of pTerm is compatible with column (i+nEq)
003020 ** of the index. If not, exit the loop. */
003021 char aff; /* Comparison affinity */
003022 char idxaff = 0; /* Indexed columns affinity */
003023 CollSeq *pColl; /* Comparison collation sequence */
003024 Expr *pLhs, *pRhs;
003025
003026 assert( ExprUseXList(pTerm->pExpr->pLeft) );
003027 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
003028 pRhs = pTerm->pExpr->pRight;
003029 if( ExprUseXSelect(pRhs) ){
003030 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
003031 }else{
003032 pRhs = pRhs->x.pList->a[i].pExpr;
003033 }
003034
003035 /* Check that the LHS of the comparison is a column reference to
003036 ** the right column of the right source table. And that the sort
003037 ** order of the index column is the same as the sort order of the
003038 ** leftmost index column. */
003039 if( pLhs->op!=TK_COLUMN
003040 || pLhs->iTable!=iCur
003041 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
003042 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
003043 ){
003044 break;
003045 }
003046
003047 testcase( pLhs->iColumn==XN_ROWID );
003048 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
003049 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
003050 if( aff!=idxaff ) break;
003051
003052 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
003053 if( pColl==0 ) break;
003054 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
003055 }
003056 return i;
003057 }
003058
003059 /*
003060 ** Adjust the cost C by the costMult factor T. This only occurs if
003061 ** compiled with -DSQLITE_ENABLE_COSTMULT
003062 */
003063 #ifdef SQLITE_ENABLE_COSTMULT
003064 # define ApplyCostMultiplier(C,T) C += T
003065 #else
003066 # define ApplyCostMultiplier(C,T)
003067 #endif
003068
003069 /*
003070 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
003071 ** index pIndex. Try to match one more.
003072 **
003073 ** When this function is called, pBuilder->pNew->nOut contains the
003074 ** number of rows expected to be visited by filtering using the nEq
003075 ** terms only. If it is modified, this value is restored before this
003076 ** function returns.
003077 **
003078 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
003079 ** a fake index used for the INTEGER PRIMARY KEY.
003080 */
003081 static int whereLoopAddBtreeIndex(
003082 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
003083 SrcItem *pSrc, /* FROM clause term being analyzed */
003084 Index *pProbe, /* An index on pSrc */
003085 LogEst nInMul /* log(Number of iterations due to IN) */
003086 ){
003087 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */
003088 Parse *pParse = pWInfo->pParse; /* Parsing context */
003089 sqlite3 *db = pParse->db; /* Database connection malloc context */
003090 WhereLoop *pNew; /* Template WhereLoop under construction */
003091 WhereTerm *pTerm; /* A WhereTerm under consideration */
003092 int opMask; /* Valid operators for constraints */
003093 WhereScan scan; /* Iterator for WHERE terms */
003094 Bitmask saved_prereq; /* Original value of pNew->prereq */
003095 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
003096 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
003097 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
003098 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
003099 u16 saved_nSkip; /* Original value of pNew->nSkip */
003100 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
003101 LogEst saved_nOut; /* Original value of pNew->nOut */
003102 int rc = SQLITE_OK; /* Return code */
003103 LogEst rSize; /* Number of rows in the table */
003104 LogEst rLogSize; /* Logarithm of table size */
003105 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
003106
003107 pNew = pBuilder->pNew;
003108 assert( db->mallocFailed==0 || pParse->nErr>0 );
003109 if( pParse->nErr ){
003110 return pParse->rc;
003111 }
003112 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
003113 pProbe->pTable->zName,pProbe->zName,
003114 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));
003115
003116 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
003117 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
003118 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
003119 opMask = WO_LT|WO_LE;
003120 }else{
003121 assert( pNew->u.btree.nBtm==0 );
003122 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
003123 }
003124 if( pProbe->bUnordered || pProbe->bLowQual ){
003125 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
003126 if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){
003127 opMask &= ~(WO_EQ|WO_IN|WO_IS);
003128 }
003129 }
003130
003131 assert( pNew->u.btree.nEq<pProbe->nColumn );
003132 assert( pNew->u.btree.nEq<pProbe->nKeyCol
003133 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY );
003134
003135 saved_nEq = pNew->u.btree.nEq;
003136 saved_nBtm = pNew->u.btree.nBtm;
003137 saved_nTop = pNew->u.btree.nTop;
003138 saved_nSkip = pNew->nSkip;
003139 saved_nLTerm = pNew->nLTerm;
003140 saved_wsFlags = pNew->wsFlags;
003141 saved_prereq = pNew->prereq;
003142 saved_nOut = pNew->nOut;
003143 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
003144 opMask, pProbe);
003145 pNew->rSetup = 0;
003146 rSize = pProbe->aiRowLogEst[0];
003147 rLogSize = estLog(rSize);
003148 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
003149 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
003150 LogEst rCostIdx;
003151 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
003152 int nIn = 0;
003153 #ifdef SQLITE_ENABLE_STAT4
003154 int nRecValid = pBuilder->nRecValid;
003155 #endif
003156 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
003157 && indexColumnNotNull(pProbe, saved_nEq)
003158 ){
003159 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
003160 }
003161 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003162
003163 /* Do not allow the upper bound of a LIKE optimization range constraint
003164 ** to mix with a lower range bound from some other source */
003165 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
003166
003167 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
003168 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
003169 ){
003170 continue;
003171 }
003172 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
003173 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
003174 }else{
003175 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
003176 }
003177 pNew->wsFlags = saved_wsFlags;
003178 pNew->u.btree.nEq = saved_nEq;
003179 pNew->u.btree.nBtm = saved_nBtm;
003180 pNew->u.btree.nTop = saved_nTop;
003181 pNew->nLTerm = saved_nLTerm;
003182 if( pNew->nLTerm>=pNew->nLSlot
003183 && whereLoopResize(db, pNew, pNew->nLTerm+1)
003184 ){
003185 break; /* OOM while trying to enlarge the pNew->aLTerm array */
003186 }
003187 pNew->aLTerm[pNew->nLTerm++] = pTerm;
003188 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
003189
003190 assert( nInMul==0
003191 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
003192 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
003193 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
003194 );
003195
003196 if( eOp & WO_IN ){
003197 Expr *pExpr = pTerm->pExpr;
003198 if( ExprUseXSelect(pExpr) ){
003199 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
003200 int i;
003201 nIn = 46; assert( 46==sqlite3LogEst(25) );
003202
003203 /* The expression may actually be of the form (x, y) IN (SELECT...).
003204 ** In this case there is a separate term for each of (x) and (y).
003205 ** However, the nIn multiplier should only be applied once, not once
003206 ** for each such term. The following loop checks that pTerm is the
003207 ** first such term in use, and sets nIn back to 0 if it is not. */
003208 for(i=0; i<pNew->nLTerm-1; i++){
003209 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
003210 }
003211 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
003212 /* "x IN (value, value, ...)" */
003213 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
003214 }
003215 if( pProbe->hasStat1 && rLogSize>=10 ){
003216 LogEst M, logK, x;
003217 /* Let:
003218 ** N = the total number of rows in the table
003219 ** K = the number of entries on the RHS of the IN operator
003220 ** M = the number of rows in the table that match terms to the
003221 ** to the left in the same index. If the IN operator is on
003222 ** the left-most index column, M==N.
003223 **
003224 ** Given the definitions above, it is better to omit the IN operator
003225 ** from the index lookup and instead do a scan of the M elements,
003226 ** testing each scanned row against the IN operator separately, if:
003227 **
003228 ** M*log(K) < K*log(N)
003229 **
003230 ** Our estimates for M, K, and N might be inaccurate, so we build in
003231 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
003232 ** with the index, as using an index has better worst-case behavior.
003233 ** If we do not have real sqlite_stat1 data, always prefer to use
003234 ** the index. Do not bother with this optimization on very small
003235 ** tables (less than 2 rows) as it is pointless in that case.
003236 */
003237 M = pProbe->aiRowLogEst[saved_nEq];
003238 logK = estLog(nIn);
003239 /* TUNING v----- 10 to bias toward indexed IN */
003240 x = M + logK + 10 - (nIn + rLogSize);
003241 if( x>=0 ){
003242 WHERETRACE(0x40,
003243 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
003244 "prefers indexed lookup\n",
003245 saved_nEq, M, logK, nIn, rLogSize, x));
003246 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
003247 WHERETRACE(0x40,
003248 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003249 " nInMul=%d) prefers skip-scan\n",
003250 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003251 pNew->wsFlags |= WHERE_IN_SEEKSCAN;
003252 }else{
003253 WHERETRACE(0x40,
003254 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003255 " nInMul=%d) prefers normal scan\n",
003256 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003257 continue;
003258 }
003259 }
003260 pNew->wsFlags |= WHERE_COLUMN_IN;
003261 }else if( eOp & (WO_EQ|WO_IS) ){
003262 int iCol = pProbe->aiColumn[saved_nEq];
003263 pNew->wsFlags |= WHERE_COLUMN_EQ;
003264 assert( saved_nEq==pNew->u.btree.nEq );
003265 if( iCol==XN_ROWID
003266 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
003267 ){
003268 if( iCol==XN_ROWID || pProbe->uniqNotNull
003269 || (pProbe->nKeyCol==1 && pProbe->onError && (eOp & WO_EQ))
003270 ){
003271 pNew->wsFlags |= WHERE_ONEROW;
003272 }else{
003273 pNew->wsFlags |= WHERE_UNQ_WANTED;
003274 }
003275 }
003276 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
003277 }else if( eOp & WO_ISNULL ){
003278 pNew->wsFlags |= WHERE_COLUMN_NULL;
003279 }else{
003280 int nVecLen = whereRangeVectorLen(
003281 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
003282 );
003283 if( eOp & (WO_GT|WO_GE) ){
003284 testcase( eOp & WO_GT );
003285 testcase( eOp & WO_GE );
003286 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
003287 pNew->u.btree.nBtm = nVecLen;
003288 pBtm = pTerm;
003289 pTop = 0;
003290 if( pTerm->wtFlags & TERM_LIKEOPT ){
003291 /* Range constraints that come from the LIKE optimization are
003292 ** always used in pairs. */
003293 pTop = &pTerm[1];
003294 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
003295 assert( pTop->wtFlags & TERM_LIKEOPT );
003296 assert( pTop->eOperator==WO_LT );
003297 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
003298 pNew->aLTerm[pNew->nLTerm++] = pTop;
003299 pNew->wsFlags |= WHERE_TOP_LIMIT;
003300 pNew->u.btree.nTop = 1;
003301 }
003302 }else{
003303 assert( eOp & (WO_LT|WO_LE) );
003304 testcase( eOp & WO_LT );
003305 testcase( eOp & WO_LE );
003306 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
003307 pNew->u.btree.nTop = nVecLen;
003308 pTop = pTerm;
003309 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
003310 pNew->aLTerm[pNew->nLTerm-2] : 0;
003311 }
003312 }
003313
003314 /* At this point pNew->nOut is set to the number of rows expected to
003315 ** be visited by the index scan before considering term pTerm, or the
003316 ** values of nIn and nInMul. In other words, assuming that all
003317 ** "x IN(...)" terms are replaced with "x = ?". This block updates
003318 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
003319 assert( pNew->nOut==saved_nOut );
003320 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003321 /* Adjust nOut using stat4 data. Or, if there is no stat4
003322 ** data, using some other estimate. */
003323 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
003324 }else{
003325 int nEq = ++pNew->u.btree.nEq;
003326 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
003327
003328 assert( pNew->nOut==saved_nOut );
003329 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
003330 assert( (eOp & WO_IN) || nIn==0 );
003331 testcase( eOp & WO_IN );
003332 pNew->nOut += pTerm->truthProb;
003333 pNew->nOut -= nIn;
003334 }else{
003335 #ifdef SQLITE_ENABLE_STAT4
003336 tRowcnt nOut = 0;
003337 if( nInMul==0
003338 && pProbe->nSample
003339 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
003340 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
003341 && OptimizationEnabled(db, SQLITE_Stat4)
003342 ){
003343 Expr *pExpr = pTerm->pExpr;
003344 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
003345 testcase( eOp & WO_EQ );
003346 testcase( eOp & WO_IS );
003347 testcase( eOp & WO_ISNULL );
003348 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
003349 }else{
003350 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
003351 }
003352 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
003353 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
003354 if( nOut ){
003355 pNew->nOut = sqlite3LogEst(nOut);
003356 if( nEq==1
003357 /* TUNING: Mark terms as "low selectivity" if they seem likely
003358 ** to be true for half or more of the rows in the table.
003359 ** See tag-202002240-1 */
003360 && pNew->nOut+10 > pProbe->aiRowLogEst[0]
003361 ){
003362 #if WHERETRACE_ENABLED /* 0x01 */
003363 if( sqlite3WhereTrace & 0x20 ){
003364 sqlite3DebugPrintf(
003365 "STAT4 determines term has low selectivity:\n");
003366 sqlite3WhereTermPrint(pTerm, 999);
003367 }
003368 #endif
003369 pTerm->wtFlags |= TERM_HIGHTRUTH;
003370 if( pTerm->wtFlags & TERM_HEURTRUTH ){
003371 /* If the term has previously been used with an assumption of
003372 ** higher selectivity, then set the flag to rerun the
003373 ** loop computations. */
003374 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
003375 }
003376 }
003377 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
003378 pNew->nOut -= nIn;
003379 }
003380 }
003381 if( nOut==0 )
003382 #endif
003383 {
003384 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
003385 if( eOp & WO_ISNULL ){
003386 /* TUNING: If there is no likelihood() value, assume that a
003387 ** "col IS NULL" expression matches twice as many rows
003388 ** as (col=?). */
003389 pNew->nOut += 10;
003390 }
003391 }
003392 }
003393 }
003394
003395 /* Set rCostIdx to the estimated cost of visiting selected rows in the
003396 ** index. The estimate is the sum of two values:
003397 ** 1. The cost of doing one search-by-key to find the first matching
003398 ** entry
003399 ** 2. Stepping forward in the index pNew->nOut times to find all
003400 ** additional matching entries.
003401 */
003402 assert( pSrc->pSTab->szTabRow>0 );
003403 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003404 /* The pProbe->szIdxRow is low for an IPK table since the interior
003405 ** pages are small. Thus szIdxRow gives a good estimate of seek cost.
003406 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
003407 ** under-estimate the scanning cost. */
003408 rCostIdx = pNew->nOut + 16;
003409 }else{
003410 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow;
003411 }
003412 rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx);
003413
003414 /* Estimate the cost of running the loop. If all data is coming
003415 ** from the index, then this is just the cost of doing the index
003416 ** lookup and scan. But if some data is coming out of the main table,
003417 ** we also have to add in the cost of doing pNew->nOut searches to
003418 ** locate the row in the main table that corresponds to the index entry.
003419 */
003420 pNew->rRun = rCostIdx;
003421 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
003422 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
003423 }
003424 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
003425
003426 nOutUnadjusted = pNew->nOut;
003427 pNew->rRun += nInMul + nIn;
003428 pNew->nOut += nInMul + nIn;
003429 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
003430 rc = whereLoopInsert(pBuilder, pNew);
003431
003432 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003433 pNew->nOut = saved_nOut;
003434 }else{
003435 pNew->nOut = nOutUnadjusted;
003436 }
003437
003438 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
003439 && pNew->u.btree.nEq<pProbe->nColumn
003440 && (pNew->u.btree.nEq<pProbe->nKeyCol ||
003441 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
003442 ){
003443 if( pNew->u.btree.nEq>3 ){
003444 sqlite3ProgressCheck(pParse);
003445 }
003446 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
003447 }
003448 pNew->nOut = saved_nOut;
003449 #ifdef SQLITE_ENABLE_STAT4
003450 pBuilder->nRecValid = nRecValid;
003451 #endif
003452 }
003453 pNew->prereq = saved_prereq;
003454 pNew->u.btree.nEq = saved_nEq;
003455 pNew->u.btree.nBtm = saved_nBtm;
003456 pNew->u.btree.nTop = saved_nTop;
003457 pNew->nSkip = saved_nSkip;
003458 pNew->wsFlags = saved_wsFlags;
003459 pNew->nOut = saved_nOut;
003460 pNew->nLTerm = saved_nLTerm;
003461
003462 /* Consider using a skip-scan if there are no WHERE clause constraints
003463 ** available for the left-most terms of the index, and if the average
003464 ** number of repeats in the left-most terms is at least 18.
003465 **
003466 ** The magic number 18 is selected on the basis that scanning 17 rows
003467 ** is almost always quicker than an index seek (even though if the index
003468 ** contains fewer than 2^17 rows we assume otherwise in other parts of
003469 ** the code). And, even if it is not, it should not be too much slower.
003470 ** On the other hand, the extra seeks could end up being significantly
003471 ** more expensive. */
003472 assert( 42==sqlite3LogEst(18) );
003473 if( saved_nEq==saved_nSkip
003474 && saved_nEq+1<pProbe->nKeyCol
003475 && saved_nEq==pNew->nLTerm
003476 && pProbe->noSkipScan==0
003477 && pProbe->hasStat1!=0
003478 && OptimizationEnabled(db, SQLITE_SkipScan)
003479 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
003480 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
003481 ){
003482 LogEst nIter;
003483 pNew->u.btree.nEq++;
003484 pNew->nSkip++;
003485 pNew->aLTerm[pNew->nLTerm++] = 0;
003486 pNew->wsFlags |= WHERE_SKIPSCAN;
003487 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
003488 pNew->nOut -= nIter;
003489 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
003490 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
003491 nIter += 5;
003492 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
003493 pNew->nOut = saved_nOut;
003494 pNew->u.btree.nEq = saved_nEq;
003495 pNew->nSkip = saved_nSkip;
003496 pNew->wsFlags = saved_wsFlags;
003497 }
003498
003499 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
003500 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
003501 return rc;
003502 }
003503
003504 /*
003505 ** Return True if it is possible that pIndex might be useful in
003506 ** implementing the ORDER BY clause in pBuilder.
003507 **
003508 ** Return False if pBuilder does not contain an ORDER BY clause or
003509 ** if there is no way for pIndex to be useful in implementing that
003510 ** ORDER BY clause.
003511 */
003512 static int indexMightHelpWithOrderBy(
003513 WhereLoopBuilder *pBuilder,
003514 Index *pIndex,
003515 int iCursor
003516 ){
003517 ExprList *pOB;
003518 ExprList *aColExpr;
003519 int ii, jj;
003520
003521 if( pIndex->bUnordered ) return 0;
003522 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
003523 for(ii=0; ii<pOB->nExpr; ii++){
003524 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
003525 if( NEVER(pExpr==0) ) continue;
003526 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN)
003527 && pExpr->iTable==iCursor
003528 ){
003529 if( pExpr->iColumn<0 ) return 1;
003530 for(jj=0; jj<pIndex->nKeyCol; jj++){
003531 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
003532 }
003533 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
003534 for(jj=0; jj<pIndex->nKeyCol; jj++){
003535 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
003536 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
003537 return 1;
003538 }
003539 }
003540 }
003541 }
003542 return 0;
003543 }
003544
003545 /* Check to see if a partial index with pPartIndexWhere can be used
003546 ** in the current query. Return true if it can be and false if not.
003547 */
003548 static int whereUsablePartialIndex(
003549 int iTab, /* The table for which we want an index */
003550 u8 jointype, /* The JT_* flags on the join */
003551 WhereClause *pWC, /* The WHERE clause of the query */
003552 Expr *pWhere /* The WHERE clause from the partial index */
003553 ){
003554 int i;
003555 WhereTerm *pTerm;
003556 Parse *pParse;
003557
003558 if( jointype & JT_LTORJ ) return 0;
003559 pParse = pWC->pWInfo->pParse;
003560 while( pWhere->op==TK_AND ){
003561 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
003562 pWhere = pWhere->pRight;
003563 }
003564 if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
003565 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
003566 Expr *pExpr;
003567 pExpr = pTerm->pExpr;
003568 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
003569 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
003570 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
003571 && (pTerm->wtFlags & TERM_VNULL)==0
003572 ){
003573 return 1;
003574 }
003575 }
003576 return 0;
003577 }
003578
003579 /*
003580 ** pIdx is an index containing expressions. Check it see if any of the
003581 ** expressions in the index match the pExpr expression.
003582 */
003583 static int exprIsCoveredByIndex(
003584 const Expr *pExpr,
003585 const Index *pIdx,
003586 int iTabCur
003587 ){
003588 int i;
003589 for(i=0; i<pIdx->nColumn; i++){
003590 if( pIdx->aiColumn[i]==XN_EXPR
003591 && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0
003592 ){
003593 return 1;
003594 }
003595 }
003596 return 0;
003597 }
003598
003599 /*
003600 ** Structure passed to the whereIsCoveringIndex Walker callback.
003601 */
003602 typedef struct CoveringIndexCheck CoveringIndexCheck;
003603 struct CoveringIndexCheck {
003604 Index *pIdx; /* The index */
003605 int iTabCur; /* Cursor number for the corresponding table */
003606 u8 bExpr; /* Uses an indexed expression */
003607 u8 bUnidx; /* Uses an unindexed column not within an indexed expr */
003608 };
003609
003610 /*
003611 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
003612 **
003613 ** If the Expr node references the table with cursor pCk->iTabCur, then
003614 ** make sure that column is covered by the index pCk->pIdx. We know that
003615 ** all columns less than 63 (really BMS-1) are covered, so we don't need
003616 ** to check them. But we do need to check any column at 63 or greater.
003617 **
003618 ** If the index does not cover the column, then set pWalk->eCode to
003619 ** non-zero and return WRC_Abort to stop the search.
003620 **
003621 ** If this node does not disprove that the index can be a covering index,
003622 ** then just return WRC_Continue, to continue the search.
003623 **
003624 ** If pCk->pIdx contains indexed expressions and one of those expressions
003625 ** matches pExpr, then prune the search.
003626 */
003627 static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
003628 int i; /* Loop counter */
003629 const Index *pIdx; /* The index of interest */
003630 const i16 *aiColumn; /* Columns contained in the index */
003631 u16 nColumn; /* Number of columns in the index */
003632 CoveringIndexCheck *pCk; /* Info about this search */
003633
003634 pCk = pWalk->u.pCovIdxCk;
003635 pIdx = pCk->pIdx;
003636 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){
003637 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
003638 if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue;
003639 pIdx = pWalk->u.pCovIdxCk->pIdx;
003640 aiColumn = pIdx->aiColumn;
003641 nColumn = pIdx->nColumn;
003642 for(i=0; i<nColumn; i++){
003643 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
003644 }
003645 pCk->bUnidx = 1;
003646 return WRC_Abort;
003647 }else if( pIdx->bHasExpr
003648 && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){
003649 pCk->bExpr = 1;
003650 return WRC_Prune;
003651 }
003652 return WRC_Continue;
003653 }
003654
003655
003656 /*
003657 ** pIdx is an index that covers all of the low-number columns used by
003658 ** pWInfo->pSelect (columns from 0 through 62) or an index that has
003659 ** expressions terms. Hence, we cannot determine whether or not it is
003660 ** a covering index by using the colUsed bitmasks. We have to do a search
003661 ** to see if the index is covering. This routine does that search.
003662 **
003663 ** The return value is one of these:
003664 **
003665 ** 0 The index is definitely not a covering index
003666 **
003667 ** WHERE_IDX_ONLY The index is definitely a covering index
003668 **
003669 ** WHERE_EXPRIDX The index is likely a covering index, but it is
003670 ** difficult to determine precisely because of the
003671 ** expressions that are indexed. Score it as a
003672 ** covering index, but still keep the main table open
003673 ** just in case we need it.
003674 **
003675 ** This routine is an optimization. It is always safe to return zero.
003676 ** But returning one of the other two values when zero should have been
003677 ** returned can lead to incorrect bytecode and assertion faults.
003678 */
003679 static SQLITE_NOINLINE u32 whereIsCoveringIndex(
003680 WhereInfo *pWInfo, /* The WHERE clause context */
003681 Index *pIdx, /* Index that is being tested */
003682 int iTabCur /* Cursor for the table being indexed */
003683 ){
003684 int i, rc;
003685 struct CoveringIndexCheck ck;
003686 Walker w;
003687 if( pWInfo->pSelect==0 ){
003688 /* We don't have access to the full query, so we cannot check to see
003689 ** if pIdx is covering. Assume it is not. */
003690 return 0;
003691 }
003692 if( pIdx->bHasExpr==0 ){
003693 for(i=0; i<pIdx->nColumn; i++){
003694 if( pIdx->aiColumn[i]>=BMS-1 ) break;
003695 }
003696 if( i>=pIdx->nColumn ){
003697 /* pIdx does not index any columns greater than 62, but we know from
003698 ** colMask that columns greater than 62 are used, so this is not a
003699 ** covering index */
003700 return 0;
003701 }
003702 }
003703 ck.pIdx = pIdx;
003704 ck.iTabCur = iTabCur;
003705 ck.bExpr = 0;
003706 ck.bUnidx = 0;
003707 memset(&w, 0, sizeof(w));
003708 w.xExprCallback = whereIsCoveringIndexWalkCallback;
003709 w.xSelectCallback = sqlite3SelectWalkNoop;
003710 w.u.pCovIdxCk = &ck;
003711 sqlite3WalkSelect(&w, pWInfo->pSelect);
003712 if( ck.bUnidx ){
003713 rc = 0;
003714 }else if( ck.bExpr ){
003715 rc = WHERE_EXPRIDX;
003716 }else{
003717 rc = WHERE_IDX_ONLY;
003718 }
003719 return rc;
003720 }
003721
003722 /*
003723 ** This is an sqlite3ParserAddCleanup() callback that is invoked to
003724 ** free the Parse->pIdxEpr list when the Parse object is destroyed.
003725 */
003726 static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
003727 IndexedExpr **pp = (IndexedExpr**)pObject;
003728 while( *pp!=0 ){
003729 IndexedExpr *p = *pp;
003730 *pp = p->pIENext;
003731 sqlite3ExprDelete(db, p->pExpr);
003732 sqlite3DbFreeNN(db, p);
003733 }
003734 }
003735
003736 /*
003737 ** This function is called for a partial index - one with a WHERE clause - in
003738 ** two scenarios. In both cases, it determines whether or not the WHERE
003739 ** clause on the index implies that a column of the table may be safely
003740 ** replaced by a constant expression. For example, in the following
003741 ** SELECT:
003742 **
003743 ** CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>;
003744 ** SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?;
003745 **
003746 ** The "a" in the select-list may be replaced by <expr>, iff:
003747 **
003748 ** (a) <expr> is a constant expression, and
003749 ** (b) The (a=<expr>) comparison uses the BINARY collation sequence, and
003750 ** (c) Column "a" has an affinity other than NONE or BLOB.
003751 **
003752 ** If argument pItem is NULL, then pMask must not be NULL. In this case this
003753 ** function is being called as part of determining whether or not pIdx
003754 ** is a covering index. This function clears any bits in (*pMask)
003755 ** corresponding to columns that may be replaced by constants as described
003756 ** above.
003757 **
003758 ** Otherwise, if pItem is not NULL, then this function is being called
003759 ** as part of coding a loop that uses index pIdx. In this case, add entries
003760 ** to the Parse.pIdxPartExpr list for each column that can be replaced
003761 ** by a constant.
003762 */
003763 static void wherePartIdxExpr(
003764 Parse *pParse, /* Parse context */
003765 Index *pIdx, /* Partial index being processed */
003766 Expr *pPart, /* WHERE clause being processed */
003767 Bitmask *pMask, /* Mask to clear bits in */
003768 int iIdxCur, /* Cursor number for index */
003769 SrcItem *pItem /* The FROM clause entry for the table */
003770 ){
003771 assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 );
003772 assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) );
003773
003774 if( pPart->op==TK_AND ){
003775 wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem);
003776 pPart = pPart->pLeft;
003777 }
003778
003779 if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){
003780 Expr *pLeft = pPart->pLeft;
003781 Expr *pRight = pPart->pRight;
003782 u8 aff;
003783
003784 if( pLeft->op!=TK_COLUMN ) return;
003785 if( !sqlite3ExprIsConstant(0, pRight) ) return;
003786 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return;
003787 if( pLeft->iColumn<0 ) return;
003788 aff = pIdx->pTable->aCol[pLeft->iColumn].affinity;
003789 if( aff>=SQLITE_AFF_TEXT ){
003790 if( pItem ){
003791 sqlite3 *db = pParse->db;
003792 IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p));
003793 if( p ){
003794 int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0;
003795 p->pExpr = sqlite3ExprDup(db, pRight, 0);
003796 p->iDataCur = pItem->iCursor;
003797 p->iIdxCur = iIdxCur;
003798 p->iIdxCol = pLeft->iColumn;
003799 p->bMaybeNullRow = bNullRow;
003800 p->pIENext = pParse->pIdxPartExpr;
003801 p->aff = aff;
003802 pParse->pIdxPartExpr = p;
003803 if( p->pIENext==0 ){
003804 void *pArg = (void*)&pParse->pIdxPartExpr;
003805 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
003806 }
003807 }
003808 }else if( pLeft->iColumn<(BMS-1) ){
003809 *pMask &= ~((Bitmask)1 << pLeft->iColumn);
003810 }
003811 }
003812 }
003813 }
003814
003815
003816 /*
003817 ** Add all WhereLoop objects for a single table of the join where the table
003818 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
003819 ** a b-tree table, not a virtual table.
003820 **
003821 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
003822 ** are calculated as follows:
003823 **
003824 ** For a full scan, assuming the table (or index) contains nRow rows:
003825 **
003826 ** cost = nRow * 3.0 // full-table scan
003827 ** cost = nRow * K // scan of covering index
003828 ** cost = nRow * (K+3.0) // scan of non-covering index
003829 **
003830 ** where K is a value between 1.1 and 3.0 set based on the relative
003831 ** estimated average size of the index and table records.
003832 **
003833 ** For an index scan, where nVisit is the number of index rows visited
003834 ** by the scan, and nSeek is the number of seek operations required on
003835 ** the index b-tree:
003836 **
003837 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index
003838 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
003839 **
003840 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the
003841 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
003842 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
003843 **
003844 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
003845 ** of uncertainty. For this reason, scoring is designed to pick plans that
003846 ** "do the least harm" if the estimates are inaccurate. For example, a
003847 ** log(nRow) factor is omitted from a non-covering index scan in order to
003848 ** bias the scoring in favor of using an index, since the worst-case
003849 ** performance of using an index is far better than the worst-case performance
003850 ** of a full table scan.
003851 */
003852 static int whereLoopAddBtree(
003853 WhereLoopBuilder *pBuilder, /* WHERE clause information */
003854 Bitmask mPrereq /* Extra prerequisites for using this table */
003855 ){
003856 WhereInfo *pWInfo; /* WHERE analysis context */
003857 Index *pProbe; /* An index we are evaluating */
003858 Index sPk; /* A fake index object for the primary key */
003859 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
003860 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
003861 SrcList *pTabList; /* The FROM clause */
003862 SrcItem *pSrc; /* The FROM clause btree term to add */
003863 WhereLoop *pNew; /* Template WhereLoop object */
003864 int rc = SQLITE_OK; /* Return code */
003865 int iSortIdx = 1; /* Index number */
003866 int b; /* A boolean value */
003867 LogEst rSize; /* number of rows in the table */
003868 WhereClause *pWC; /* The parsed WHERE clause */
003869 Table *pTab; /* Table being queried */
003870
003871 pNew = pBuilder->pNew;
003872 pWInfo = pBuilder->pWInfo;
003873 pTabList = pWInfo->pTabList;
003874 pSrc = pTabList->a + pNew->iTab;
003875 pTab = pSrc->pSTab;
003876 pWC = pBuilder->pWC;
003877 assert( !IsVirtual(pSrc->pSTab) );
003878
003879 if( pSrc->fg.isIndexedBy ){
003880 assert( pSrc->fg.isCte==0 );
003881 /* An INDEXED BY clause specifies a particular index to use */
003882 pProbe = pSrc->u2.pIBIndex;
003883 }else if( !HasRowid(pTab) ){
003884 pProbe = pTab->pIndex;
003885 }else{
003886 /* There is no INDEXED BY clause. Create a fake Index object in local
003887 ** variable sPk to represent the rowid primary key index. Make this
003888 ** fake index the first in a chain of Index objects with all of the real
003889 ** indices to follow */
003890 Index *pFirst; /* First of real indices on the table */
003891 memset(&sPk, 0, sizeof(Index));
003892 sPk.nKeyCol = 1;
003893 sPk.nColumn = 1;
003894 sPk.aiColumn = &aiColumnPk;
003895 sPk.aiRowLogEst = aiRowEstPk;
003896 sPk.onError = OE_Replace;
003897 sPk.pTable = pTab;
003898 sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */
003899 sPk.idxType = SQLITE_IDXTYPE_IPK;
003900 aiRowEstPk[0] = pTab->nRowLogEst;
003901 aiRowEstPk[1] = 0;
003902 pFirst = pSrc->pSTab->pIndex;
003903 if( pSrc->fg.notIndexed==0 ){
003904 /* The real indices of the table are only considered if the
003905 ** NOT INDEXED qualifier is omitted from the FROM clause */
003906 sPk.pNext = pFirst;
003907 }
003908 pProbe = &sPk;
003909 }
003910 rSize = pTab->nRowLogEst;
003911
003912 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
003913 /* Automatic indexes */
003914 if( !pBuilder->pOrSet /* Not part of an OR optimization */
003915 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0
003916 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
003917 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */
003918 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
003919 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
003920 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
003921 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
003922 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
003923 ){
003924 /* Generate auto-index WhereLoops */
003925 LogEst rLogSize; /* Logarithm of the number of rows in the table */
003926 WhereTerm *pTerm;
003927 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
003928 rLogSize = estLog(rSize);
003929 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
003930 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003931 if( termCanDriveIndex(pTerm, pSrc, 0) ){
003932 pNew->u.btree.nEq = 1;
003933 pNew->nSkip = 0;
003934 pNew->u.btree.pIndex = 0;
003935 pNew->nLTerm = 1;
003936 pNew->aLTerm[0] = pTerm;
003937 /* TUNING: One-time cost for computing the automatic index is
003938 ** estimated to be X*N*log2(N) where N is the number of rows in
003939 ** the table being indexed and where X is 7 (LogEst=28) for normal
003940 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
003941 ** of X is smaller for views and subqueries so that the query planner
003942 ** will be more aggressive about generating automatic indexes for
003943 ** those objects, since there is no opportunity to add schema
003944 ** indexes on subqueries and views. */
003945 pNew->rSetup = rLogSize + rSize;
003946 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){
003947 pNew->rSetup += 28;
003948 }else{
003949 pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes
003950 ** on ephemeral materializations of views */
003951 }
003952 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
003953 if( pNew->rSetup<0 ) pNew->rSetup = 0;
003954 /* TUNING: Each index lookup yields 20 rows in the table. This
003955 ** is more than the usual guess of 10 rows, since we have no way
003956 ** of knowing how selective the index will ultimately be. It would
003957 ** not be unreasonable to make this value much larger. */
003958 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
003959 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
003960 pNew->wsFlags = WHERE_AUTO_INDEX;
003961 pNew->prereq = mPrereq | pTerm->prereqRight;
003962 rc = whereLoopInsert(pBuilder, pNew);
003963 }
003964 }
003965 }
003966 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
003967
003968 /* Loop over all indices. If there was an INDEXED BY clause, then only
003969 ** consider index pProbe. */
003970 for(; rc==SQLITE_OK && pProbe;
003971 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
003972 ){
003973 if( pProbe->pPartIdxWhere!=0
003974 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC,
003975 pProbe->pPartIdxWhere)
003976 ){
003977 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
003978 continue; /* Partial index inappropriate for this query */
003979 }
003980 if( pProbe->bNoQuery ) continue;
003981 rSize = pProbe->aiRowLogEst[0];
003982 pNew->u.btree.nEq = 0;
003983 pNew->u.btree.nBtm = 0;
003984 pNew->u.btree.nTop = 0;
003985 pNew->nSkip = 0;
003986 pNew->nLTerm = 0;
003987 pNew->iSortIdx = 0;
003988 pNew->rSetup = 0;
003989 pNew->prereq = mPrereq;
003990 pNew->nOut = rSize;
003991 pNew->u.btree.pIndex = pProbe;
003992 pNew->u.btree.pOrderBy = 0;
003993 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
003994
003995 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
003996 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
003997 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003998 /* Integer primary key index */
003999 pNew->wsFlags = WHERE_IPK;
004000
004001 /* Full table scan */
004002 pNew->iSortIdx = b ? iSortIdx : 0;
004003 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
004004 ** extra cost designed to discourage the use of full table scans,
004005 ** since index lookups have better worst-case performance if our
004006 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
004007 ** (to 2.75) if we have valid STAT4 information for the table.
004008 ** At 2.75, a full table scan is preferred over using an index on
004009 ** a column with just two distinct values where each value has about
004010 ** an equal number of appearances. Without STAT4 data, we still want
004011 ** to use an index in that case, since the constraint might be for
004012 ** the scarcer of the two values, and in that case an index lookup is
004013 ** better.
004014 */
004015 #ifdef SQLITE_ENABLE_STAT4
004016 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
004017 #else
004018 pNew->rRun = rSize + 16;
004019 #endif
004020 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
004021 whereLoopOutputAdjust(pWC, pNew, rSize);
004022 if( pSrc->fg.isSubquery ){
004023 if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE;
004024 pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy;
004025 }
004026 rc = whereLoopInsert(pBuilder, pNew);
004027 pNew->nOut = rSize;
004028 if( rc ) break;
004029 }else{
004030 Bitmask m;
004031 if( pProbe->isCovering ){
004032 m = 0;
004033 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
004034 }else{
004035 m = pSrc->colUsed & pProbe->colNotIdxed;
004036 if( pProbe->pPartIdxWhere ){
004037 wherePartIdxExpr(
004038 pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0
004039 );
004040 }
004041 pNew->wsFlags = WHERE_INDEXED;
004042 if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
004043 u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
004044 if( isCov==0 ){
004045 WHERETRACE(0x200,
004046 ("-> %s is not a covering index"
004047 " according to whereIsCoveringIndex()\n", pProbe->zName));
004048 assert( m!=0 );
004049 }else{
004050 m = 0;
004051 pNew->wsFlags |= isCov;
004052 if( isCov & WHERE_IDX_ONLY ){
004053 WHERETRACE(0x200,
004054 ("-> %s is a covering expression index"
004055 " according to whereIsCoveringIndex()\n", pProbe->zName));
004056 }else{
004057 assert( isCov==WHERE_EXPRIDX );
004058 WHERETRACE(0x200,
004059 ("-> %s might be a covering expression index"
004060 " according to whereIsCoveringIndex()\n", pProbe->zName));
004061 }
004062 }
004063 }else if( m==0
004064 && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700))
004065 ){
004066 WHERETRACE(0x200,
004067 ("-> %s a covering index according to bitmasks\n",
004068 pProbe->zName, m==0 ? "is" : "is not"));
004069 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
004070 }
004071 }
004072
004073 /* Full scan via index */
004074 if( b
004075 || !HasRowid(pTab)
004076 || pProbe->pPartIdxWhere!=0
004077 || pSrc->fg.isIndexedBy
004078 || ( m==0
004079 && pProbe->bUnordered==0
004080 && (pProbe->szIdxRow<pTab->szTabRow)
004081 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
004082 && sqlite3GlobalConfig.bUseCis
004083 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
004084 )
004085 ){
004086 pNew->iSortIdx = b ? iSortIdx : 0;
004087
004088 /* The cost of visiting the index rows is N*K, where K is
004089 ** between 1.1 and 3.0, depending on the relative sizes of the
004090 ** index and table rows. */
004091 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
004092 if( m!=0 ){
004093 /* If this is a non-covering index scan, add in the cost of
004094 ** doing table lookups. The cost will be 3x the number of
004095 ** lookups. Take into account WHERE clause terms that can be
004096 ** satisfied using just the index, and that do not require a
004097 ** table lookup. */
004098 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
004099 int ii;
004100 int iCur = pSrc->iCursor;
004101 WhereClause *pWC2 = &pWInfo->sWC;
004102 for(ii=0; ii<pWC2->nTerm; ii++){
004103 WhereTerm *pTerm = &pWC2->a[ii];
004104 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
004105 break;
004106 }
004107 /* pTerm can be evaluated using just the index. So reduce
004108 ** the expected number of table lookups accordingly */
004109 if( pTerm->truthProb<=0 ){
004110 nLookup += pTerm->truthProb;
004111 }else{
004112 nLookup--;
004113 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
004114 }
004115 }
004116
004117 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
004118 }
004119 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
004120 whereLoopOutputAdjust(pWC, pNew, rSize);
004121 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){
004122 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
004123 ** because the cursor used to access the index might not be
004124 ** positioned to the correct row during the right-join no-match
004125 ** loop. */
004126 }else{
004127 rc = whereLoopInsert(pBuilder, pNew);
004128 }
004129 pNew->nOut = rSize;
004130 if( rc ) break;
004131 }
004132 }
004133
004134 pBuilder->bldFlags1 = 0;
004135 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
004136 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
004137 /* If a non-unique index is used, or if a prefix of the key for
004138 ** unique index is used (making the index functionally non-unique)
004139 ** then the sqlite_stat1 data becomes important for scoring the
004140 ** plan */
004141 pTab->tabFlags |= TF_MaybeReanalyze;
004142 }
004143 #ifdef SQLITE_ENABLE_STAT4
004144 sqlite3Stat4ProbeFree(pBuilder->pRec);
004145 pBuilder->nRecValid = 0;
004146 pBuilder->pRec = 0;
004147 #endif
004148 }
004149 return rc;
004150 }
004151
004152 #ifndef SQLITE_OMIT_VIRTUALTABLE
004153
004154 /*
004155 ** Return true if pTerm is a virtual table LIMIT or OFFSET term.
004156 */
004157 static int isLimitTerm(WhereTerm *pTerm){
004158 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
004159 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
004160 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
004161 }
004162
004163 /*
004164 ** Return true if the first nCons constraints in the pUsage array are
004165 ** marked as in-use (have argvIndex>0). False otherwise.
004166 */
004167 static int allConstraintsUsed(
004168 struct sqlite3_index_constraint_usage *aUsage,
004169 int nCons
004170 ){
004171 int ii;
004172 for(ii=0; ii<nCons; ii++){
004173 if( aUsage[ii].argvIndex<=0 ) return 0;
004174 }
004175 return 1;
004176 }
004177
004178 /*
004179 ** Argument pIdxInfo is already populated with all constraints that may
004180 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
004181 ** function marks a subset of those constraints usable, invokes the
004182 ** xBestIndex method and adds the returned plan to pBuilder.
004183 **
004184 ** A constraint is marked usable if:
004185 **
004186 ** * Argument mUsable indicates that its prerequisites are available, and
004187 **
004188 ** * It is not one of the operators specified in the mExclude mask passed
004189 ** as the fourth argument (which in practice is either WO_IN or 0).
004190 **
004191 ** Argument mPrereq is a mask of tables that must be scanned before the
004192 ** virtual table in question. These are added to the plans prerequisites
004193 ** before it is added to pBuilder.
004194 **
004195 ** Output parameter *pbIn is set to true if the plan added to pBuilder
004196 ** uses one or more WO_IN terms, or false otherwise.
004197 */
004198 static int whereLoopAddVirtualOne(
004199 WhereLoopBuilder *pBuilder,
004200 Bitmask mPrereq, /* Mask of tables that must be used. */
004201 Bitmask mUsable, /* Mask of usable tables */
004202 u16 mExclude, /* Exclude terms using these operators */
004203 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
004204 u16 mNoOmit, /* Do not omit these constraints */
004205 int *pbIn, /* OUT: True if plan uses an IN(...) op */
004206 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */
004207 ){
004208 WhereClause *pWC = pBuilder->pWC;
004209 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004210 struct sqlite3_index_constraint *pIdxCons;
004211 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
004212 int i;
004213 int mxTerm;
004214 int rc = SQLITE_OK;
004215 WhereLoop *pNew = pBuilder->pNew;
004216 Parse *pParse = pBuilder->pWInfo->pParse;
004217 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
004218 int nConstraint = pIdxInfo->nConstraint;
004219
004220 assert( (mUsable & mPrereq)==mPrereq );
004221 *pbIn = 0;
004222 pNew->prereq = mPrereq;
004223
004224 /* Set the usable flag on the subset of constraints identified by
004225 ** arguments mUsable and mExclude. */
004226 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004227 for(i=0; i<nConstraint; i++, pIdxCons++){
004228 WhereTerm *pTerm = termFromWhereClause(pWC, pIdxCons->iTermOffset);
004229 pIdxCons->usable = 0;
004230 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
004231 && (pTerm->eOperator & mExclude)==0
004232 && (pbRetryLimit || !isLimitTerm(pTerm))
004233 ){
004234 pIdxCons->usable = 1;
004235 }
004236 }
004237
004238 /* Initialize the output fields of the sqlite3_index_info structure */
004239 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
004240 assert( pIdxInfo->needToFreeIdxStr==0 );
004241 pIdxInfo->idxStr = 0;
004242 pIdxInfo->idxNum = 0;
004243 pIdxInfo->orderByConsumed = 0;
004244 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
004245 pIdxInfo->estimatedRows = 25;
004246 pIdxInfo->idxFlags = 0;
004247 pHidden->mHandleIn = 0;
004248
004249 /* Invoke the virtual table xBestIndex() method */
004250 rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo);
004251 if( rc ){
004252 if( rc==SQLITE_CONSTRAINT ){
004253 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
004254 ** that the particular combination of parameters provided is unusable.
004255 ** Make no entries in the loop table.
004256 */
004257 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n"));
004258 freeIdxStr(pIdxInfo);
004259 return SQLITE_OK;
004260 }
004261 return rc;
004262 }
004263
004264 mxTerm = -1;
004265 assert( pNew->nLSlot>=nConstraint );
004266 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint );
004267 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab));
004268 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004269 for(i=0; i<nConstraint; i++, pIdxCons++){
004270 int iTerm;
004271 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
004272 WhereTerm *pTerm;
004273 int j = pIdxCons->iTermOffset;
004274 if( iTerm>=nConstraint
004275 || j<0
004276 || (pTerm = termFromWhereClause(pWC, j))==0
004277 || pNew->aLTerm[iTerm]!=0
004278 || pIdxCons->usable==0
004279 ){
004280 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName);
004281 freeIdxStr(pIdxInfo);
004282 return SQLITE_ERROR;
004283 }
004284 testcase( iTerm==nConstraint-1 );
004285 testcase( j==0 );
004286 testcase( j==pWC->nTerm-1 );
004287 pNew->prereq |= pTerm->prereqRight;
004288 assert( iTerm<pNew->nLSlot );
004289 pNew->aLTerm[iTerm] = pTerm;
004290 if( iTerm>mxTerm ) mxTerm = iTerm;
004291 testcase( iTerm==15 );
004292 testcase( iTerm==16 );
004293 if( pUsage[i].omit ){
004294 if( i<16 && ((1<<i)&mNoOmit)==0 ){
004295 testcase( i!=iTerm );
004296 pNew->u.vtab.omitMask |= 1<<iTerm;
004297 }else{
004298 testcase( i!=iTerm );
004299 }
004300 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){
004301 pNew->u.vtab.bOmitOffset = 1;
004302 }
004303 }
004304 if( SMASKBIT32(i) & pHidden->mHandleIn ){
004305 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
004306 }else if( (pTerm->eOperator & WO_IN)!=0 ){
004307 /* A virtual table that is constrained by an IN clause may not
004308 ** consume the ORDER BY clause because (1) the order of IN terms
004309 ** is not necessarily related to the order of output terms and
004310 ** (2) Multiple outputs from a single IN value will not merge
004311 ** together. */
004312 pIdxInfo->orderByConsumed = 0;
004313 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
004314 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
004315 }
004316
004317 /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET
004318 ** terms. And if there are any, they should follow all other terms. */
004319 assert( pbRetryLimit || !isLimitTerm(pTerm) );
004320 assert( !isLimitTerm(pTerm) || i>=nConstraint-2 );
004321 assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) );
004322
004323 if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){
004324 /* If there is an IN(...) term handled as an == (separate call to
004325 ** xFilter for each value on the RHS of the IN) and a LIMIT or
004326 ** OFFSET term handled as well, the plan is unusable. Similarly,
004327 ** if there is a LIMIT/OFFSET and there are other unused terms,
004328 ** the plan cannot be used. In these cases set variable *pbRetryLimit
004329 ** to true to tell the caller to retry with LIMIT and OFFSET
004330 ** disabled. */
004331 freeIdxStr(pIdxInfo);
004332 *pbRetryLimit = 1;
004333 return SQLITE_OK;
004334 }
004335 }
004336 }
004337
004338 pNew->nLTerm = mxTerm+1;
004339 for(i=0; i<=mxTerm; i++){
004340 if( pNew->aLTerm[i]==0 ){
004341 /* The non-zero argvIdx values must be contiguous. Raise an
004342 ** error if they are not */
004343 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName);
004344 freeIdxStr(pIdxInfo);
004345 return SQLITE_ERROR;
004346 }
004347 }
004348 assert( pNew->nLTerm<=pNew->nLSlot );
004349 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
004350 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
004351 pIdxInfo->needToFreeIdxStr = 0;
004352 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
004353 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
004354 pIdxInfo->nOrderBy : 0);
004355 pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0;
004356 pNew->rSetup = 0;
004357 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
004358 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
004359
004360 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
004361 ** that the scan will visit at most one row. Clear it otherwise. */
004362 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
004363 pNew->wsFlags |= WHERE_ONEROW;
004364 }else{
004365 pNew->wsFlags &= ~WHERE_ONEROW;
004366 }
004367 rc = whereLoopInsert(pBuilder, pNew);
004368 if( pNew->u.vtab.needFree ){
004369 sqlite3_free(pNew->u.vtab.idxStr);
004370 pNew->u.vtab.needFree = 0;
004371 }
004372 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
004373 *pbIn, (sqlite3_uint64)mPrereq,
004374 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
004375
004376 return rc;
004377 }
004378
004379 /*
004380 ** Return the collating sequence for a constraint passed into xBestIndex.
004381 **
004382 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
004383 ** This routine depends on there being a HiddenIndexInfo structure immediately
004384 ** following the sqlite3_index_info structure.
004385 **
004386 ** Return a pointer to the collation name:
004387 **
004388 ** 1. If there is an explicit COLLATE operator on the constraint, return it.
004389 **
004390 ** 2. Else, if the column has an alternative collation, return that.
004391 **
004392 ** 3. Otherwise, return "BINARY".
004393 */
004394 const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
004395 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004396 const char *zRet = 0;
004397 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
004398 CollSeq *pC = 0;
004399 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
004400 Expr *pX = termFromWhereClause(pHidden->pWC, iTerm)->pExpr;
004401 if( pX->pLeft ){
004402 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
004403 }
004404 zRet = (pC ? pC->zName : sqlite3StrBINARY);
004405 }
004406 return zRet;
004407 }
004408
004409 /*
004410 ** Return true if constraint iCons is really an IN(...) constraint, or
004411 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
004412 ** or clear (if bHandle==0) the flag to handle it using an iterator.
004413 */
004414 int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
004415 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004416 u32 m = SMASKBIT32(iCons);
004417 if( m & pHidden->mIn ){
004418 if( bHandle==0 ){
004419 pHidden->mHandleIn &= ~m;
004420 }else if( bHandle>0 ){
004421 pHidden->mHandleIn |= m;
004422 }
004423 return 1;
004424 }
004425 return 0;
004426 }
004427
004428 /*
004429 ** This interface is callable from within the xBestIndex callback only.
004430 **
004431 ** If possible, set (*ppVal) to point to an object containing the value
004432 ** on the right-hand-side of constraint iCons.
004433 */
004434 int sqlite3_vtab_rhs_value(
004435 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */
004436 int iCons, /* Constraint for which RHS is wanted */
004437 sqlite3_value **ppVal /* Write value extracted here */
004438 ){
004439 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1];
004440 sqlite3_value *pVal = 0;
004441 int rc = SQLITE_OK;
004442 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
004443 rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */
004444 }else{
004445 if( pH->aRhs[iCons]==0 ){
004446 WhereTerm *pTerm = termFromWhereClause(
004447 pH->pWC, pIdxInfo->aConstraint[iCons].iTermOffset
004448 );
004449 rc = sqlite3ValueFromExpr(
004450 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
004451 SQLITE_AFF_BLOB, &pH->aRhs[iCons]
004452 );
004453 testcase( rc!=SQLITE_OK );
004454 }
004455 pVal = pH->aRhs[iCons];
004456 }
004457 *ppVal = pVal;
004458
004459 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */
004460 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */
004461 }
004462
004463 return rc;
004464 }
004465
004466 /*
004467 ** Return true if ORDER BY clause may be handled as DISTINCT.
004468 */
004469 int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
004470 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004471 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 );
004472 return pHidden->eDistinct;
004473 }
004474
004475 /*
004476 ** Cause the prepared statement that is associated with a call to
004477 ** xBestIndex to potentially use all schemas. If the statement being
004478 ** prepared is read-only, then just start read transactions on all
004479 ** schemas. But if this is a write operation, start writes on all
004480 ** schemas.
004481 **
004482 ** This is used by the (built-in) sqlite_dbpage virtual table.
004483 */
004484 void sqlite3VtabUsesAllSchemas(Parse *pParse){
004485 int nDb = pParse->db->nDb;
004486 int i;
004487 for(i=0; i<nDb; i++){
004488 sqlite3CodeVerifySchema(pParse, i);
004489 }
004490 if( DbMaskNonZero(pParse->writeMask) ){
004491 for(i=0; i<nDb; i++){
004492 sqlite3BeginWriteOperation(pParse, 0, i);
004493 }
004494 }
004495 }
004496
004497 /*
004498 ** Add all WhereLoop objects for a table of the join identified by
004499 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
004500 **
004501 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
004502 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
004503 ** entries that occur before the virtual table in the FROM clause and are
004504 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
004505 ** mUnusable mask contains all FROM clause entries that occur after the
004506 ** virtual table and are separated from it by at least one LEFT or
004507 ** CROSS JOIN.
004508 **
004509 ** For example, if the query were:
004510 **
004511 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
004512 **
004513 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
004514 **
004515 ** All the tables in mPrereq must be scanned before the current virtual
004516 ** table. So any terms for which all prerequisites are satisfied by
004517 ** mPrereq may be specified as "usable" in all calls to xBestIndex.
004518 ** Conversely, all tables in mUnusable must be scanned after the current
004519 ** virtual table, so any terms for which the prerequisites overlap with
004520 ** mUnusable should always be configured as "not-usable" for xBestIndex.
004521 */
004522 static int whereLoopAddVirtual(
004523 WhereLoopBuilder *pBuilder, /* WHERE clause information */
004524 Bitmask mPrereq, /* Tables that must be scanned before this one */
004525 Bitmask mUnusable /* Tables that must be scanned after this one */
004526 ){
004527 int rc = SQLITE_OK; /* Return code */
004528 WhereInfo *pWInfo; /* WHERE analysis context */
004529 Parse *pParse; /* The parsing context */
004530 WhereClause *pWC; /* The WHERE clause */
004531 SrcItem *pSrc; /* The FROM clause term to search */
004532 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
004533 int nConstraint; /* Number of constraints in p */
004534 int bIn; /* True if plan uses IN(...) operator */
004535 WhereLoop *pNew;
004536 Bitmask mBest; /* Tables used by best possible plan */
004537 u16 mNoOmit;
004538 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */
004539
004540 assert( (mPrereq & mUnusable)==0 );
004541 pWInfo = pBuilder->pWInfo;
004542 pParse = pWInfo->pParse;
004543 pWC = pBuilder->pWC;
004544 pNew = pBuilder->pNew;
004545 pSrc = &pWInfo->pTabList->a[pNew->iTab];
004546 assert( IsVirtual(pSrc->pSTab) );
004547 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
004548 if( p==0 ) return SQLITE_NOMEM_BKPT;
004549 pNew->rSetup = 0;
004550 pNew->wsFlags = WHERE_VIRTUALTABLE;
004551 pNew->nLTerm = 0;
004552 pNew->u.vtab.needFree = 0;
004553 nConstraint = p->nConstraint;
004554 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
004555 freeIndexInfo(pParse->db, p);
004556 return SQLITE_NOMEM_BKPT;
004557 }
004558
004559 /* First call xBestIndex() with all constraints usable. */
004560 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName));
004561 WHERETRACE(0x800, (" VirtualOne: all usable\n"));
004562 rc = whereLoopAddVirtualOne(
004563 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
004564 );
004565 if( bRetry ){
004566 assert( rc==SQLITE_OK );
004567 rc = whereLoopAddVirtualOne(
004568 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0
004569 );
004570 }
004571
004572 /* If the call to xBestIndex() with all terms enabled produced a plan
004573 ** that does not require any source tables (IOW: a plan with mBest==0)
004574 ** and does not use an IN(...) operator, then there is no point in making
004575 ** any further calls to xBestIndex() since they will all return the same
004576 ** result (if the xBestIndex() implementation is sane). */
004577 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
004578 int seenZero = 0; /* True if a plan with no prereqs seen */
004579 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
004580 Bitmask mPrev = 0;
004581 Bitmask mBestNoIn = 0;
004582
004583 /* If the plan produced by the earlier call uses an IN(...) term, call
004584 ** xBestIndex again, this time with IN(...) terms disabled. */
004585 if( bIn ){
004586 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n"));
004587 rc = whereLoopAddVirtualOne(
004588 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0);
004589 assert( bIn==0 );
004590 mBestNoIn = pNew->prereq & ~mPrereq;
004591 if( mBestNoIn==0 ){
004592 seenZero = 1;
004593 seenZeroNoIN = 1;
004594 }
004595 }
004596
004597 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
004598 ** in the set of terms that apply to the current virtual table. */
004599 while( rc==SQLITE_OK ){
004600 int i;
004601 Bitmask mNext = ALLBITS;
004602 assert( mNext>0 );
004603 for(i=0; i<nConstraint; i++){
004604 int iTerm = p->aConstraint[i].iTermOffset;
004605 Bitmask mThis = termFromWhereClause(pWC, iTerm)->prereqRight & ~mPrereq;
004606 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
004607 }
004608 mPrev = mNext;
004609 if( mNext==ALLBITS ) break;
004610 if( mNext==mBest || mNext==mBestNoIn ) continue;
004611 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
004612 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
004613 rc = whereLoopAddVirtualOne(
004614 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0);
004615 if( pNew->prereq==mPrereq ){
004616 seenZero = 1;
004617 if( bIn==0 ) seenZeroNoIN = 1;
004618 }
004619 }
004620
004621 /* If the calls to xBestIndex() in the above loop did not find a plan
004622 ** that requires no source tables at all (i.e. one guaranteed to be
004623 ** usable), make a call here with all source tables disabled */
004624 if( rc==SQLITE_OK && seenZero==0 ){
004625 WHERETRACE(0x800, (" VirtualOne: all disabled\n"));
004626 rc = whereLoopAddVirtualOne(
004627 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0);
004628 if( bIn==0 ) seenZeroNoIN = 1;
004629 }
004630
004631 /* If the calls to xBestIndex() have so far failed to find a plan
004632 ** that requires no source tables at all and does not use an IN(...)
004633 ** operator, make a final call to obtain one here. */
004634 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
004635 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n"));
004636 rc = whereLoopAddVirtualOne(
004637 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0);
004638 }
004639 }
004640
004641 freeIndexInfo(pParse->db, p);
004642 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc));
004643 return rc;
004644 }
004645 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004646
004647 /*
004648 ** Add WhereLoop entries to handle OR terms. This works for either
004649 ** btrees or virtual tables.
004650 */
004651 static int whereLoopAddOr(
004652 WhereLoopBuilder *pBuilder,
004653 Bitmask mPrereq,
004654 Bitmask mUnusable
004655 ){
004656 WhereInfo *pWInfo = pBuilder->pWInfo;
004657 WhereClause *pWC;
004658 WhereLoop *pNew;
004659 WhereTerm *pTerm, *pWCEnd;
004660 int rc = SQLITE_OK;
004661 int iCur;
004662 WhereClause tempWC;
004663 WhereLoopBuilder sSubBuild;
004664 WhereOrSet sSum, sCur;
004665 SrcItem *pItem;
004666
004667 pWC = pBuilder->pWC;
004668 pWCEnd = pWC->a + pWC->nTerm;
004669 pNew = pBuilder->pNew;
004670 memset(&sSum, 0, sizeof(sSum));
004671 pItem = pWInfo->pTabList->a + pNew->iTab;
004672 iCur = pItem->iCursor;
004673
004674 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
004675 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK;
004676
004677 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
004678 if( (pTerm->eOperator & WO_OR)!=0
004679 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
004680 ){
004681 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
004682 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
004683 WhereTerm *pOrTerm;
004684 int once = 1;
004685 int i, j;
004686
004687 sSubBuild = *pBuilder;
004688 sSubBuild.pOrSet = &sCur;
004689
004690 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm));
004691 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
004692 if( (pOrTerm->eOperator & WO_AND)!=0 ){
004693 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
004694 }else if( pOrTerm->leftCursor==iCur ){
004695 tempWC.pWInfo = pWC->pWInfo;
004696 tempWC.pOuter = pWC;
004697 tempWC.op = TK_AND;
004698 tempWC.nTerm = 1;
004699 tempWC.nBase = 1;
004700 tempWC.a = pOrTerm;
004701 sSubBuild.pWC = &tempWC;
004702 }else{
004703 continue;
004704 }
004705 sCur.n = 0;
004706 #ifdef WHERETRACE_ENABLED
004707 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
004708 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
004709 if( sqlite3WhereTrace & 0x20000 ){
004710 sqlite3WhereClausePrint(sSubBuild.pWC);
004711 }
004712 #endif
004713 #ifndef SQLITE_OMIT_VIRTUALTABLE
004714 if( IsVirtual(pItem->pSTab) ){
004715 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
004716 }else
004717 #endif
004718 {
004719 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
004720 }
004721 if( rc==SQLITE_OK ){
004722 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
004723 }
004724 testcase( rc==SQLITE_NOMEM && sCur.n>0 );
004725 testcase( rc==SQLITE_DONE );
004726 if( sCur.n==0 ){
004727 sSum.n = 0;
004728 break;
004729 }else if( once ){
004730 whereOrMove(&sSum, &sCur);
004731 once = 0;
004732 }else{
004733 WhereOrSet sPrev;
004734 whereOrMove(&sPrev, &sSum);
004735 sSum.n = 0;
004736 for(i=0; i<sPrev.n; i++){
004737 for(j=0; j<sCur.n; j++){
004738 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
004739 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
004740 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
004741 }
004742 }
004743 }
004744 }
004745 pNew->nLTerm = 1;
004746 pNew->aLTerm[0] = pTerm;
004747 pNew->wsFlags = WHERE_MULTI_OR;
004748 pNew->rSetup = 0;
004749 pNew->iSortIdx = 0;
004750 memset(&pNew->u, 0, sizeof(pNew->u));
004751 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
004752 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
004753 ** of all sub-scans required by the OR-scan. However, due to rounding
004754 ** errors, it may be that the cost of the OR-scan is equal to its
004755 ** most expensive sub-scan. Add the smallest possible penalty
004756 ** (equivalent to multiplying the cost by 1.07) to ensure that
004757 ** this does not happen. Otherwise, for WHERE clauses such as the
004758 ** following where there is an index on "y":
004759 **
004760 ** WHERE likelihood(x=?, 0.99) OR y=?
004761 **
004762 ** the planner may elect to "OR" together a full-table scan and an
004763 ** index lookup. And other similarly odd results. */
004764 pNew->rRun = sSum.a[i].rRun + 1;
004765 pNew->nOut = sSum.a[i].nOut;
004766 pNew->prereq = sSum.a[i].prereq;
004767 rc = whereLoopInsert(pBuilder, pNew);
004768 }
004769 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm));
004770 }
004771 }
004772 return rc;
004773 }
004774
004775 /*
004776 ** Add all WhereLoop objects for all tables
004777 */
004778 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
004779 WhereInfo *pWInfo = pBuilder->pWInfo;
004780 Bitmask mPrereq = 0;
004781 Bitmask mPrior = 0;
004782 int iTab;
004783 SrcList *pTabList = pWInfo->pTabList;
004784 SrcItem *pItem;
004785 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel];
004786 sqlite3 *db = pWInfo->pParse->db;
004787 int rc = SQLITE_OK;
004788 int bFirstPastRJ = 0;
004789 int hasRightJoin = 0;
004790 WhereLoop *pNew;
004791
004792
004793 /* Loop over the tables in the join, from left to right */
004794 pNew = pBuilder->pNew;
004795
004796 /* Verify that pNew has already been initialized */
004797 assert( pNew->nLTerm==0 );
004798 assert( pNew->wsFlags==0 );
004799 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
004800 assert( pNew->aLTerm!=0 );
004801
004802 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
004803 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
004804 Bitmask mUnusable = 0;
004805 pNew->iTab = iTab;
004806 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
004807 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
004808 if( bFirstPastRJ
004809 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
004810 ){
004811 /* Add prerequisites to prevent reordering of FROM clause terms
004812 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
004813 ** prevents the right operand of a RIGHT JOIN from being swapped with
004814 ** other elements even further to the right.
004815 **
004816 ** The JT_LTORJ case and the hasRightJoin flag work together to
004817 ** prevent FROM-clause terms from moving from the right side of
004818 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
004819 ** is itself on the left side of a RIGHT JOIN.
004820 */
004821 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1;
004822 mPrereq |= mPrior;
004823 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
004824 }else if( !hasRightJoin ){
004825 mPrereq = 0;
004826 }
004827 #ifndef SQLITE_OMIT_VIRTUALTABLE
004828 if( IsVirtual(pItem->pSTab) ){
004829 SrcItem *p;
004830 for(p=&pItem[1]; p<pEnd; p++){
004831 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
004832 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
004833 }
004834 }
004835 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
004836 }else
004837 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004838 {
004839 rc = whereLoopAddBtree(pBuilder, mPrereq);
004840 }
004841 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){
004842 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
004843 }
004844 mPrior |= pNew->maskSelf;
004845 if( rc || db->mallocFailed ){
004846 if( rc==SQLITE_DONE ){
004847 /* We hit the query planner search limit set by iPlanLimit */
004848 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search");
004849 rc = SQLITE_OK;
004850 }else{
004851 break;
004852 }
004853 }
004854 }
004855
004856 whereLoopClear(db, pNew);
004857 return rc;
004858 }
004859
004860 /* Implementation of the order-by-subquery optimization:
004861 **
004862 ** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really
004863 ** a subquery or CTE that has an ORDER BY clause. See if any of the terms
004864 ** in the subquery ORDER BY clause will satisfy pOrderBy from the outer
004865 ** query. Mark off all satisfied terms (by setting bits in *pOBSat) and
004866 ** return TRUE if they do. If not, return false.
004867 **
004868 ** Example:
004869 **
004870 ** CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b));
004871 ** CREATE TABLE t2(x,y);
004872 ** WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y)
004873 ** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b;
004874 **
004875 ** The CTE named "t3" comes out in the natural order of "p", so the first
004876 ** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3"
004877 ** and sorting only needs to occur on the second term "b".
004878 **
004879 ** Limitations:
004880 **
004881 ** (1) The optimization is not applied if the outer ORDER BY contains
004882 ** a COLLATE clause. The optimization might be applied if the
004883 ** outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as
004884 ** long as the subquery ORDER BY does the same. But if the
004885 ** outer ORDER BY uses COLLATE, even a redundant COLLATE, the
004886 ** optimization is bypassed.
004887 **
004888 ** (2) The subquery ORDER BY terms must exactly match subquery result
004889 ** columns, including any COLLATE annotations. This routine relies
004890 ** on iOrderByCol to do matching between order by terms and result
004891 ** columns, and iOrderByCol will not be set if the result column
004892 ** and ORDER BY collations differ.
004893 **
004894 ** (3) The subquery and outer ORDER BY can be in opposite directions as
004895 ** long as the subquery is materialized. If the subquery is
004896 ** implemented as a co-routine, the sort orders must be in the same
004897 ** direction because there is no way to run a co-routine backwards.
004898 */
004899 static SQLITE_NOINLINE int wherePathMatchSubqueryOB(
004900 WhereInfo *pWInfo, /* The WHERE clause */
004901 WhereLoop *pLoop, /* The nested loop term that is a subquery */
004902 int iLoop, /* Which level of the nested loop. 0==outermost */
004903 int iCur, /* Cursor used by the this loop */
004904 ExprList *pOrderBy, /* The ORDER BY clause on the whole query */
004905 Bitmask *pRevMask, /* When loops need to go in reverse order */
004906 Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */
004907 ){
004908 int iOB; /* Index into pOrderBy->a[] */
004909 int jSub; /* Index into pSubOB->a[] */
004910 u8 rev = 0; /* True if iOB and jSub sort in opposite directions */
004911 u8 revIdx = 0; /* Sort direction for jSub */
004912 Expr *pOBExpr; /* Current term of outer ORDER BY */
004913 ExprList *pSubOB; /* Complete ORDER BY on the subquery */
004914
004915 pSubOB = pLoop->u.btree.pOrderBy;
004916 assert( pSubOB!=0 );
004917 for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){}
004918 for(jSub=0; jSub<pSubOB->nExpr && iOB<pOrderBy->nExpr; jSub++, iOB++){
004919 if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break;
004920 pOBExpr = pOrderBy->a[iOB].pExpr;
004921 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break;
004922 if( pOBExpr->iTable!=iCur ) break;
004923 if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break;
004924 if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){
004925 u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */
004926 u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */
004927 if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){
004928 break;
004929 }
004930 revIdx = sfSub & KEYINFO_ORDER_DESC;
004931 if( jSub>0 ){
004932 if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){
004933 break;
004934 }
004935 }else{
004936 rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC);
004937 if( rev ){
004938 if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){
004939 /* Cannot run a co-routine in reverse order */
004940 break;
004941 }
004942 *pRevMask |= MASKBIT(iLoop);
004943 }
004944 }
004945 }
004946 *pOBSat |= MASKBIT(iOB);
004947 }
004948 return jSub>0;
004949 }
004950
004951 /*
004952 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
004953 ** parameters) to see if it outputs rows in the requested ORDER BY
004954 ** (or GROUP BY) without requiring a separate sort operation. Return N:
004955 **
004956 ** N>0: N terms of the ORDER BY clause are satisfied
004957 ** N==0: No terms of the ORDER BY clause are satisfied
004958 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
004959 **
004960 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
004961 ** strict. With GROUP BY and DISTINCT the only requirement is that
004962 ** equivalent rows appear immediately adjacent to one another. GROUP BY
004963 ** and DISTINCT do not require rows to appear in any particular order as long
004964 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
004965 ** the pOrderBy terms can be matched in any order. With ORDER BY, the
004966 ** pOrderBy terms must be matched in strict left-to-right order.
004967 */
004968 static i8 wherePathSatisfiesOrderBy(
004969 WhereInfo *pWInfo, /* The WHERE clause */
004970 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
004971 WherePath *pPath, /* The WherePath to check */
004972 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
004973 u16 nLoop, /* Number of entries in pPath->aLoop[] */
004974 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
004975 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
004976 ){
004977 u8 revSet; /* True if rev is known */
004978 u8 rev; /* Composite sort order */
004979 u8 revIdx; /* Index sort order */
004980 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
004981 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
004982 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
004983 u16 eqOpMask; /* Allowed equality operators */
004984 u16 nKeyCol; /* Number of key columns in pIndex */
004985 u16 nColumn; /* Total number of ordered columns in the index */
004986 u16 nOrderBy; /* Number terms in the ORDER BY clause */
004987 int iLoop; /* Index of WhereLoop in pPath being processed */
004988 int i, j; /* Loop counters */
004989 int iCur; /* Cursor number for current WhereLoop */
004990 int iColumn; /* A column number within table iCur */
004991 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
004992 WhereTerm *pTerm; /* A single term of the WHERE clause */
004993 Expr *pOBExpr; /* An expression from the ORDER BY clause */
004994 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
004995 Index *pIndex; /* The index associated with pLoop */
004996 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
004997 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
004998 Bitmask obDone; /* Mask of all ORDER BY terms */
004999 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
005000 Bitmask ready; /* Mask of inner loops */
005001
005002 /*
005003 ** We say the WhereLoop is "one-row" if it generates no more than one
005004 ** row of output. A WhereLoop is one-row if all of the following are true:
005005 ** (a) All index columns match with WHERE_COLUMN_EQ.
005006 ** (b) The index is unique
005007 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
005008 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
005009 **
005010 ** We say the WhereLoop is "order-distinct" if the set of columns from
005011 ** that WhereLoop that are in the ORDER BY clause are different for every
005012 ** row of the WhereLoop. Every one-row WhereLoop is automatically
005013 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
005014 ** is not order-distinct. To be order-distinct is not quite the same as being
005015 ** UNIQUE since a UNIQUE column or index can have multiple rows that
005016 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
005017 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
005018 **
005019 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
005020 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
005021 ** automatically order-distinct.
005022 */
005023
005024 assert( pOrderBy!=0 );
005025 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
005026
005027 nOrderBy = pOrderBy->nExpr;
005028 testcase( nOrderBy==BMS-1 );
005029 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
005030 isOrderDistinct = 1;
005031 obDone = MASKBIT(nOrderBy)-1;
005032 orderDistinctMask = 0;
005033 ready = 0;
005034 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
005035 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){
005036 eqOpMask |= WO_IN;
005037 }
005038 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
005039 if( iLoop>0 ) ready |= pLoop->maskSelf;
005040 if( iLoop<nLoop ){
005041 pLoop = pPath->aLoop[iLoop];
005042 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
005043 }else{
005044 pLoop = pLast;
005045 }
005046 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
005047 if( pLoop->u.vtab.isOrdered
005048 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
005049 ){
005050 obSat = obDone;
005051 }
005052 break;
005053 }else if( wctrlFlags & WHERE_DISTINCTBY ){
005054 pLoop->u.btree.nDistinctCol = 0;
005055 }
005056 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
005057
005058 /* Mark off any ORDER BY term X that is a column in the table of
005059 ** the current loop for which there is term in the WHERE
005060 ** clause of the form X IS NULL or X=? that reference only outer
005061 ** loops.
005062 */
005063 for(i=0; i<nOrderBy; i++){
005064 if( MASKBIT(i) & obSat ) continue;
005065 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
005066 if( NEVER(pOBExpr==0) ) continue;
005067 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
005068 if( pOBExpr->iTable!=iCur ) continue;
005069 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
005070 ~ready, eqOpMask, 0);
005071 if( pTerm==0 ) continue;
005072 if( pTerm->eOperator==WO_IN ){
005073 /* IN terms are only valid for sorting in the ORDER BY LIMIT
005074 ** optimization, and then only if they are actually used
005075 ** by the query plan */
005076 assert( wctrlFlags &
005077 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
005078 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
005079 if( j>=pLoop->nLTerm ) continue;
005080 }
005081 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
005082 Parse *pParse = pWInfo->pParse;
005083 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr);
005084 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr);
005085 assert( pColl1 );
005086 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){
005087 continue;
005088 }
005089 testcase( pTerm->pExpr->op==TK_IS );
005090 }
005091 obSat |= MASKBIT(i);
005092 }
005093
005094 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
005095 if( pLoop->wsFlags & WHERE_IPK ){
005096 if( pLoop->u.btree.pOrderBy
005097 && OptimizationEnabled(db, SQLITE_OrderBySubq)
005098 && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur,
005099 pOrderBy,pRevMask, &obSat)
005100 ){
005101 nColumn = 0;
005102 isOrderDistinct = 0;
005103 }else{
005104 nColumn = 1;
005105 }
005106 pIndex = 0;
005107 nKeyCol = 0;
005108 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
005109 return 0;
005110 }else{
005111 nKeyCol = pIndex->nKeyCol;
005112 nColumn = pIndex->nColumn;
005113 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
005114 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
005115 || !HasRowid(pIndex->pTable));
005116 /* All relevant terms of the index must also be non-NULL in order
005117 ** for isOrderDistinct to be true. So the isOrderDistinct value
005118 ** computed here might be a false positive. Corrections will be
005119 ** made at tag-20210426-1 below */
005120 isOrderDistinct = IsUniqueIndex(pIndex)
005121 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0;
005122 }
005123
005124 /* Loop through all columns of the index and deal with the ones
005125 ** that are not constrained by == or IN.
005126 */
005127 rev = revSet = 0;
005128 distinctColumns = 0;
005129 for(j=0; j<nColumn; j++){
005130 u8 bOnce = 1; /* True to run the ORDER BY search loop */
005131
005132 assert( j>=pLoop->u.btree.nEq
005133 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
005134 );
005135 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
005136 u16 eOp = pLoop->aLTerm[j]->eOperator;
005137
005138 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
005139 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
005140 ** terms imply that the index is not UNIQUE NOT NULL in which case
005141 ** the loop need to be marked as not order-distinct because it can
005142 ** have repeated NULL rows.
005143 **
005144 ** If the current term is a column of an ((?,?) IN (SELECT...))
005145 ** expression for which the SELECT returns more than one column,
005146 ** check that it is the only column used by this loop. Otherwise,
005147 ** if it is one of two or more, none of the columns can be
005148 ** considered to match an ORDER BY term.
005149 */
005150 if( (eOp & eqOpMask)!=0 ){
005151 if( eOp & (WO_ISNULL|WO_IS) ){
005152 testcase( eOp & WO_ISNULL );
005153 testcase( eOp & WO_IS );
005154 testcase( isOrderDistinct );
005155 isOrderDistinct = 0;
005156 }
005157 continue;
005158 }else if( ALWAYS(eOp & WO_IN) ){
005159 /* ALWAYS() justification: eOp is an equality operator due to the
005160 ** j<pLoop->u.btree.nEq constraint above. Any equality other
005161 ** than WO_IN is captured by the previous "if". So this one
005162 ** always has to be WO_IN. */
005163 Expr *pX = pLoop->aLTerm[j]->pExpr;
005164 for(i=j+1; i<pLoop->u.btree.nEq; i++){
005165 if( pLoop->aLTerm[i]->pExpr==pX ){
005166 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
005167 bOnce = 0;
005168 break;
005169 }
005170 }
005171 }
005172 }
005173
005174 /* Get the column number in the table (iColumn) and sort order
005175 ** (revIdx) for the j-th column of the index.
005176 */
005177 if( pIndex ){
005178 iColumn = pIndex->aiColumn[j];
005179 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC;
005180 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
005181 }else{
005182 iColumn = XN_ROWID;
005183 revIdx = 0;
005184 }
005185
005186 /* An unconstrained column that might be NULL means that this
005187 ** WhereLoop is not well-ordered. tag-20210426-1
005188 */
005189 if( isOrderDistinct ){
005190 if( iColumn>=0
005191 && j>=pLoop->u.btree.nEq
005192 && pIndex->pTable->aCol[iColumn].notNull==0
005193 ){
005194 isOrderDistinct = 0;
005195 }
005196 if( iColumn==XN_EXPR ){
005197 isOrderDistinct = 0;
005198 }
005199 }
005200
005201 /* Find the ORDER BY term that corresponds to the j-th column
005202 ** of the index and mark that ORDER BY term having been satisfied.
005203 */
005204 isMatch = 0;
005205 for(i=0; bOnce && i<nOrderBy; i++){
005206 if( MASKBIT(i) & obSat ) continue;
005207 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
005208 testcase( wctrlFlags & WHERE_GROUPBY );
005209 testcase( wctrlFlags & WHERE_DISTINCTBY );
005210 if( NEVER(pOBExpr==0) ) continue;
005211 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
005212 if( iColumn>=XN_ROWID ){
005213 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
005214 if( pOBExpr->iTable!=iCur ) continue;
005215 if( pOBExpr->iColumn!=iColumn ) continue;
005216 }else{
005217 Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr;
005218 if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){
005219 continue;
005220 }
005221 }
005222 if( iColumn!=XN_ROWID ){
005223 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
005224 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
005225 }
005226 if( wctrlFlags & WHERE_DISTINCTBY ){
005227 pLoop->u.btree.nDistinctCol = j+1;
005228 }
005229 isMatch = 1;
005230 break;
005231 }
005232 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
005233 /* Make sure the sort order is compatible in an ORDER BY clause.
005234 ** Sort order is irrelevant for a GROUP BY clause. */
005235 if( revSet ){
005236 if( (rev ^ revIdx)
005237 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
005238 ){
005239 isMatch = 0;
005240 }
005241 }else{
005242 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC);
005243 if( rev ) *pRevMask |= MASKBIT(iLoop);
005244 revSet = 1;
005245 }
005246 }
005247 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){
005248 if( j==pLoop->u.btree.nEq ){
005249 pLoop->wsFlags |= WHERE_BIGNULL_SORT;
005250 }else{
005251 isMatch = 0;
005252 }
005253 }
005254 if( isMatch ){
005255 if( iColumn==XN_ROWID ){
005256 testcase( distinctColumns==0 );
005257 distinctColumns = 1;
005258 }
005259 obSat |= MASKBIT(i);
005260 }else{
005261 /* No match found */
005262 if( j==0 || j<nKeyCol ){
005263 testcase( isOrderDistinct!=0 );
005264 isOrderDistinct = 0;
005265 }
005266 break;
005267 }
005268 } /* end Loop over all index columns */
005269 if( distinctColumns ){
005270 testcase( isOrderDistinct==0 );
005271 isOrderDistinct = 1;
005272 }
005273 } /* end-if not one-row */
005274
005275 /* Mark off any other ORDER BY terms that reference pLoop */
005276 if( isOrderDistinct ){
005277 orderDistinctMask |= pLoop->maskSelf;
005278 for(i=0; i<nOrderBy; i++){
005279 Expr *p;
005280 Bitmask mTerm;
005281 if( MASKBIT(i) & obSat ) continue;
005282 p = pOrderBy->a[i].pExpr;
005283 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
005284 if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue;
005285 if( (mTerm&~orderDistinctMask)==0 ){
005286 obSat |= MASKBIT(i);
005287 }
005288 }
005289 }
005290 } /* End the loop over all WhereLoops from outer-most down to inner-most */
005291 if( obSat==obDone ) return (i8)nOrderBy;
005292 if( !isOrderDistinct ){
005293 for(i=nOrderBy-1; i>0; i--){
005294 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0;
005295 if( (obSat&m)==m ) return i;
005296 }
005297 return 0;
005298 }
005299 return -1;
005300 }
005301
005302
005303 /*
005304 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
005305 ** the planner assumes that the specified pOrderBy list is actually a GROUP
005306 ** BY clause - and so any order that groups rows as required satisfies the
005307 ** request.
005308 **
005309 ** Normally, in this case it is not possible for the caller to determine
005310 ** whether or not the rows are really being delivered in sorted order, or
005311 ** just in some other order that provides the required grouping. However,
005312 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
005313 ** this function may be called on the returned WhereInfo object. It returns
005314 ** true if the rows really will be sorted in the specified order, or false
005315 ** otherwise.
005316 **
005317 ** For example, assuming:
005318 **
005319 ** CREATE INDEX i1 ON t1(x, Y);
005320 **
005321 ** then
005322 **
005323 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
005324 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
005325 */
005326 int sqlite3WhereIsSorted(WhereInfo *pWInfo){
005327 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
005328 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
005329 return pWInfo->sorted;
005330 }
005331
005332 #ifdef WHERETRACE_ENABLED
005333 /* For debugging use only: */
005334 static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
005335 static char zName[65];
005336 int i;
005337 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
005338 if( pLast ) zName[i++] = pLast->cId;
005339 zName[i] = 0;
005340 return zName;
005341 }
005342 #endif
005343
005344 /*
005345 ** Return the cost of sorting nRow rows, assuming that the keys have
005346 ** nOrderby columns and that the first nSorted columns are already in
005347 ** order.
005348 */
005349 static LogEst whereSortingCost(
005350 WhereInfo *pWInfo, /* Query planning context */
005351 LogEst nRow, /* Estimated number of rows to sort */
005352 int nOrderBy, /* Number of ORDER BY clause terms */
005353 int nSorted /* Number of initial ORDER BY terms naturally in order */
005354 ){
005355 /* Estimated cost of a full external sort, where N is
005356 ** the number of rows to sort is:
005357 **
005358 ** cost = (K * N * log(N)).
005359 **
005360 ** Or, if the order-by clause has X terms but only the last Y
005361 ** terms are out of order, then block-sorting will reduce the
005362 ** sorting cost to:
005363 **
005364 ** cost = (K * N * log(N)) * (Y/X)
005365 **
005366 ** The constant K is at least 2.0 but will be larger if there are a
005367 ** large number of columns to be sorted, as the sorting time is
005368 ** proportional to the amount of content to be sorted. The algorithm
005369 ** does not currently distinguish between fat columns (BLOBs and TEXTs)
005370 ** and skinny columns (INTs). It just uses the number of columns as
005371 ** an approximation for the row width.
005372 **
005373 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
005374 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert.
005375 */
005376 LogEst rSortCost, nCol;
005377 assert( pWInfo->pSelect!=0 );
005378 assert( pWInfo->pSelect->pEList!=0 );
005379 /* TUNING: sorting cost proportional to the number of output columns: */
005380 nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30);
005381 rSortCost = nRow + nCol;
005382 if( nSorted>0 ){
005383 /* Scale the result by (Y/X) */
005384 rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
005385 }
005386
005387 /* Multiple by log(M) where M is the number of output rows.
005388 ** Use the LIMIT for M if it is smaller. Or if this sort is for
005389 ** a DISTINCT operator, M will be the number of distinct output
005390 ** rows, so fudge it downwards a bit.
005391 */
005392 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){
005393 rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */
005394 if( nSorted!=0 ){
005395 rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */
005396 }
005397 if( pWInfo->iLimit<nRow ){
005398 nRow = pWInfo->iLimit;
005399 }
005400 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
005401 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
005402 ** reduces the number of output rows by a factor of 2 */
005403 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); }
005404 }
005405 rSortCost += estLog(nRow);
005406 return rSortCost;
005407 }
005408
005409 /*
005410 ** Compute the maximum number of paths in the solver algorithm, for
005411 ** queries that have three or more terms in the FROM clause. Queries with
005412 ** two or fewer FROM clause terms are handled by the caller.
005413 **
005414 ** Query planning is NP-hard. We must limit the number of paths at
005415 ** each step of the solver search algorithm to avoid exponential behavior.
005416 **
005417 ** The value returned is a tuning parameter. Currently the value is:
005418 **
005419 ** 18 for star queries
005420 ** 12 otherwise
005421 **
005422 ** For the purposes of SQLite, a star-query is defined as a query
005423 ** with a large central table that is joined against four or more
005424 ** smaller tables. The central table is called the "fact" table.
005425 ** The smaller tables that get joined are "dimension tables".
005426 **
005427 ** SIDE EFFECT: (and really the whole point of this subroutine)
005428 **
005429 ** If pWInfo describes a star-query, then the cost on WhereLoops for the
005430 ** fact table is reduced. This heuristic helps keep fact tables in
005431 ** outer loops. Without this heuristic, paths with fact tables in outer
005432 ** loops tend to get pruned by the mxChoice limit on the number of paths,
005433 ** resulting in poor query plans. The total amount of heuristic cost
005434 ** adjustment is stored in pWInfo->nOutStarDelta and the cost adjustment
005435 ** for each WhereLoop is stored in its rStarDelta field.
005436 */
005437 static int computeMxChoice(WhereInfo *pWInfo, LogEst nRowEst){
005438 int nLoop = pWInfo->nLevel; /* Number of terms in the join */
005439 if( nRowEst==0 && nLoop>=5 ){
005440 /* Check to see if we are dealing with a star schema and if so, reduce
005441 ** the cost of fact tables relative to dimension tables, as a heuristic
005442 ** to help keep the fact tables in outer loops.
005443 */
005444 int iLoop; /* Counter over join terms */
005445 Bitmask m; /* Bitmask for current loop */
005446 assert( pWInfo->nOutStarDelta==0 );
005447 for(iLoop=0, m=1; iLoop<nLoop; iLoop++, m<<=1){
005448 WhereLoop *pWLoop; /* For looping over WhereLoops */
005449 int nDep = 0; /* Number of dimension tables */
005450 LogEst rDelta; /* Heuristic cost adjustment */
005451 Bitmask mSeen = 0; /* Mask of dimension tables */
005452 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005453 if( (pWLoop->prereq & m)!=0 && (pWLoop->maskSelf & mSeen)==0 ){
005454 nDep++;
005455 mSeen |= pWLoop->maskSelf;
005456 }
005457 }
005458 if( nDep<=3 ) continue;
005459 rDelta = 15*(nDep-3);
005460 #ifdef WHERETRACE_ENABLED /* 0x4 */
005461 if( sqlite3WhereTrace&0x4 ){
005462 SrcItem *pItem = pWInfo->pTabList->a + iLoop;
005463 sqlite3DebugPrintf("Fact-table %s: %d dimensions, cost reduced %d\n",
005464 pItem->zAlias ? pItem->zAlias : pItem->pSTab->zName,
005465 nDep, rDelta);
005466 }
005467 #endif
005468 if( pWInfo->nOutStarDelta==0 ){
005469 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005470 pWLoop->rStarDelta = 0;
005471 }
005472 }
005473 pWInfo->nOutStarDelta += rDelta;
005474 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005475 if( pWLoop->maskSelf==m ){
005476 pWLoop->rRun -= rDelta;
005477 pWLoop->nOut -= rDelta;
005478 pWLoop->rStarDelta = rDelta;
005479 }
005480 }
005481 }
005482 }
005483 return pWInfo->nOutStarDelta>0 ? 18 : 12;
005484 }
005485
005486 /*
005487 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
005488 ** attempts to find the lowest cost path that visits each WhereLoop
005489 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
005490 **
005491 ** Assume that the total number of output rows that will need to be sorted
005492 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
005493 ** costs if nRowEst==0.
005494 **
005495 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
005496 ** error occurs.
005497 */
005498 static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
005499 int mxChoice; /* Maximum number of simultaneous paths tracked */
005500 int nLoop; /* Number of terms in the join */
005501 Parse *pParse; /* Parsing context */
005502 int iLoop; /* Loop counter over the terms of the join */
005503 int ii, jj; /* Loop counters */
005504 int mxI = 0; /* Index of next entry to replace */
005505 int nOrderBy; /* Number of ORDER BY clause terms */
005506 LogEst mxCost = 0; /* Maximum cost of a set of paths */
005507 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
005508 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
005509 WherePath *aFrom; /* All nFrom paths at the previous level */
005510 WherePath *aTo; /* The nTo best paths at the current level */
005511 WherePath *pFrom; /* An element of aFrom[] that we are working on */
005512 WherePath *pTo; /* An element of aTo[] that we are working on */
005513 WhereLoop *pWLoop; /* One of the WhereLoop objects */
005514 WhereLoop **pX; /* Used to divy up the pSpace memory */
005515 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
005516 char *pSpace; /* Temporary memory used by this routine */
005517 int nSpace; /* Bytes of space allocated at pSpace */
005518
005519 pParse = pWInfo->pParse;
005520 nLoop = pWInfo->nLevel;
005521 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n",
005522 nRowEst, pParse->nQueryLoop));
005523 /* TUNING: mxChoice is the maximum number of possible paths to preserve
005524 ** at each step. Based on the number of loops in the FROM clause:
005525 **
005526 ** nLoop mxChoice
005527 ** ----- --------
005528 ** 1 1 // the most common case
005529 ** 2 5
005530 ** 3+ 12 or 18 // see computeMxChoice()
005531 */
005532 if( nLoop<=1 ){
005533 mxChoice = 1;
005534 }else if( nLoop==2 ){
005535 mxChoice = 5;
005536 }else{
005537 mxChoice = computeMxChoice(pWInfo, nRowEst);
005538 }
005539 assert( nLoop<=pWInfo->pTabList->nSrc );
005540
005541 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
005542 ** case the purpose of this call is to estimate the number of rows returned
005543 ** by the overall query. Once this estimate has been obtained, the caller
005544 ** will invoke this function a second time, passing the estimate as the
005545 ** nRowEst parameter. */
005546 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
005547 nOrderBy = 0;
005548 }else{
005549 nOrderBy = pWInfo->pOrderBy->nExpr;
005550 }
005551
005552 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
005553 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
005554 nSpace += sizeof(LogEst) * nOrderBy;
005555 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
005556 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
005557 aTo = (WherePath*)pSpace;
005558 aFrom = aTo+mxChoice;
005559 memset(aFrom, 0, sizeof(aFrom[0]));
005560 pX = (WhereLoop**)(aFrom+mxChoice);
005561 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
005562 pFrom->aLoop = pX;
005563 }
005564 if( nOrderBy ){
005565 /* If there is an ORDER BY clause and it is not being ignored, set up
005566 ** space for the aSortCost[] array. Each element of the aSortCost array
005567 ** is either zero - meaning it has not yet been initialized - or the
005568 ** cost of sorting nRowEst rows of data where the first X terms of
005569 ** the ORDER BY clause are already in order, where X is the array
005570 ** index. */
005571 aSortCost = (LogEst*)pX;
005572 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
005573 }
005574 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
005575 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
005576
005577 /* Seed the search with a single WherePath containing zero WhereLoops.
005578 **
005579 ** TUNING: Do not let the number of iterations go above 28. If the cost
005580 ** of computing an automatic index is not paid back within the first 28
005581 ** rows, then do not use the automatic index. */
005582 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
005583 nFrom = 1;
005584 assert( aFrom[0].isOrdered==0 );
005585 if( nOrderBy ){
005586 /* If nLoop is zero, then there are no FROM terms in the query. Since
005587 ** in this case the query may return a maximum of one row, the results
005588 ** are already in the requested order. Set isOrdered to nOrderBy to
005589 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
005590 ** -1, indicating that the result set may or may not be ordered,
005591 ** depending on the loops added to the current plan. */
005592 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
005593 }
005594
005595 /* Compute successively longer WherePaths using the previous generation
005596 ** of WherePaths as the basis for the next. Keep track of the mxChoice
005597 ** best paths at each generation */
005598 for(iLoop=0; iLoop<nLoop; iLoop++){
005599 nTo = 0;
005600 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
005601 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005602 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
005603 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
005604 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
005605 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
005606 Bitmask maskNew; /* Mask of src visited by (..) */
005607 Bitmask revMask; /* Mask of rev-order loops for (..) */
005608
005609 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
005610 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
005611 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){
005612 /* Do not use an automatic index if the this loop is expected
005613 ** to run less than 1.25 times. It is tempting to also exclude
005614 ** automatic index usage on an outer loop, but sometimes an automatic
005615 ** index is useful in the outer loop of a correlated subquery. */
005616 assert( 10==sqlite3LogEst(2) );
005617 continue;
005618 }
005619
005620 /* At this point, pWLoop is a candidate to be the next loop.
005621 ** Compute its cost */
005622 rUnsorted = pWLoop->rRun + pFrom->nRow;
005623 if( pWLoop->rSetup ){
005624 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup, rUnsorted);
005625 }
005626 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
005627 nOut = pFrom->nRow + pWLoop->nOut;
005628 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
005629 isOrdered = pFrom->isOrdered;
005630 if( isOrdered<0 ){
005631 revMask = 0;
005632 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
005633 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
005634 iLoop, pWLoop, &revMask);
005635 }else{
005636 revMask = pFrom->revLoop;
005637 }
005638 if( isOrdered>=0 && isOrdered<nOrderBy ){
005639 if( aSortCost[isOrdered]==0 ){
005640 aSortCost[isOrdered] = whereSortingCost(
005641 pWInfo, nRowEst, nOrderBy, isOrdered
005642 );
005643 }
005644 /* TUNING: Add a small extra penalty (3) to sorting as an
005645 ** extra encouragement to the query planner to select a plan
005646 ** where the rows emerge in the correct order without any sorting
005647 ** required. */
005648 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3;
005649
005650 WHERETRACE(0x002,
005651 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
005652 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
005653 rUnsorted, rCost));
005654 }else{
005655 rCost = rUnsorted;
005656 rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */
005657 }
005658
005659 /* Check to see if pWLoop should be added to the set of
005660 ** mxChoice best-so-far paths.
005661 **
005662 ** First look for an existing path among best-so-far paths
005663 ** that covers the same set of loops and has the same isOrdered
005664 ** setting as the current path candidate.
005665 **
005666 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
005667 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
005668 ** of legal values for isOrdered, -1..64.
005669 */
005670 testcase( nTo==0 );
005671 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
005672 if( pTo->maskLoop==maskNew
005673 && ((pTo->isOrdered^isOrdered)&0x80)==0
005674 ){
005675 testcase( jj==nTo-1 );
005676 break;
005677 }
005678 }
005679 if( jj>=nTo ){
005680 /* None of the existing best-so-far paths match the candidate. */
005681 if( nTo>=mxChoice
005682 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
005683 ){
005684 /* The current candidate is no better than any of the mxChoice
005685 ** paths currently in the best-so-far buffer. So discard
005686 ** this candidate as not viable. */
005687 #ifdef WHERETRACE_ENABLED /* 0x4 */
005688 if( sqlite3WhereTrace&0x4 ){
005689 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
005690 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005691 isOrdered>=0 ? isOrdered+'0' : '?');
005692 }
005693 #endif
005694 continue;
005695 }
005696 /* If we reach this points it means that the new candidate path
005697 ** needs to be added to the set of best-so-far paths. */
005698 if( nTo<mxChoice ){
005699 /* Increase the size of the aTo set by one */
005700 jj = nTo++;
005701 }else{
005702 /* New path replaces the prior worst to keep count below mxChoice */
005703 jj = mxI;
005704 }
005705 pTo = &aTo[jj];
005706 #ifdef WHERETRACE_ENABLED /* 0x4 */
005707 if( sqlite3WhereTrace&0x4 ){
005708 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
005709 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005710 isOrdered>=0 ? isOrdered+'0' : '?');
005711 }
005712 #endif
005713 }else{
005714 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
005715 ** same set of loops and has the same isOrdered setting as the
005716 ** candidate path. Check to see if the candidate should replace
005717 ** pTo or if the candidate should be skipped.
005718 **
005719 ** The conditional is an expanded vector comparison equivalent to:
005720 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
005721 */
005722 if( pTo->rCost<rCost
005723 || (pTo->rCost==rCost
005724 && (pTo->nRow<nOut
005725 || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted)
005726 )
005727 )
005728 ){
005729 #ifdef WHERETRACE_ENABLED /* 0x4 */
005730 if( sqlite3WhereTrace&0x4 ){
005731 sqlite3DebugPrintf(
005732 "Skip %s cost=%-3d,%3d,%3d order=%c",
005733 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005734 isOrdered>=0 ? isOrdered+'0' : '?');
005735 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
005736 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005737 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005738 }
005739 #endif
005740 /* Discard the candidate path from further consideration */
005741 testcase( pTo->rCost==rCost );
005742 continue;
005743 }
005744 testcase( pTo->rCost==rCost+1 );
005745 /* Control reaches here if the candidate path is better than the
005746 ** pTo path. Replace pTo with the candidate. */
005747 #ifdef WHERETRACE_ENABLED /* 0x4 */
005748 if( sqlite3WhereTrace&0x4 ){
005749 sqlite3DebugPrintf(
005750 "Update %s cost=%-3d,%3d,%3d order=%c",
005751 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005752 isOrdered>=0 ? isOrdered+'0' : '?');
005753 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
005754 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005755 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005756 }
005757 #endif
005758 }
005759 /* pWLoop is a winner. Add it to the set of best so far */
005760 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
005761 pTo->revLoop = revMask;
005762 pTo->nRow = nOut;
005763 pTo->rCost = rCost;
005764 pTo->rUnsorted = rUnsorted;
005765 pTo->isOrdered = isOrdered;
005766 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
005767 pTo->aLoop[iLoop] = pWLoop;
005768 if( nTo>=mxChoice ){
005769 mxI = 0;
005770 mxCost = aTo[0].rCost;
005771 mxUnsorted = aTo[0].nRow;
005772 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
005773 if( pTo->rCost>mxCost
005774 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
005775 ){
005776 mxCost = pTo->rCost;
005777 mxUnsorted = pTo->rUnsorted;
005778 mxI = jj;
005779 }
005780 }
005781 }
005782 }
005783 }
005784
005785 #ifdef WHERETRACE_ENABLED /* >=2 */
005786 if( sqlite3WhereTrace & 0x02 ){
005787 LogEst rMin, rFloor = 0;
005788 int nDone = 0;
005789 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
005790 while( nDone<nTo ){
005791 rMin = 0x7fff;
005792 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005793 if( pTo->rCost>rFloor && pTo->rCost<rMin ) rMin = pTo->rCost;
005794 }
005795 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005796 if( pTo->rCost==rMin ){
005797 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
005798 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005799 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
005800 if( pTo->isOrdered>0 ){
005801 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
005802 }else{
005803 sqlite3DebugPrintf("\n");
005804 }
005805 nDone++;
005806 }
005807 }
005808 rFloor = rMin;
005809 }
005810 }
005811 #endif
005812
005813 /* Swap the roles of aFrom and aTo for the next generation */
005814 pFrom = aTo;
005815 aTo = aFrom;
005816 aFrom = pFrom;
005817 nFrom = nTo;
005818 }
005819
005820 if( nFrom==0 ){
005821 sqlite3ErrorMsg(pParse, "no query solution");
005822 sqlite3StackFreeNN(pParse->db, pSpace);
005823 return SQLITE_ERROR;
005824 }
005825
005826 /* Find the lowest cost path. pFrom will be left pointing to that path */
005827 pFrom = aFrom;
005828 for(ii=1; ii<nFrom; ii++){
005829 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
005830 }
005831 assert( pWInfo->nLevel==nLoop );
005832 /* Load the lowest cost path into pWInfo */
005833 for(iLoop=0; iLoop<nLoop; iLoop++){
005834 WhereLevel *pLevel = pWInfo->a + iLoop;
005835 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
005836 pLevel->iFrom = pWLoop->iTab;
005837 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
005838 }
005839 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
005840 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
005841 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
005842 && nRowEst
005843 ){
005844 Bitmask notUsed;
005845 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
005846 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used);
005847 if( rc==pWInfo->pResultSet->nExpr ){
005848 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005849 }
005850 }
005851 pWInfo->bOrderedInnerLoop = 0;
005852 if( pWInfo->pOrderBy ){
005853 pWInfo->nOBSat = pFrom->isOrdered;
005854 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
005855 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
005856 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005857 }
005858 /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */
005859 assert( pWInfo->pSelect->pOrderBy==0
005860 || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr );
005861 }else{
005862 pWInfo->revMask = pFrom->revLoop;
005863 if( pWInfo->nOBSat<=0 ){
005864 pWInfo->nOBSat = 0;
005865 if( nLoop>0 ){
005866 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
005867 if( (wsFlags & WHERE_ONEROW)==0
005868 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
005869 ){
005870 Bitmask m = 0;
005871 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
005872 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
005873 testcase( wsFlags & WHERE_IPK );
005874 testcase( wsFlags & WHERE_COLUMN_IN );
005875 if( rc==pWInfo->pOrderBy->nExpr ){
005876 pWInfo->bOrderedInnerLoop = 1;
005877 pWInfo->revMask = m;
005878 }
005879 }
005880 }
005881 }else if( nLoop
005882 && pWInfo->nOBSat==1
005883 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0
005884 ){
005885 pWInfo->bOrderedInnerLoop = 1;
005886 }
005887 }
005888 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
005889 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
005890 ){
005891 Bitmask revMask = 0;
005892 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
005893 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
005894 );
005895 assert( pWInfo->sorted==0 );
005896 if( nOrder==pWInfo->pOrderBy->nExpr ){
005897 pWInfo->sorted = 1;
005898 pWInfo->revMask = revMask;
005899 }
005900 }
005901 }
005902
005903 pWInfo->nRowOut = pFrom->nRow + pWInfo->nOutStarDelta;
005904
005905 /* Free temporary memory and return success */
005906 sqlite3StackFreeNN(pParse->db, pSpace);
005907 return SQLITE_OK;
005908 }
005909
005910 /*
005911 ** This routine implements a heuristic designed to improve query planning.
005912 ** This routine is called in between the first and second call to
005913 ** wherePathSolver(). Hence the name "Interstage" "Heuristic".
005914 **
005915 ** The first call to wherePathSolver() (hereafter just "solver()") computes
005916 ** the best path without regard to the order of the outputs. The second call
005917 ** to the solver() builds upon the first call to try to find an alternative
005918 ** path that satisfies the ORDER BY clause.
005919 **
005920 ** This routine looks at the results of the first solver() run, and for
005921 ** every FROM clause term in the resulting query plan that uses an equality
005922 ** constraint against an index, disable other WhereLoops for that same
005923 ** FROM clause term that would try to do a full-table scan. This prevents
005924 ** an index search from being converted into a full-table scan in order to
005925 ** satisfy an ORDER BY clause, since even though we might get slightly better
005926 ** performance using the full-scan without sorting if the output size
005927 ** estimates are very precise, we might also get severe performance
005928 ** degradation using the full-scan if the output size estimate is too large.
005929 ** It is better to err on the side of caution.
005930 **
005931 ** Except, if the first solver() call generated a full-table scan in an outer
005932 ** loop then stop this analysis at the first full-scan, since the second
005933 ** solver() run might try to swap that full-scan for another in order to
005934 ** get the output into the correct order. In other words, we allow a
005935 ** rewrite like this:
005936 **
005937 ** First Solver() Second Solver()
005938 ** |-- SCAN t1 |-- SCAN t2
005939 ** |-- SEARCH t2 `-- SEARCH t1
005940 ** `-- SORT USING B-TREE
005941 **
005942 ** The purpose of this routine is to disallow rewrites such as:
005943 **
005944 ** First Solver() Second Solver()
005945 ** |-- SEARCH t1 |-- SCAN t2 <--- bad!
005946 ** |-- SEARCH t2 `-- SEARCH t1
005947 ** `-- SORT USING B-TREE
005948 **
005949 ** See test cases in test/whereN.test for the real-world query that
005950 ** originally provoked this heuristic.
005951 */
005952 static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){
005953 int i;
005954 #ifdef WHERETRACE_ENABLED
005955 int once = 0;
005956 #endif
005957 for(i=0; i<pWInfo->nLevel; i++){
005958 WhereLoop *p = pWInfo->a[i].pWLoop;
005959 if( p==0 ) break;
005960 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue;
005961 if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){
005962 u8 iTab = p->iTab;
005963 WhereLoop *pLoop;
005964 for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){
005965 if( pLoop->iTab!=iTab ) continue;
005966 if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){
005967 /* Auto-index and index-constrained loops allowed to remain */
005968 continue;
005969 }
005970 #ifdef WHERETRACE_ENABLED
005971 if( sqlite3WhereTrace & 0x80 ){
005972 if( once==0 ){
005973 sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n");
005974 once = 1;
005975 }
005976 sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC);
005977 }
005978 #endif /* WHERETRACE_ENABLED */
005979 pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */
005980 }
005981 }else{
005982 break;
005983 }
005984 }
005985 }
005986
005987 /*
005988 ** Most queries use only a single table (they are not joins) and have
005989 ** simple == constraints against indexed fields. This routine attempts
005990 ** to plan those simple cases using much less ceremony than the
005991 ** general-purpose query planner, and thereby yield faster sqlite3_prepare()
005992 ** times for the common case.
005993 **
005994 ** Return non-zero on success, if this query can be handled by this
005995 ** no-frills query planner. Return zero if this query needs the
005996 ** general-purpose query planner.
005997 */
005998 static int whereShortCut(WhereLoopBuilder *pBuilder){
005999 WhereInfo *pWInfo;
006000 SrcItem *pItem;
006001 WhereClause *pWC;
006002 WhereTerm *pTerm;
006003 WhereLoop *pLoop;
006004 int iCur;
006005 int j;
006006 Table *pTab;
006007 Index *pIdx;
006008 WhereScan scan;
006009
006010 pWInfo = pBuilder->pWInfo;
006011 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
006012 assert( pWInfo->pTabList->nSrc>=1 );
006013 pItem = pWInfo->pTabList->a;
006014 pTab = pItem->pSTab;
006015 if( IsVirtual(pTab) ) return 0;
006016 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
006017 testcase( pItem->fg.isIndexedBy );
006018 testcase( pItem->fg.notIndexed );
006019 return 0;
006020 }
006021 iCur = pItem->iCursor;
006022 pWC = &pWInfo->sWC;
006023 pLoop = pBuilder->pNew;
006024 pLoop->wsFlags = 0;
006025 pLoop->nSkip = 0;
006026 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0);
006027 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
006028 if( pTerm ){
006029 testcase( pTerm->eOperator & WO_IS );
006030 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
006031 pLoop->aLTerm[0] = pTerm;
006032 pLoop->nLTerm = 1;
006033 pLoop->u.btree.nEq = 1;
006034 /* TUNING: Cost of a rowid lookup is 10 */
006035 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
006036 }else{
006037 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
006038 int opMask;
006039 assert( pLoop->aLTermSpace==pLoop->aLTerm );
006040 if( !IsUniqueIndex(pIdx)
006041 || pIdx->pPartIdxWhere!=0
006042 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
006043 ) continue;
006044 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
006045 for(j=0; j<pIdx->nKeyCol; j++){
006046 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx);
006047 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
006048 if( pTerm==0 ) break;
006049 testcase( pTerm->eOperator & WO_IS );
006050 pLoop->aLTerm[j] = pTerm;
006051 }
006052 if( j!=pIdx->nKeyCol ) continue;
006053 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
006054 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){
006055 pLoop->wsFlags |= WHERE_IDX_ONLY;
006056 }
006057 pLoop->nLTerm = j;
006058 pLoop->u.btree.nEq = j;
006059 pLoop->u.btree.pIndex = pIdx;
006060 /* TUNING: Cost of a unique index lookup is 15 */
006061 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
006062 break;
006063 }
006064 }
006065 if( pLoop->wsFlags ){
006066 pLoop->nOut = (LogEst)1;
006067 pWInfo->a[0].pWLoop = pLoop;
006068 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] );
006069 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
006070 pWInfo->a[0].iTabCur = iCur;
006071 pWInfo->nRowOut = 1;
006072 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
006073 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
006074 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006075 }
006076 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS;
006077 #ifdef SQLITE_DEBUG
006078 pLoop->cId = '0';
006079 #endif
006080 #ifdef WHERETRACE_ENABLED
006081 if( sqlite3WhereTrace & 0x02 ){
006082 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
006083 }
006084 #endif
006085 return 1;
006086 }
006087 return 0;
006088 }
006089
006090 /*
006091 ** Helper function for exprIsDeterministic().
006092 */
006093 static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
006094 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){
006095 pWalker->eCode = 0;
006096 return WRC_Abort;
006097 }
006098 return WRC_Continue;
006099 }
006100
006101 /*
006102 ** Return true if the expression contains no non-deterministic SQL
006103 ** functions. Do not consider non-deterministic SQL functions that are
006104 ** part of sub-select statements.
006105 */
006106 static int exprIsDeterministic(Expr *p){
006107 Walker w;
006108 memset(&w, 0, sizeof(w));
006109 w.eCode = 1;
006110 w.xExprCallback = exprNodeIsDeterministic;
006111 w.xSelectCallback = sqlite3SelectWalkFail;
006112 sqlite3WalkExpr(&w, p);
006113 return w.eCode;
006114 }
006115
006116
006117 #ifdef WHERETRACE_ENABLED
006118 /*
006119 ** Display all WhereLoops in pWInfo
006120 */
006121 static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
006122 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
006123 WhereLoop *p;
006124 int i;
006125 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
006126 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
006127 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
006128 p->cId = zLabel[i%(sizeof(zLabel)-1)];
006129 sqlite3WhereLoopPrint(p, pWC);
006130 }
006131 }
006132 }
006133 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
006134 #else
006135 # define WHERETRACE_ALL_LOOPS(W,C)
006136 #endif
006137
006138 /* Attempt to omit tables from a join that do not affect the result.
006139 ** For a table to not affect the result, the following must be true:
006140 **
006141 ** 1) The query must not be an aggregate.
006142 ** 2) The table must be the RHS of a LEFT JOIN.
006143 ** 3) Either the query must be DISTINCT, or else the ON or USING clause
006144 ** must contain a constraint that limits the scan of the table to
006145 ** at most a single row.
006146 ** 4) The table must not be referenced by any part of the query apart
006147 ** from its own USING or ON clause.
006148 ** 5) The table must not have an inner-join ON or USING clause if there is
006149 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause
006150 ** might move from the right side to the left side of the RIGHT JOIN.
006151 ** Note: Due to (2), this condition can only arise if the table is
006152 ** the right-most table of a subquery that was flattened into the
006153 ** main query and that subquery was the right-hand operand of an
006154 ** inner join that held an ON or USING clause.
006155 ** 6) The ORDER BY clause has 63 or fewer terms
006156 ** 7) The omit-noop-join optimization is enabled.
006157 **
006158 ** Items (1), (6), and (7) are checked by the caller.
006159 **
006160 ** For example, given:
006161 **
006162 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
006163 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
006164 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
006165 **
006166 ** then table t2 can be omitted from the following:
006167 **
006168 ** SELECT v1, v3 FROM t1
006169 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
006170 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
006171 **
006172 ** or from:
006173 **
006174 ** SELECT DISTINCT v1, v3 FROM t1
006175 ** LEFT JOIN t2
006176 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
006177 */
006178 static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
006179 WhereInfo *pWInfo,
006180 Bitmask notReady
006181 ){
006182 int i;
006183 Bitmask tabUsed;
006184 int hasRightJoin;
006185
006186 /* Preconditions checked by the caller */
006187 assert( pWInfo->nLevel>=2 );
006188 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) );
006189
006190 /* These two preconditions checked by the caller combine to guarantee
006191 ** condition (1) of the header comment */
006192 assert( pWInfo->pResultSet!=0 );
006193 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) );
006194
006195 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet);
006196 if( pWInfo->pOrderBy ){
006197 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
006198 }
006199 hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0;
006200 for(i=pWInfo->nLevel-1; i>=1; i--){
006201 WhereTerm *pTerm, *pEnd;
006202 SrcItem *pItem;
006203 WhereLoop *pLoop;
006204 Bitmask m1;
006205 pLoop = pWInfo->a[i].pWLoop;
006206 pItem = &pWInfo->pTabList->a[pLoop->iTab];
006207 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
006208 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0
006209 && (pLoop->wsFlags & WHERE_ONEROW)==0
006210 ){
006211 continue;
006212 }
006213 if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
006214 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm;
006215 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
006216 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
006217 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON)
006218 || pTerm->pExpr->w.iJoin!=pItem->iCursor
006219 ){
006220 break;
006221 }
006222 }
006223 if( hasRightJoin
006224 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
006225 && pTerm->pExpr->w.iJoin==pItem->iCursor
006226 ){
006227 break; /* restriction (5) */
006228 }
006229 }
006230 if( pTerm<pEnd ) continue;
006231 WHERETRACE(0xffffffff,("-> omit unused FROM-clause term %c\n",pLoop->cId));
006232 m1 = MASKBIT(i)-1;
006233 testcase( ((pWInfo->revMask>>1) & ~m1)!=0 );
006234 pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1);
006235 notReady &= ~pLoop->maskSelf;
006236 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
006237 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
006238 pTerm->wtFlags |= TERM_CODED;
006239 }
006240 }
006241 if( i!=pWInfo->nLevel-1 ){
006242 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
006243 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
006244 }
006245 pWInfo->nLevel--;
006246 assert( pWInfo->nLevel>0 );
006247 }
006248 return notReady;
006249 }
006250
006251 /*
006252 ** Check to see if there are any SEARCH loops that might benefit from
006253 ** using a Bloom filter. Consider a Bloom filter if:
006254 **
006255 ** (1) The SEARCH happens more than N times where N is the number
006256 ** of rows in the table that is being considered for the Bloom
006257 ** filter.
006258 ** (2) Some searches are expected to find zero rows. (This is determined
006259 ** by the WHERE_SELFCULL flag on the term.)
006260 ** (3) Bloom-filter processing is not disabled. (Checked by the
006261 ** caller.)
006262 ** (4) The size of the table being searched is known by ANALYZE.
006263 **
006264 ** This block of code merely checks to see if a Bloom filter would be
006265 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
006266 ** WhereLoop. The implementation of the Bloom filter comes further
006267 ** down where the code for each WhereLoop is generated.
006268 */
006269 static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
006270 const WhereInfo *pWInfo
006271 ){
006272 int i;
006273 LogEst nSearch = 0;
006274
006275 assert( pWInfo->nLevel>=2 );
006276 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
006277 for(i=0; i<pWInfo->nLevel; i++){
006278 WhereLoop *pLoop = pWInfo->a[i].pWLoop;
006279 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
006280 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
006281 Table *pTab = pItem->pSTab;
006282 if( (pTab->tabFlags & TF_HasStat1)==0 ) break;
006283 pTab->tabFlags |= TF_MaybeReanalyze;
006284 if( i>=1
006285 && (pLoop->wsFlags & reqFlags)==reqFlags
006286 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
006287 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
006288 ){
006289 if( nSearch > pTab->nRowLogEst ){
006290 testcase( pItem->fg.jointype & JT_LEFT );
006291 pLoop->wsFlags |= WHERE_BLOOMFILTER;
006292 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
006293 WHERETRACE(0xffffffff, (
006294 "-> use Bloom-filter on loop %c because there are ~%.1e "
006295 "lookups into %s which has only ~%.1e rows\n",
006296 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
006297 (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
006298 }
006299 }
006300 nSearch += pLoop->nOut;
006301 if( pWInfo->nOutStarDelta ) nSearch += pLoop->rStarDelta;
006302 }
006303 }
006304
006305 /*
006306 ** The index pIdx is used by a query and contains one or more expressions.
006307 ** In other words pIdx is an index on an expression. iIdxCur is the cursor
006308 ** number for the index and iDataCur is the cursor number for the corresponding
006309 ** table.
006310 **
006311 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for
006312 ** each of the expressions in the index so that the expression code generator
006313 ** will know to replace occurrences of the indexed expression with
006314 ** references to the corresponding column of the index.
006315 */
006316 static SQLITE_NOINLINE void whereAddIndexedExpr(
006317 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */
006318 Index *pIdx, /* The index-on-expression that contains the expressions */
006319 int iIdxCur, /* Cursor number for pIdx */
006320 SrcItem *pTabItem /* The FROM clause entry for the table */
006321 ){
006322 int i;
006323 IndexedExpr *p;
006324 Table *pTab;
006325 assert( pIdx->bHasExpr );
006326 pTab = pIdx->pTable;
006327 for(i=0; i<pIdx->nColumn; i++){
006328 Expr *pExpr;
006329 int j = pIdx->aiColumn[i];
006330 if( j==XN_EXPR ){
006331 pExpr = pIdx->aColExpr->a[i].pExpr;
006332 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
006333 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
006334 }else{
006335 continue;
006336 }
006337 if( sqlite3ExprIsConstant(0,pExpr) ) continue;
006338 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
006339 if( p==0 ) break;
006340 p->pIENext = pParse->pIdxEpr;
006341 #ifdef WHERETRACE_ENABLED
006342 if( sqlite3WhereTrace & 0x200 ){
006343 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i);
006344 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr);
006345 }
006346 #endif
006347 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
006348 p->iDataCur = pTabItem->iCursor;
006349 p->iIdxCur = iIdxCur;
006350 p->iIdxCol = i;
006351 p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
006352 if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){
006353 p->aff = pIdx->zColAff[i];
006354 }
006355 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
006356 p->zIdxName = pIdx->zName;
006357 #endif
006358 pParse->pIdxEpr = p;
006359 if( p->pIENext==0 ){
006360 void *pArg = (void*)&pParse->pIdxEpr;
006361 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
006362 }
006363 }
006364 }
006365
006366 /*
006367 ** Set the reverse-scan order mask to one for all tables in the query
006368 ** with the exception of MATERIALIZED common table expressions that have
006369 ** their own internal ORDER BY clauses.
006370 **
006371 ** This implements the PRAGMA reverse_unordered_selects=ON setting.
006372 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
006373 */
006374 static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
006375 int ii;
006376 for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
006377 SrcItem *pItem = &pWInfo->pTabList->a[ii];
006378 if( !pItem->fg.isCte
006379 || pItem->u2.pCteUse->eM10d!=M10d_Yes
006380 || NEVER(pItem->fg.isSubquery==0)
006381 || pItem->u4.pSubq->pSelect->pOrderBy==0
006382 ){
006383 pWInfo->revMask |= MASKBIT(ii);
006384 }
006385 }
006386 }
006387
006388 /*
006389 ** Generate the beginning of the loop used for WHERE clause processing.
006390 ** The return value is a pointer to an opaque structure that contains
006391 ** information needed to terminate the loop. Later, the calling routine
006392 ** should invoke sqlite3WhereEnd() with the return value of this function
006393 ** in order to complete the WHERE clause processing.
006394 **
006395 ** If an error occurs, this routine returns NULL.
006396 **
006397 ** The basic idea is to do a nested loop, one loop for each table in
006398 ** the FROM clause of a select. (INSERT and UPDATE statements are the
006399 ** same as a SELECT with only a single table in the FROM clause.) For
006400 ** example, if the SQL is this:
006401 **
006402 ** SELECT * FROM t1, t2, t3 WHERE ...;
006403 **
006404 ** Then the code generated is conceptually like the following:
006405 **
006406 ** foreach row1 in t1 do \ Code generated
006407 ** foreach row2 in t2 do |-- by sqlite3WhereBegin()
006408 ** foreach row3 in t3 do /
006409 ** ...
006410 ** end \ Code generated
006411 ** end |-- by sqlite3WhereEnd()
006412 ** end /
006413 **
006414 ** Note that the loops might not be nested in the order in which they
006415 ** appear in the FROM clause if a different order is better able to make
006416 ** use of indices. Note also that when the IN operator appears in
006417 ** the WHERE clause, it might result in additional nested loops for
006418 ** scanning through all values on the right-hand side of the IN.
006419 **
006420 ** There are Btree cursors associated with each table. t1 uses cursor
006421 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
006422 ** And so forth. This routine generates code to open those VDBE cursors
006423 ** and sqlite3WhereEnd() generates the code to close them.
006424 **
006425 ** The code that sqlite3WhereBegin() generates leaves the cursors named
006426 ** in pTabList pointing at their appropriate entries. The [...] code
006427 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract
006428 ** data from the various tables of the loop.
006429 **
006430 ** If the WHERE clause is empty, the foreach loops must each scan their
006431 ** entire tables. Thus a three-way join is an O(N^3) operation. But if
006432 ** the tables have indices and there are terms in the WHERE clause that
006433 ** refer to those indices, a complete table scan can be avoided and the
006434 ** code will run much faster. Most of the work of this routine is checking
006435 ** to see if there are indices that can be used to speed up the loop.
006436 **
006437 ** Terms of the WHERE clause are also used to limit which rows actually
006438 ** make it to the "..." in the middle of the loop. After each "foreach",
006439 ** terms of the WHERE clause that use only terms in that loop and outer
006440 ** loops are evaluated and if false a jump is made around all subsequent
006441 ** inner loops (or around the "..." if the test occurs within the inner-
006442 ** most loop)
006443 **
006444 ** OUTER JOINS
006445 **
006446 ** An outer join of tables t1 and t2 is conceptually coded as follows:
006447 **
006448 ** foreach row1 in t1 do
006449 ** flag = 0
006450 ** foreach row2 in t2 do
006451 ** start:
006452 ** ...
006453 ** flag = 1
006454 ** end
006455 ** if flag==0 then
006456 ** move the row2 cursor to a null row
006457 ** goto start
006458 ** fi
006459 ** end
006460 **
006461 ** ORDER BY CLAUSE PROCESSING
006462 **
006463 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
006464 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
006465 ** if there is one. If there is no ORDER BY clause or if this routine
006466 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
006467 **
006468 ** The iIdxCur parameter is the cursor number of an index. If
006469 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
006470 ** to use for OR clause processing. The WHERE clause should use this
006471 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
006472 ** the first cursor in an array of cursors for all indices. iIdxCur should
006473 ** be used to compute the appropriate cursor depending on which index is
006474 ** used.
006475 */
006476 WhereInfo *sqlite3WhereBegin(
006477 Parse *pParse, /* The parser context */
006478 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
006479 Expr *pWhere, /* The WHERE clause */
006480 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
006481 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
006482 Select *pSelect, /* The entire SELECT statement */
006483 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
006484 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
006485 ** If WHERE_USE_LIMIT, then the limit amount */
006486 ){
006487 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
006488 int nTabList; /* Number of elements in pTabList */
006489 WhereInfo *pWInfo; /* Will become the return value of this function */
006490 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
006491 Bitmask notReady; /* Cursors that are not yet positioned */
006492 WhereLoopBuilder sWLB; /* The WhereLoop builder */
006493 WhereMaskSet *pMaskSet; /* The expression mask set */
006494 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
006495 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
006496 int ii; /* Loop counter */
006497 sqlite3 *db; /* Database connection */
006498 int rc; /* Return code */
006499 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
006500
006501 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
006502 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
006503 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006504 ));
006505
006506 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
006507 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006508 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
006509
006510 /* Variable initialization */
006511 db = pParse->db;
006512 memset(&sWLB, 0, sizeof(sWLB));
006513
006514 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
006515 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
006516 if( pOrderBy && pOrderBy->nExpr>=BMS ){
006517 pOrderBy = 0;
006518 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006519 wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */
006520 }
006521
006522 /* The number of tables in the FROM clause is limited by the number of
006523 ** bits in a Bitmask
006524 */
006525 testcase( pTabList->nSrc==BMS );
006526 if( pTabList->nSrc>BMS ){
006527 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
006528 return 0;
006529 }
006530
006531 /* This function normally generates a nested loop for all tables in
006532 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
006533 ** only generate code for the first table in pTabList and assume that
006534 ** any cursors associated with subsequent tables are uninitialized.
006535 */
006536 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
006537
006538 /* Allocate and initialize the WhereInfo structure that will become the
006539 ** return value. A single allocation is used to store the WhereInfo
006540 ** struct, the contents of WhereInfo.a[], the WhereClause structure
006541 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
006542 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
006543 ** some architectures. Hence the ROUND8() below.
006544 */
006545 nByteWInfo = ROUND8P(sizeof(WhereInfo));
006546 if( nTabList>1 ){
006547 nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel));
006548 }
006549 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
006550 if( db->mallocFailed ){
006551 sqlite3DbFree(db, pWInfo);
006552 pWInfo = 0;
006553 goto whereBeginError;
006554 }
006555 pWInfo->pParse = pParse;
006556 pWInfo->pTabList = pTabList;
006557 pWInfo->pOrderBy = pOrderBy;
006558 #if WHERETRACE_ENABLED
006559 pWInfo->pWhere = pWhere;
006560 #endif
006561 pWInfo->pResultSet = pResultSet;
006562 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
006563 pWInfo->nLevel = nTabList;
006564 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse);
006565 pWInfo->wctrlFlags = wctrlFlags;
006566 pWInfo->iLimit = iAuxArg;
006567 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
006568 pWInfo->pSelect = pSelect;
006569 memset(&pWInfo->nOBSat, 0,
006570 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
006571 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
006572 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
006573 pMaskSet = &pWInfo->sMaskSet;
006574 pMaskSet->n = 0;
006575 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be
006576 ** a valid cursor number, to avoid an initial
006577 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
006578 sWLB.pWInfo = pWInfo;
006579 sWLB.pWC = &pWInfo->sWC;
006580 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
006581 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
006582 whereLoopInit(sWLB.pNew);
006583 #ifdef SQLITE_DEBUG
006584 sWLB.pNew->cId = '*';
006585 #endif
006586
006587 /* Split the WHERE clause into separate subexpressions where each
006588 ** subexpression is separated by an AND operator.
006589 */
006590 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
006591 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
006592
006593 /* Special case: No FROM clause
006594 */
006595 if( nTabList==0 ){
006596 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
006597 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0
006598 && OptimizationEnabled(db, SQLITE_DistinctOpt)
006599 ){
006600 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006601 }
006602 if( ALWAYS(pWInfo->pSelect)
006603 && (pWInfo->pSelect->selFlags & SF_MultiValue)==0
006604 ){
006605 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
006606 }
006607 }else{
006608 /* Assign a bit from the bitmask to every term in the FROM clause.
006609 **
006610 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
006611 **
006612 ** The rule of the previous sentence ensures that if X is the bitmask for
006613 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
006614 ** Knowing the bitmask for all tables to the left of a left join is
006615 ** important. Ticket #3015.
006616 **
006617 ** Note that bitmasks are created for all pTabList->nSrc tables in
006618 ** pTabList, not just the first nTabList tables. nTabList is normally
006619 ** equal to pTabList->nSrc but might be shortened to 1 if the
006620 ** WHERE_OR_SUBCLAUSE flag is set.
006621 */
006622 ii = 0;
006623 do{
006624 createMask(pMaskSet, pTabList->a[ii].iCursor);
006625 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
006626 }while( (++ii)<pTabList->nSrc );
006627 #ifdef SQLITE_DEBUG
006628 {
006629 Bitmask mx = 0;
006630 for(ii=0; ii<pTabList->nSrc; ii++){
006631 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
006632 assert( m>=mx );
006633 mx = m;
006634 }
006635 }
006636 #endif
006637 }
006638
006639 /* Analyze all of the subexpressions. */
006640 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
006641 if( pSelect && pSelect->pLimit ){
006642 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
006643 }
006644 if( pParse->nErr ) goto whereBeginError;
006645
006646 /* The False-WHERE-Term-Bypass optimization:
006647 **
006648 ** If there are WHERE terms that are false, then no rows will be output,
006649 ** so skip over all of the code generated here.
006650 **
006651 ** Conditions:
006652 **
006653 ** (1) The WHERE term must not refer to any tables in the join.
006654 ** (2) The term must not come from an ON clause on the
006655 ** right-hand side of a LEFT or FULL JOIN.
006656 ** (3) The term must not come from an ON clause, or there must be
006657 ** no RIGHT or FULL OUTER joins in pTabList.
006658 ** (4) If the expression contains non-deterministic functions
006659 ** that are not within a sub-select. This is not required
006660 ** for correctness but rather to preserves SQLite's legacy
006661 ** behaviour in the following two cases:
006662 **
006663 ** WHERE random()>0; -- eval random() once per row
006664 ** WHERE (SELECT random())>0; -- eval random() just once overall
006665 **
006666 ** Note that the Where term need not be a constant in order for this
006667 ** optimization to apply, though it does need to be constant relative to
006668 ** the current subquery (condition 1). The term might include variables
006669 ** from outer queries so that the value of the term changes from one
006670 ** invocation of the current subquery to the next.
006671 */
006672 for(ii=0; ii<sWLB.pWC->nBase; ii++){
006673 WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */
006674 Expr *pX; /* The expression of pT */
006675 if( pT->wtFlags & TERM_VIRTUAL ) continue;
006676 pX = pT->pExpr;
006677 assert( pX!=0 );
006678 assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
006679 if( pT->prereqAll==0 /* Conditions (1) and (2) */
006680 && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */
006681 && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */
006682 && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
006683 ){
006684 sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
006685 pT->wtFlags |= TERM_CODED;
006686 }
006687 }
006688
006689 if( wctrlFlags & WHERE_WANT_DISTINCT ){
006690 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
006691 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
006692 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
006693 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006694 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT;
006695 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
006696 /* The DISTINCT marking is pointless. Ignore it. */
006697 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006698 }else if( pOrderBy==0 ){
006699 /* Try to ORDER BY the result set to make distinct processing easier */
006700 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
006701 pWInfo->pOrderBy = pResultSet;
006702 }
006703 }
006704
006705 /* Construct the WhereLoop objects */
006706 #if defined(WHERETRACE_ENABLED)
006707 if( sqlite3WhereTrace & 0xffffffff ){
006708 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
006709 if( wctrlFlags & WHERE_USE_LIMIT ){
006710 sqlite3DebugPrintf(", limit: %d", iAuxArg);
006711 }
006712 sqlite3DebugPrintf(")\n");
006713 if( sqlite3WhereTrace & 0x8000 ){
006714 Select sSelect;
006715 memset(&sSelect, 0, sizeof(sSelect));
006716 sSelect.selFlags = SF_WhereBegin;
006717 sSelect.pSrc = pTabList;
006718 sSelect.pWhere = pWhere;
006719 sSelect.pOrderBy = pOrderBy;
006720 sSelect.pEList = pResultSet;
006721 sqlite3TreeViewSelect(0, &sSelect, 0);
006722 }
006723 if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */
006724 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
006725 sqlite3WhereClausePrint(sWLB.pWC);
006726 }
006727 }
006728 #endif
006729
006730 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
006731 rc = whereLoopAddAll(&sWLB);
006732 if( rc ) goto whereBeginError;
006733
006734 #ifdef SQLITE_ENABLE_STAT4
006735 /* If one or more WhereTerm.truthProb values were used in estimating
006736 ** loop parameters, but then those truthProb values were subsequently
006737 ** changed based on STAT4 information while computing subsequent loops,
006738 ** then we need to rerun the whole loop building process so that all
006739 ** loops will be built using the revised truthProb values. */
006740 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
006741 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006742 WHERETRACE(0xffffffff,
006743 ("**** Redo all loop computations due to"
006744 " TERM_HIGHTRUTH changes ****\n"));
006745 while( pWInfo->pLoops ){
006746 WhereLoop *p = pWInfo->pLoops;
006747 pWInfo->pLoops = p->pNextLoop;
006748 whereLoopDelete(db, p);
006749 }
006750 rc = whereLoopAddAll(&sWLB);
006751 if( rc ) goto whereBeginError;
006752 }
006753 #endif
006754 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006755
006756 wherePathSolver(pWInfo, 0);
006757 if( db->mallocFailed ) goto whereBeginError;
006758 if( pWInfo->pOrderBy ){
006759 whereInterstageHeuristic(pWInfo);
006760 wherePathSolver(pWInfo, pWInfo->nRowOut<0 ? 1 : pWInfo->nRowOut+1);
006761 if( db->mallocFailed ) goto whereBeginError;
006762 }
006763
006764 /* TUNING: Assume that a DISTINCT clause on a subquery reduces
006765 ** the output size by a factor of 8 (LogEst -30).
006766 */
006767 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
006768 WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
006769 pWInfo->nRowOut, pWInfo->nRowOut-30));
006770 pWInfo->nRowOut -= 30;
006771 }
006772
006773 }
006774 assert( pWInfo->pTabList!=0 );
006775 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
006776 whereReverseScanOrder(pWInfo);
006777 }
006778 if( pParse->nErr ){
006779 goto whereBeginError;
006780 }
006781 assert( db->mallocFailed==0 );
006782 #ifdef WHERETRACE_ENABLED
006783 if( sqlite3WhereTrace ){
006784 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
006785 if( pWInfo->nOBSat>0 ){
006786 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
006787 }
006788 switch( pWInfo->eDistinct ){
006789 case WHERE_DISTINCT_UNIQUE: {
006790 sqlite3DebugPrintf(" DISTINCT=unique");
006791 break;
006792 }
006793 case WHERE_DISTINCT_ORDERED: {
006794 sqlite3DebugPrintf(" DISTINCT=ordered");
006795 break;
006796 }
006797 case WHERE_DISTINCT_UNORDERED: {
006798 sqlite3DebugPrintf(" DISTINCT=unordered");
006799 break;
006800 }
006801 }
006802 sqlite3DebugPrintf("\n");
006803 for(ii=0; ii<pWInfo->nLevel; ii++){
006804 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
006805 }
006806 }
006807 #endif
006808
006809 /* Attempt to omit tables from a join that do not affect the result.
006810 ** See the comment on whereOmitNoopJoin() for further information.
006811 **
006812 ** This query optimization is factored out into a separate "no-inline"
006813 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
006814 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
006815 ** some C-compiler optimizers from in-lining the
006816 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
006817 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
006818 */
006819 notReady = ~(Bitmask)0;
006820 if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */
006821 && pResultSet!=0 /* Condition (1) */
006822 && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */
006823 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */
006824 ){
006825 notReady = whereOmitNoopJoin(pWInfo, notReady);
006826 nTabList = pWInfo->nLevel;
006827 assert( nTabList>0 );
006828 }
006829
006830 /* Check to see if there are any SEARCH loops that might benefit from
006831 ** using a Bloom filter.
006832 */
006833 if( pWInfo->nLevel>=2
006834 && OptimizationEnabled(db, SQLITE_BloomFilter)
006835 ){
006836 whereCheckIfBloomFilterIsUseful(pWInfo);
006837 }
006838
006839 #if defined(WHERETRACE_ENABLED)
006840 if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */
006841 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
006842 sqlite3WhereClausePrint(sWLB.pWC);
006843 }
006844 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n"));
006845 #endif
006846 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
006847
006848 /* If the caller is an UPDATE or DELETE statement that is requesting
006849 ** to use a one-pass algorithm, determine if this is appropriate.
006850 **
006851 ** A one-pass approach can be used if the caller has requested one
006852 ** and either (a) the scan visits at most one row or (b) each
006853 ** of the following are true:
006854 **
006855 ** * the caller has indicated that a one-pass approach can be used
006856 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
006857 ** * the table is not a virtual table, and
006858 ** * either the scan does not use the OR optimization or the caller
006859 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
006860 ** for DELETE).
006861 **
006862 ** The last qualification is because an UPDATE statement uses
006863 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
006864 ** use a one-pass approach, and this is not set accurately for scans
006865 ** that use the OR optimization.
006866 */
006867 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
006868 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
006869 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
006870 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
006871 assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) );
006872 if( bOnerow || (
006873 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
006874 && !IsVirtual(pTabList->a[0].pSTab)
006875 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
006876 && OptimizationEnabled(db, SQLITE_OnePass)
006877 )){
006878 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
006879 if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){
006880 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
006881 bFordelete = OPFLAG_FORDELETE;
006882 }
006883 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
006884 }
006885 }
006886 }
006887
006888 /* Open all tables in the pTabList and any indices selected for
006889 ** searching those tables.
006890 */
006891 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
006892 Table *pTab; /* Table to open */
006893 int iDb; /* Index of database containing table/index */
006894 SrcItem *pTabItem;
006895
006896 pTabItem = &pTabList->a[pLevel->iFrom];
006897 pTab = pTabItem->pSTab;
006898 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
006899 pLoop = pLevel->pWLoop;
006900 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
006901 /* Do nothing */
006902 }else
006903 #ifndef SQLITE_OMIT_VIRTUALTABLE
006904 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
006905 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
006906 int iCur = pTabItem->iCursor;
006907 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
006908 }else if( IsVirtual(pTab) ){
006909 /* noop */
006910 }else
006911 #endif
006912 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0
006913 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0)
006914 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0
006915 ){
006916 int op = OP_OpenRead;
006917 if( pWInfo->eOnePass!=ONEPASS_OFF ){
006918 op = OP_OpenWrite;
006919 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
006920 };
006921 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
006922 assert( pTabItem->iCursor==pLevel->iTabCur );
006923 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
006924 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
006925 if( pWInfo->eOnePass==ONEPASS_OFF
006926 && pTab->nCol<BMS
006927 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
006928 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
006929 ){
006930 /* If we know that only a prefix of the record will be used,
006931 ** it is advantageous to reduce the "column count" field in
006932 ** the P4 operand of the OP_OpenRead/Write opcode. */
006933 Bitmask b = pTabItem->colUsed;
006934 int n = 0;
006935 for(; b; b=b>>1, n++){}
006936 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
006937 assert( n<=pTab->nCol );
006938 }
006939 #ifdef SQLITE_ENABLE_CURSOR_HINTS
006940 if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){
006941 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
006942 }else
006943 #endif
006944 {
006945 sqlite3VdbeChangeP5(v, bFordelete);
006946 }
006947 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006948 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
006949 (const u8*)&pTabItem->colUsed, P4_INT64);
006950 #endif
006951 }else{
006952 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
006953 }
006954 if( pLoop->wsFlags & WHERE_INDEXED ){
006955 Index *pIx = pLoop->u.btree.pIndex;
006956 int iIndexCur;
006957 int op = OP_OpenRead;
006958 /* iAuxArg is always set to a positive value if ONEPASS is possible */
006959 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
006960 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
006961 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
006962 ){
006963 /* This is one term of an OR-optimization using the PRIMARY KEY of a
006964 ** WITHOUT ROWID table. No need for a separate index */
006965 iIndexCur = pLevel->iTabCur;
006966 op = 0;
006967 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
006968 Index *pJ = pTabItem->pSTab->pIndex;
006969 iIndexCur = iAuxArg;
006970 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
006971 while( ALWAYS(pJ) && pJ!=pIx ){
006972 iIndexCur++;
006973 pJ = pJ->pNext;
006974 }
006975 op = OP_OpenWrite;
006976 pWInfo->aiCurOnePass[1] = iIndexCur;
006977 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
006978 iIndexCur = iAuxArg;
006979 op = OP_ReopenIdx;
006980 }else{
006981 iIndexCur = pParse->nTab++;
006982 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
006983 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
006984 }
006985 if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){
006986 wherePartIdxExpr(
006987 pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem
006988 );
006989 }
006990 }
006991 pLevel->iIdxCur = iIndexCur;
006992 assert( pIx!=0 );
006993 assert( pIx->pSchema==pTab->pSchema );
006994 assert( iIndexCur>=0 );
006995 if( op ){
006996 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
006997 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
006998 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
006999 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
007000 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
007001 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
007002 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
007003 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
007004 ){
007005 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
007006 }
007007 VdbeComment((v, "%s", pIx->zName));
007008 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
007009 {
007010 u64 colUsed = 0;
007011 int ii, jj;
007012 for(ii=0; ii<pIx->nColumn; ii++){
007013 jj = pIx->aiColumn[ii];
007014 if( jj<0 ) continue;
007015 if( jj>63 ) jj = 63;
007016 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
007017 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
007018 }
007019 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
007020 (u8*)&colUsed, P4_INT64);
007021 }
007022 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
007023 }
007024 }
007025 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
007026 if( (pTabItem->fg.jointype & JT_RIGHT)!=0
007027 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0
007028 ){
007029 WhereRightJoin *pRJ = pLevel->pRJ;
007030 pRJ->iMatch = pParse->nTab++;
007031 pRJ->regBloom = ++pParse->nMem;
007032 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
007033 pRJ->regReturn = ++pParse->nMem;
007034 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
007035 assert( pTab==pTabItem->pSTab );
007036 if( HasRowid(pTab) ){
007037 KeyInfo *pInfo;
007038 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
007039 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0);
007040 if( pInfo ){
007041 pInfo->aColl[0] = 0;
007042 pInfo->aSortFlags[0] = 0;
007043 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO);
007044 }
007045 }else{
007046 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
007047 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol);
007048 sqlite3VdbeSetP4KeyInfo(pParse, pPk);
007049 }
007050 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
007051 /* The nature of RIGHT JOIN processing is such that it messes up
007052 ** the output order. So omit any ORDER BY/GROUP BY elimination
007053 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
007054 pWInfo->nOBSat = 0;
007055 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED;
007056 }
007057 }
007058 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
007059 if( db->mallocFailed ) goto whereBeginError;
007060
007061 /* Generate the code to do the search. Each iteration of the for
007062 ** loop below generates code for a single nested loop of the VM
007063 ** program.
007064 */
007065 for(ii=0; ii<nTabList; ii++){
007066 int addrExplain;
007067 int wsFlags;
007068 SrcItem *pSrc;
007069 if( pParse->nErr ) goto whereBeginError;
007070 pLevel = &pWInfo->a[ii];
007071 wsFlags = pLevel->pWLoop->wsFlags;
007072 pSrc = &pTabList->a[pLevel->iFrom];
007073 if( pSrc->fg.isMaterialized ){
007074 Subquery *pSubq;
007075 int iOnce = 0;
007076 assert( pSrc->fg.isSubquery );
007077 pSubq = pSrc->u4.pSubq;
007078 if( pSrc->fg.isCorrelated==0 ){
007079 iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
007080 }else{
007081 iOnce = 0;
007082 }
007083 sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub);
007084 VdbeComment((v, "materialize %!S", pSrc));
007085 if( iOnce ) sqlite3VdbeJumpHere(v, iOnce);
007086 }
007087 assert( pTabList == pWInfo->pTabList );
007088 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
007089 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
007090 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
007091 constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel);
007092 #endif
007093 }else{
007094 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
007095 }
007096 if( db->mallocFailed ) goto whereBeginError;
007097 }
007098 addrExplain = sqlite3WhereExplainOneScan(
007099 pParse, pTabList, pLevel, wctrlFlags
007100 );
007101 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
007102 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady);
007103 pWInfo->iContinue = pLevel->addrCont;
007104 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
007105 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
007106 }
007107 }
007108
007109 /* Done. */
007110 VdbeModuleComment((v, "Begin WHERE-core"));
007111 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
007112 return pWInfo;
007113
007114 /* Jump here if malloc fails */
007115 whereBeginError:
007116 if( pWInfo ){
007117 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
007118 whereInfoFree(db, pWInfo);
007119 }
007120 #ifdef WHERETRACE_ENABLED
007121 /* Prevent harmless compiler warnings about debugging routines
007122 ** being declared but never used */
007123 sqlite3ShowWhereLoopList(0);
007124 #endif /* WHERETRACE_ENABLED */
007125 return 0;
007126 }
007127
007128 /*
007129 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
007130 ** index rather than the main table. In SQLITE_DEBUG mode, we want
007131 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
007132 ** does that.
007133 */
007134 #ifndef SQLITE_DEBUG
007135 # define OpcodeRewriteTrace(D,K,P) /* no-op */
007136 #else
007137 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
007138 static void sqlite3WhereOpcodeRewriteTrace(
007139 sqlite3 *db,
007140 int pc,
007141 VdbeOp *pOp
007142 ){
007143 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
007144 sqlite3VdbePrintOp(0, pc, pOp);
007145 }
007146 #endif
007147
007148 /*
007149 ** Generate the end of the WHERE loop. See comments on
007150 ** sqlite3WhereBegin() for additional information.
007151 */
007152 void sqlite3WhereEnd(WhereInfo *pWInfo){
007153 Parse *pParse = pWInfo->pParse;
007154 Vdbe *v = pParse->pVdbe;
007155 int i;
007156 WhereLevel *pLevel;
007157 WhereLoop *pLoop;
007158 SrcList *pTabList = pWInfo->pTabList;
007159 sqlite3 *db = pParse->db;
007160 int iEnd = sqlite3VdbeCurrentAddr(v);
007161 int nRJ = 0;
007162
007163 /* Generate loop termination code.
007164 */
007165 VdbeModuleComment((v, "End WHERE-core"));
007166 for(i=pWInfo->nLevel-1; i>=0; i--){
007167 int addr;
007168 pLevel = &pWInfo->a[i];
007169 if( pLevel->pRJ ){
007170 /* Terminate the subroutine that forms the interior of the loop of
007171 ** the RIGHT JOIN table */
007172 WhereRightJoin *pRJ = pLevel->pRJ;
007173 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
007174 pLevel->addrCont = 0;
007175 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
007176 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
007177 VdbeCoverage(v);
007178 nRJ++;
007179 }
007180 pLoop = pLevel->pWLoop;
007181 if( pLevel->op!=OP_Noop ){
007182 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
007183 int addrSeek = 0;
007184 Index *pIdx;
007185 int n;
007186 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
007187 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
007188 && (pLoop->wsFlags & WHERE_INDEXED)!=0
007189 && (pIdx = pLoop->u.btree.pIndex)->hasStat1
007190 && (n = pLoop->u.btree.nDistinctCol)>0
007191 && pIdx->aiRowLogEst[n]>=36
007192 ){
007193 int r1 = pParse->nMem+1;
007194 int j, op;
007195 for(j=0; j<n; j++){
007196 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
007197 }
007198 pParse->nMem += n+1;
007199 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
007200 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
007201 VdbeCoverageIf(v, op==OP_SeekLT);
007202 VdbeCoverageIf(v, op==OP_SeekGT);
007203 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
007204 }
007205 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
007206 /* The common case: Advance to the next row */
007207 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
007208 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
007209 sqlite3VdbeChangeP5(v, pLevel->p5);
007210 VdbeCoverage(v);
007211 VdbeCoverageIf(v, pLevel->op==OP_Next);
007212 VdbeCoverageIf(v, pLevel->op==OP_Prev);
007213 VdbeCoverageIf(v, pLevel->op==OP_VNext);
007214 if( pLevel->regBignull ){
007215 sqlite3VdbeResolveLabel(v, pLevel->addrBignull);
007216 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1);
007217 VdbeCoverage(v);
007218 }
007219 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
007220 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
007221 #endif
007222 }else if( pLevel->addrCont ){
007223 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
007224 }
007225 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
007226 struct InLoop *pIn;
007227 int j;
007228 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
007229 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
007230 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull
007231 || pParse->db->mallocFailed );
007232 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
007233 if( pIn->eEndLoopOp!=OP_Noop ){
007234 if( pIn->nPrefix ){
007235 int bEarlyOut =
007236 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
007237 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
007238 if( pLevel->iLeftJoin ){
007239 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
007240 ** opened yet. This occurs for WHERE clauses such as
007241 ** "a = ? AND b IN (...)", where the index is on (a, b). If
007242 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
007243 ** never have been coded, but the body of the loop run to
007244 ** return the null-row. So, if the cursor is not open yet,
007245 ** jump over the OP_Next or OP_Prev instruction about to
007246 ** be coded. */
007247 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
007248 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
007249 VdbeCoverage(v);
007250 }
007251 if( bEarlyOut ){
007252 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
007253 sqlite3VdbeCurrentAddr(v)+2,
007254 pIn->iBase, pIn->nPrefix);
007255 VdbeCoverage(v);
007256 /* Retarget the OP_IsNull against the left operand of IN so
007257 ** it jumps past the OP_IfNoHope. This is because the
007258 ** OP_IsNull also bypasses the OP_Affinity opcode that is
007259 ** required by OP_IfNoHope. */
007260 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
007261 }
007262 }
007263 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
007264 VdbeCoverage(v);
007265 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev);
007266 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next);
007267 }
007268 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
007269 }
007270 }
007271 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
007272 if( pLevel->pRJ ){
007273 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1);
007274 VdbeCoverage(v);
007275 }
007276 if( pLevel->addrSkip ){
007277 sqlite3VdbeGoto(v, pLevel->addrSkip);
007278 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
007279 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
007280 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
007281 }
007282 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
007283 if( pLevel->addrLikeRep ){
007284 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
007285 pLevel->addrLikeRep);
007286 VdbeCoverage(v);
007287 }
007288 #endif
007289 if( pLevel->iLeftJoin ){
007290 int ws = pLoop->wsFlags;
007291 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
007292 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
007293 if( (ws & WHERE_IDX_ONLY)==0 ){
007294 SrcItem *pSrc = &pTabList->a[pLevel->iFrom];
007295 assert( pLevel->iTabCur==pSrc->iCursor );
007296 if( pSrc->fg.viaCoroutine ){
007297 int m, n;
007298 assert( pSrc->fg.isSubquery );
007299 n = pSrc->u4.pSubq->regResult;
007300 assert( pSrc->pSTab!=0 );
007301 m = pSrc->pSTab->nCol;
007302 sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1);
007303 }
007304 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
007305 }
007306 if( (ws & WHERE_INDEXED)
007307 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
007308 ){
007309 if( ws & WHERE_MULTI_OR ){
007310 Index *pIx = pLevel->u.pCoveringIdx;
007311 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema);
007312 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb);
007313 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
007314 }
007315 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
007316 }
007317 if( pLevel->op==OP_Return ){
007318 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
007319 }else{
007320 sqlite3VdbeGoto(v, pLevel->addrFirst);
007321 }
007322 sqlite3VdbeJumpHere(v, addr);
007323 }
007324 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
007325 pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName));
007326 }
007327
007328 assert( pWInfo->nLevel<=pTabList->nSrc );
007329 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
007330 int k, last;
007331 VdbeOp *pOp, *pLastOp;
007332 Index *pIdx = 0;
007333 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
007334 Table *pTab = pTabItem->pSTab;
007335 assert( pTab!=0 );
007336 pLoop = pLevel->pWLoop;
007337
007338 /* Do RIGHT JOIN processing. Generate code that will output the
007339 ** unmatched rows of the right operand of the RIGHT JOIN with
007340 ** all of the columns of the left operand set to NULL.
007341 */
007342 if( pLevel->pRJ ){
007343 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel);
007344 continue;
007345 }
007346
007347 /* For a co-routine, change all OP_Column references to the table of
007348 ** the co-routine into OP_Copy of result contained in a register.
007349 ** OP_Rowid becomes OP_Null.
007350 */
007351 if( pTabItem->fg.viaCoroutine ){
007352 testcase( pParse->db->mallocFailed );
007353 assert( pTabItem->fg.isSubquery );
007354 assert( pTabItem->u4.pSubq->regResult>=0 );
007355 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
007356 pTabItem->u4.pSubq->regResult, 0);
007357 continue;
007358 }
007359
007360 /* If this scan uses an index, make VDBE code substitutions to read data
007361 ** from the index instead of from the table where possible. In some cases
007362 ** this optimization prevents the table from ever being read, which can
007363 ** yield a significant performance boost.
007364 **
007365 ** Calls to the code generator in between sqlite3WhereBegin and
007366 ** sqlite3WhereEnd will have created code that references the table
007367 ** directly. This loop scans all that code looking for opcodes
007368 ** that reference the table and converts them into opcodes that
007369 ** reference the index.
007370 */
007371 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
007372 pIdx = pLoop->u.btree.pIndex;
007373 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
007374 pIdx = pLevel->u.pCoveringIdx;
007375 }
007376 if( pIdx
007377 && !db->mallocFailed
007378 ){
007379 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
007380 last = iEnd;
007381 }else{
007382 last = pWInfo->iEndWhere;
007383 }
007384 if( pIdx->bHasExpr ){
007385 IndexedExpr *p = pParse->pIdxEpr;
007386 while( p ){
007387 if( p->iIdxCur==pLevel->iIdxCur ){
007388 #ifdef WHERETRACE_ENABLED
007389 if( sqlite3WhereTrace & 0x200 ){
007390 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n",
007391 p->iIdxCur, p->iIdxCol);
007392 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr);
007393 }
007394 #endif
007395 p->iDataCur = -1;
007396 p->iIdxCur = -1;
007397 }
007398 p = p->pIENext;
007399 }
007400 }
007401 k = pLevel->addrBody + 1;
007402 #ifdef SQLITE_DEBUG
007403 if( db->flags & SQLITE_VdbeAddopTrace ){
007404 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
007405 pLevel->iTabCur, pLevel->iIdxCur, k, last-1);
007406 }
007407 /* Proof that the "+1" on the k value above is safe */
007408 pOp = sqlite3VdbeGetOp(v, k - 1);
007409 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
007410 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur );
007411 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
007412 #endif
007413 pOp = sqlite3VdbeGetOp(v, k);
007414 pLastOp = pOp + (last - k);
007415 assert( pOp<=pLastOp );
007416 do{
007417 if( pOp->p1!=pLevel->iTabCur ){
007418 /* no-op */
007419 }else if( pOp->opcode==OP_Column
007420 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
007421 || pOp->opcode==OP_Offset
007422 #endif
007423 ){
007424 int x = pOp->p2;
007425 assert( pIdx->pTable==pTab );
007426 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
007427 if( pOp->opcode==OP_Offset ){
007428 /* Do not need to translate the column number */
007429 }else
007430 #endif
007431 if( !HasRowid(pTab) ){
007432 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
007433 x = pPk->aiColumn[x];
007434 assert( x>=0 );
007435 }else{
007436 testcase( x!=sqlite3StorageColumnToTable(pTab,x) );
007437 x = sqlite3StorageColumnToTable(pTab,x);
007438 }
007439 x = sqlite3TableColumnToIndex(pIdx, x);
007440 if( x>=0 ){
007441 pOp->p2 = x;
007442 pOp->p1 = pLevel->iIdxCur;
007443 OpcodeRewriteTrace(db, k, pOp);
007444 }else{
007445 /* Unable to translate the table reference into an index
007446 ** reference. Verify that this is harmless - that the
007447 ** table being referenced really is open.
007448 */
007449 if( pLoop->wsFlags & WHERE_IDX_ONLY ){
007450 sqlite3ErrorMsg(pParse, "internal query planner error");
007451 pParse->rc = SQLITE_INTERNAL;
007452 }
007453 }
007454 }else if( pOp->opcode==OP_Rowid ){
007455 pOp->p1 = pLevel->iIdxCur;
007456 pOp->opcode = OP_IdxRowid;
007457 OpcodeRewriteTrace(db, k, pOp);
007458 }else if( pOp->opcode==OP_IfNullRow ){
007459 pOp->p1 = pLevel->iIdxCur;
007460 OpcodeRewriteTrace(db, k, pOp);
007461 }
007462 #ifdef SQLITE_DEBUG
007463 k++;
007464 #endif
007465 }while( (++pOp)<pLastOp );
007466 #ifdef SQLITE_DEBUG
007467 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
007468 #endif
007469 }
007470 }
007471
007472 /* The "break" point is here, just past the end of the outer loop.
007473 ** Set it.
007474 */
007475 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
007476
007477 /* Final cleanup
007478 */
007479 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
007480 whereInfoFree(db, pWInfo);
007481 pParse->withinRJSubrtn -= nRJ;
007482 return;
007483 }