@@ -94,11 +94,10 @@ struct IncrementLoopInfo {
94
94
template <typename T>
95
95
explicit IncrementLoopInfo (Fortran::semantics::Symbol &sym, const T &lower,
96
96
const T &upper, const std::optional<T> &step,
97
- bool isConcurrent = false )
97
+ bool isUnordered = false )
98
98
: loopVariableSym{&sym}, lowerExpr{Fortran::semantics::GetExpr (lower)},
99
99
upperExpr{Fortran::semantics::GetExpr (upper)},
100
- stepExpr{Fortran::semantics::GetExpr (step)},
101
- isConcurrent{isConcurrent} {}
100
+ stepExpr{Fortran::semantics::GetExpr (step)}, isUnordered{isUnordered} {}
102
101
103
102
IncrementLoopInfo (IncrementLoopInfo &&) = default ;
104
103
IncrementLoopInfo &operator =(IncrementLoopInfo &&x) = default ;
@@ -121,7 +120,7 @@ struct IncrementLoopInfo {
121
120
const Fortran::lower::SomeExpr *upperExpr;
122
121
const Fortran::lower::SomeExpr *stepExpr;
123
122
const Fortran::lower::SomeExpr *maskExpr = nullptr ;
124
- bool isConcurrent;
123
+ bool isUnordered; // do concurrent, forall
125
124
llvm::SmallVector<const Fortran::semantics::Symbol *> localSymList;
126
125
llvm::SmallVector<const Fortran::semantics::Symbol *> localInitSymList;
127
126
llvm::SmallVector<
@@ -131,7 +130,7 @@ struct IncrementLoopInfo {
131
130
mlir::Value loopVariable = nullptr ;
132
131
133
132
// Data members for structured loops.
134
- mlir::Operation *loopOp = nullptr ;
133
+ fir::DoLoopOp doLoop = nullptr ;
135
134
136
135
// Data members for unstructured loops.
137
136
bool hasRealControl = false ;
@@ -1981,7 +1980,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
1981
1980
llvm_unreachable (" illegal reduction operator" );
1982
1981
}
1983
1982
1984
- // / Collect DO CONCURRENT loop control information.
1983
+ // / Collect DO CONCURRENT or FORALL loop control information.
1985
1984
IncrementLoopNestInfo getConcurrentControl (
1986
1985
const Fortran::parser::ConcurrentHeader &header,
1987
1986
const std::list<Fortran::parser::LocalitySpec> &localityList = {}) {
@@ -2292,14 +2291,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
2292
2291
mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get (
2293
2292
builder->getContext (), {}, /* vectorize=*/ va, {}, /* unroll*/ ua,
2294
2293
/* unroll_and_jam*/ uja, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});
2295
- if (has_attrs) {
2296
- if (auto loopOp = mlir::dyn_cast<fir::DoLoopOp>(info.loopOp ))
2297
- loopOp.setLoopAnnotationAttr (la);
2298
-
2299
- if (auto doConcurrentOp =
2300
- mlir::dyn_cast<fir::DoConcurrentLoopOp>(info.loopOp ))
2301
- doConcurrentOp.setLoopAnnotationAttr (la);
2302
- }
2294
+ if (has_attrs)
2295
+ info.doLoop .setLoopAnnotationAttr (la);
2303
2296
}
2304
2297
2305
2298
// / Generate FIR to begin a structured or unstructured increment loop nest.
@@ -2308,77 +2301,96 @@ class FirConverter : public Fortran::lower::AbstractConverter {
2308
2301
llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) {
2309
2302
assert (!incrementLoopNestInfo.empty () && " empty loop nest" );
2310
2303
mlir::Location loc = toLocation ();
2304
+ mlir::Operation *boundsAndStepIP = nullptr ;
2311
2305
mlir::arith::IntegerOverflowFlags iofBackup{};
2312
2306
2313
- llvm::SmallVector<mlir::Value> nestLBs;
2314
- llvm::SmallVector<mlir::Value> nestUBs;
2315
- llvm::SmallVector<mlir::Value> nestSts;
2316
- llvm::SmallVector<mlir::Value> nestReduceOperands;
2317
- llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
2318
- bool genDoConcurrent = false ;
2319
-
2320
2307
for (IncrementLoopInfo &info : incrementLoopNestInfo) {
2321
- genDoConcurrent = info.isStructured () && info.isConcurrent ;
2322
-
2323
- if (!genDoConcurrent)
2324
- info.loopVariable = genLoopVariableAddress (loc, *info.loopVariableSym ,
2325
- info.isConcurrent );
2326
-
2327
- if (!getLoweringOptions ().getIntegerWrapAround ()) {
2328
- iofBackup = builder->getIntegerOverflowFlags ();
2329
- builder->setIntegerOverflowFlags (
2330
- mlir::arith::IntegerOverflowFlags::nsw);
2331
- }
2308
+ mlir::Value lowerValue;
2309
+ mlir::Value upperValue;
2310
+ mlir::Value stepValue;
2332
2311
2333
- nestLBs.push_back (genControlValue (info.lowerExpr , info));
2334
- nestUBs.push_back (genControlValue (info.upperExpr , info));
2335
- bool isConst = true ;
2336
- nestSts.push_back (genControlValue (
2337
- info.stepExpr , info, info.isStructured () ? nullptr : &isConst));
2312
+ {
2313
+ mlir::OpBuilder::InsertionGuard guard (*builder);
2338
2314
2339
- if (!getLoweringOptions ().getIntegerWrapAround ())
2340
- builder->setIntegerOverflowFlags (iofBackup);
2315
+ // Set the IP before the first loop in the nest so that all nest bounds
2316
+ // and step values are created outside the nest.
2317
+ if (boundsAndStepIP)
2318
+ builder->setInsertionPointAfter (boundsAndStepIP);
2341
2319
2342
- // Use a temp variable for unstructured loops with non-const step.
2343
- if (!isConst) {
2344
- mlir::Value stepValue = nestSts.back ();
2345
- info.stepVariable = builder->createTemporary (loc, stepValue.getType ());
2346
- builder->create <fir::StoreOp>(loc, stepValue, info.stepVariable );
2347
- }
2348
-
2349
- if (genDoConcurrent && nestReduceOperands.empty ()) {
2350
- // Create DO CONCURRENT reduce operands and attributes
2351
- for (const auto &reduceSym : info.reduceSymList ) {
2352
- const fir::ReduceOperationEnum reduceOperation = reduceSym.first ;
2353
- const Fortran::semantics::Symbol *sym = reduceSym.second ;
2354
- fir::ExtendedValue exv = getSymbolExtendedValue (*sym, nullptr );
2355
- nestReduceOperands.push_back (fir::getBase (exv));
2356
- auto reduceAttr =
2357
- fir::ReduceAttr::get (builder->getContext (), reduceOperation);
2358
- nestReduceAttrs.push_back (reduceAttr);
2320
+ info.loopVariable = genLoopVariableAddress (loc, *info.loopVariableSym ,
2321
+ info.isUnordered );
2322
+ if (!getLoweringOptions ().getIntegerWrapAround ()) {
2323
+ iofBackup = builder->getIntegerOverflowFlags ();
2324
+ builder->setIntegerOverflowFlags (
2325
+ mlir::arith::IntegerOverflowFlags::nsw);
2326
+ }
2327
+ lowerValue = genControlValue (info.lowerExpr , info);
2328
+ upperValue = genControlValue (info.upperExpr , info);
2329
+ bool isConst = true ;
2330
+ stepValue = genControlValue (info.stepExpr , info,
2331
+ info.isStructured () ? nullptr : &isConst);
2332
+ if (!getLoweringOptions ().getIntegerWrapAround ())
2333
+ builder->setIntegerOverflowFlags (iofBackup);
2334
+ boundsAndStepIP = stepValue.getDefiningOp ();
2335
+
2336
+ // Use a temp variable for unstructured loops with non-const step.
2337
+ if (!isConst) {
2338
+ info.stepVariable =
2339
+ builder->createTemporary (loc, stepValue.getType ());
2340
+ boundsAndStepIP =
2341
+ builder->create <fir::StoreOp>(loc, stepValue, info.stepVariable );
2359
2342
}
2360
2343
}
2361
- }
2362
2344
2363
- for (auto [info, lowerValue, upperValue, stepValue] :
2364
- llvm::zip_equal (incrementLoopNestInfo, nestLBs, nestUBs, nestSts)) {
2365
2345
// Structured loop - generate fir.do_loop.
2366
2346
if (info.isStructured ()) {
2367
- if (genDoConcurrent)
2368
- continue ;
2369
-
2370
- // The loop variable is a doLoop op argument.
2371
2347
mlir::Type loopVarType = info.getLoopVariableType ();
2372
- auto loopOp = builder->create <fir::DoLoopOp>(
2373
- loc, lowerValue, upperValue, stepValue, /* unordered=*/ false ,
2374
- /* finalCountValue=*/ true ,
2375
- builder->createConvert (loc, loopVarType, lowerValue));
2376
- info.loopOp = loopOp;
2377
- builder->setInsertionPointToStart (loopOp.getBody ());
2378
- mlir::Value loopValue = loopOp.getRegionIterArgs ()[0 ];
2379
-
2348
+ mlir::Value loopValue;
2349
+ if (info.isUnordered ) {
2350
+ llvm::SmallVector<mlir::Value> reduceOperands;
2351
+ llvm::SmallVector<mlir::Attribute> reduceAttrs;
2352
+ // Create DO CONCURRENT reduce operands and attributes
2353
+ for (const auto &reduceSym : info.reduceSymList ) {
2354
+ const fir::ReduceOperationEnum reduce_operation = reduceSym.first ;
2355
+ const Fortran::semantics::Symbol *sym = reduceSym.second ;
2356
+ fir::ExtendedValue exv = getSymbolExtendedValue (*sym, nullptr );
2357
+ reduceOperands.push_back (fir::getBase (exv));
2358
+ auto reduce_attr =
2359
+ fir::ReduceAttr::get (builder->getContext (), reduce_operation);
2360
+ reduceAttrs.push_back (reduce_attr);
2361
+ }
2362
+ // The loop variable value is explicitly updated.
2363
+ info.doLoop = builder->create <fir::DoLoopOp>(
2364
+ loc, lowerValue, upperValue, stepValue, /* unordered=*/ true ,
2365
+ /* finalCountValue=*/ false , /* iterArgs=*/ std::nullopt,
2366
+ llvm::ArrayRef<mlir::Value>(reduceOperands), reduceAttrs);
2367
+ builder->setInsertionPointToStart (info.doLoop .getBody ());
2368
+ loopValue = builder->createConvert (loc, loopVarType,
2369
+ info.doLoop .getInductionVar ());
2370
+ } else {
2371
+ // The loop variable is a doLoop op argument.
2372
+ info.doLoop = builder->create <fir::DoLoopOp>(
2373
+ loc, lowerValue, upperValue, stepValue, /* unordered=*/ false ,
2374
+ /* finalCountValue=*/ true ,
2375
+ builder->createConvert (loc, loopVarType, lowerValue));
2376
+ builder->setInsertionPointToStart (info.doLoop .getBody ());
2377
+ loopValue = info.doLoop .getRegionIterArgs ()[0 ];
2378
+ }
2380
2379
// Update the loop variable value in case it has non-index references.
2381
2380
builder->create <fir::StoreOp>(loc, loopValue, info.loopVariable );
2381
+ if (info.maskExpr ) {
2382
+ Fortran::lower::StatementContext stmtCtx;
2383
+ mlir::Value maskCond = createFIRExpr (loc, info.maskExpr , stmtCtx);
2384
+ stmtCtx.finalizeAndReset ();
2385
+ mlir::Value maskCondCast =
2386
+ builder->createConvert (loc, builder->getI1Type (), maskCond);
2387
+ auto ifOp = builder->create <fir::IfOp>(loc, maskCondCast,
2388
+ /* withElseRegion=*/ false );
2389
+ builder->setInsertionPointToStart (&ifOp.getThenRegion ().front ());
2390
+ }
2391
+ if (info.hasLocalitySpecs ())
2392
+ handleLocalitySpecs (info);
2393
+
2382
2394
addLoopAnnotationAttr (info, dirs);
2383
2395
continue ;
2384
2396
}
@@ -2442,60 +2454,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
2442
2454
builder->restoreInsertionPoint (insertPt);
2443
2455
}
2444
2456
}
2445
-
2446
- if (genDoConcurrent) {
2447
- auto loopWrapperOp = builder->create <fir::DoConcurrentOp>(loc);
2448
- builder->setInsertionPointToStart (
2449
- builder->createBlock (&loopWrapperOp.getRegion ()));
2450
-
2451
- for (IncrementLoopInfo &info : llvm::reverse (incrementLoopNestInfo)) {
2452
- info.loopVariable = genLoopVariableAddress (loc, *info.loopVariableSym ,
2453
- info.isConcurrent );
2454
- }
2455
-
2456
- builder->setInsertionPointToEnd (loopWrapperOp.getBody ());
2457
- auto loopOp = builder->create <fir::DoConcurrentLoopOp>(
2458
- loc, nestLBs, nestUBs, nestSts, nestReduceOperands,
2459
- nestReduceAttrs.empty ()
2460
- ? nullptr
2461
- : mlir::ArrayAttr::get (builder->getContext (), nestReduceAttrs),
2462
- nullptr );
2463
-
2464
- llvm::SmallVector<mlir::Type> loopBlockArgTypes (
2465
- incrementLoopNestInfo.size (), builder->getIndexType ());
2466
- llvm::SmallVector<mlir::Location> loopBlockArgLocs (
2467
- incrementLoopNestInfo.size (), loc);
2468
- mlir::Region &loopRegion = loopOp.getRegion ();
2469
- mlir::Block *loopBlock = builder->createBlock (
2470
- &loopRegion, loopRegion.begin (), loopBlockArgTypes, loopBlockArgLocs);
2471
- builder->setInsertionPointToStart (loopBlock);
2472
-
2473
- for (auto [info, blockArg] :
2474
- llvm::zip_equal (incrementLoopNestInfo, loopBlock->getArguments ())) {
2475
- info.loopOp = loopOp;
2476
- mlir::Value loopValue =
2477
- builder->createConvert (loc, info.getLoopVariableType (), blockArg);
2478
- builder->create <fir::StoreOp>(loc, loopValue, info.loopVariable );
2479
-
2480
- if (info.maskExpr ) {
2481
- Fortran::lower::StatementContext stmtCtx;
2482
- mlir::Value maskCond = createFIRExpr (loc, info.maskExpr , stmtCtx);
2483
- stmtCtx.finalizeAndReset ();
2484
- mlir::Value maskCondCast =
2485
- builder->createConvert (loc, builder->getI1Type (), maskCond);
2486
- auto ifOp = builder->create <fir::IfOp>(loc, maskCondCast,
2487
- /* withElseRegion=*/ false );
2488
- builder->setInsertionPointToStart (&ifOp.getThenRegion ().front ());
2489
- }
2490
- }
2491
-
2492
- IncrementLoopInfo &innermostInfo = incrementLoopNestInfo.back ();
2493
-
2494
- if (innermostInfo.hasLocalitySpecs ())
2495
- handleLocalitySpecs (innermostInfo);
2496
-
2497
- addLoopAnnotationAttr (innermostInfo, dirs);
2498
- }
2499
2457
}
2500
2458
2501
2459
// / Generate FIR to end a structured or unstructured increment loop nest.
@@ -2512,31 +2470,29 @@ class FirConverter : public Fortran::lower::AbstractConverter {
2512
2470
it != rend; ++it) {
2513
2471
IncrementLoopInfo &info = *it;
2514
2472
if (info.isStructured ()) {
2515
- // End fir.do_concurent.loop .
2516
- if (info.isConcurrent ) {
2517
- builder->setInsertionPointAfter (info.loopOp -> getParentOp () );
2473
+ // End fir.do_loop .
2474
+ if (info.isUnordered ) {
2475
+ builder->setInsertionPointAfter (info.doLoop );
2518
2476
continue ;
2519
2477
}
2520
-
2521
- // End fir.do_loop.
2522
2478
// Decrement tripVariable.
2523
- auto doLoopOp = mlir::cast<fir::DoLoopOp>(info.loopOp );
2524
- builder->setInsertionPointToEnd (doLoopOp.getBody ());
2479
+ builder->setInsertionPointToEnd (info.doLoop .getBody ());
2525
2480
llvm::SmallVector<mlir::Value, 2 > results;
2526
2481
results.push_back (builder->create <mlir::arith::AddIOp>(
2527
- loc, doLoopOp.getInductionVar (), doLoopOp.getStep (), iofAttr));
2482
+ loc, info.doLoop .getInductionVar (), info.doLoop .getStep (),
2483
+ iofAttr));
2528
2484
// Step loopVariable to help optimizations such as vectorization.
2529
2485
// Induction variable elimination will clean up as necessary.
2530
2486
mlir::Value step = builder->createConvert (
2531
- loc, info.getLoopVariableType (), doLoopOp .getStep ());
2487
+ loc, info.getLoopVariableType (), info. doLoop .getStep ());
2532
2488
mlir::Value loopVar =
2533
2489
builder->create <fir::LoadOp>(loc, info.loopVariable );
2534
2490
results.push_back (
2535
2491
builder->create <mlir::arith::AddIOp>(loc, loopVar, step, iofAttr));
2536
2492
builder->create <fir::ResultOp>(loc, results);
2537
- builder->setInsertionPointAfter (doLoopOp );
2493
+ builder->setInsertionPointAfter (info. doLoop );
2538
2494
// The loop control variable may be used after the loop.
2539
- builder->create <fir::StoreOp>(loc, doLoopOp .getResult (1 ),
2495
+ builder->create <fir::StoreOp>(loc, info. doLoop .getResult (1 ),
2540
2496
info.loopVariable );
2541
2497
continue ;
2542
2498
}
0 commit comments