1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "CodeGenFunction.h"
15 #include "CGCall.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Operator.h"
24
25 using namespace clang;
26 using namespace CodeGen;
27
28 namespace {
29 class AtomicInfo {
30 CodeGenFunction &CGF;
31 QualType AtomicTy;
32 QualType ValueTy;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
36 CharUnits ValueAlign;
37 CharUnits LValueAlign;
38 TypeEvaluationKind EvaluationKind;
39 bool UseLibcall;
40 LValue LVal;
41 CGBitFieldInfo BFI;
42 public:
AtomicInfo(CodeGenFunction & CGF,LValue & lvalue)43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
52 else
53 ValueTy = AtomicTy;
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
61
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
65
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
68
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
73
74 LVal = lvalue;
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .RoundUpToAlignment(lvalue.getAlignment()));
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
84 auto OffsetInChars =
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88 VoidPtrAddr, OffsetInChars.getQuantity());
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90 VoidPtrAddr,
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92 "atomic_bitfield_base");
93 BFI = OrigBFI;
94 BFI.Offset = Offset;
95 BFI.StorageSize = AtomicSizeInBits;
96 BFI.StorageOffset += OffsetInChars;
97 LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
98 lvalue.getAlignment());
99 LVal.setTBAAInfo(lvalue.getTBAAInfo());
100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101 if (AtomicTy.isNull()) {
102 llvm::APInt Size(
103 /*numBits=*/32,
104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
106 /*IndexTypeQuals=*/0);
107 }
108 AtomicAlign = ValueAlign = lvalue.getAlignment();
109 } else if (lvalue.isVectorElt()) {
110 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
111 ValueSizeInBits = C.getTypeSize(ValueTy);
112 AtomicTy = lvalue.getType();
113 AtomicSizeInBits = C.getTypeSize(AtomicTy);
114 AtomicAlign = ValueAlign = lvalue.getAlignment();
115 LVal = lvalue;
116 } else {
117 assert(lvalue.isExtVectorElt());
118 ValueTy = lvalue.getType();
119 ValueSizeInBits = C.getTypeSize(ValueTy);
120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121 lvalue.getType(), lvalue.getExtVectorAddr()
122 ->getType()
123 ->getPointerElementType()
124 ->getVectorNumElements());
125 AtomicSizeInBits = C.getTypeSize(AtomicTy);
126 AtomicAlign = ValueAlign = lvalue.getAlignment();
127 LVal = lvalue;
128 }
129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131 }
132
getAtomicType() const133 QualType getAtomicType() const { return AtomicTy; }
getValueType() const134 QualType getValueType() const { return ValueTy; }
getAtomicAlignment() const135 CharUnits getAtomicAlignment() const { return AtomicAlign; }
getValueAlignment() const136 CharUnits getValueAlignment() const { return ValueAlign; }
getAtomicSizeInBits() const137 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
getValueSizeInBits() const138 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
getEvaluationKind() const139 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
shouldUseLibcall() const140 bool shouldUseLibcall() const { return UseLibcall; }
getAtomicLValue() const141 const LValue &getAtomicLValue() const { return LVal; }
getAtomicAddress() const142 llvm::Value *getAtomicAddress() const {
143 if (LVal.isSimple())
144 return LVal.getAddress();
145 else if (LVal.isBitField())
146 return LVal.getBitFieldAddr();
147 else if (LVal.isVectorElt())
148 return LVal.getVectorAddr();
149 assert(LVal.isExtVectorElt());
150 return LVal.getExtVectorAddr();
151 }
152
153 /// Is the atomic size larger than the underlying value type?
154 ///
155 /// Note that the absence of padding does not mean that atomic
156 /// objects are completely interchangeable with non-atomic
157 /// objects: we might have promoted the alignment of a type
158 /// without making it bigger.
hasPadding() const159 bool hasPadding() const {
160 return (ValueSizeInBits != AtomicSizeInBits);
161 }
162
163 bool emitMemSetZeroIfNecessary() const;
164
getAtomicSizeValue() const165 llvm::Value *getAtomicSizeValue() const {
166 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
167 return CGF.CGM.getSize(size);
168 }
169
170 /// Cast the given pointer to an integer pointer suitable for
171 /// atomic operations.
172 llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
173
174 /// Turn an atomic-layout object into an r-value.
175 RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
176 SourceLocation loc, bool AsValue) const;
177
178 /// \brief Converts a rvalue to integer value.
179 llvm::Value *convertRValueToInt(RValue RVal) const;
180
181 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
182 AggValueSlot ResultSlot,
183 SourceLocation Loc, bool AsValue) const;
184
185 /// Copy an atomic r-value into atomic-layout memory.
186 void emitCopyIntoMemory(RValue rvalue) const;
187
188 /// Project an l-value down to the value field.
projectValue() const189 LValue projectValue() const {
190 assert(LVal.isSimple());
191 llvm::Value *addr = getAtomicAddress();
192 if (hasPadding())
193 addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
194
195 return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
196 CGF.getContext(), LVal.getTBAAInfo());
197 }
198
199 /// \brief Emits atomic load.
200 /// \returns Loaded value.
201 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
202 bool AsValue, llvm::AtomicOrdering AO,
203 bool IsVolatile);
204
205 /// \brief Emits atomic compare-and-exchange sequence.
206 /// \param Expected Expected value.
207 /// \param Desired Desired value.
208 /// \param Success Atomic ordering for success operation.
209 /// \param Failure Atomic ordering for failed operation.
210 /// \param IsWeak true if atomic operation is weak, false otherwise.
211 /// \returns Pair of values: previous value from storage (value type) and
212 /// boolean flag (i1 type) with true if success and false otherwise.
213 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
214 RValue Expected, RValue Desired,
215 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
216 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
217 bool IsWeak = false);
218
219 /// \brief Emits atomic update.
220 /// \param AO Atomic ordering.
221 /// \param UpdateOp Update operation for the current lvalue.
222 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
223 const llvm::function_ref<RValue(RValue)> &UpdateOp,
224 bool IsVolatile);
225 /// \brief Emits atomic update.
226 /// \param AO Atomic ordering.
227 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
228 bool IsVolatile);
229
230 /// Materialize an atomic r-value in atomic-layout memory.
231 llvm::Value *materializeRValue(RValue rvalue) const;
232
233 /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
234 /// libcalls.
235 static AtomicExpr::AtomicOrderingKind
236 translateAtomicOrdering(const llvm::AtomicOrdering AO);
237
238 private:
239 bool requiresMemSetZero(llvm::Type *type) const;
240
241 /// \brief Creates temp alloca for intermediate operations on atomic value.
242 llvm::Value *CreateTempAlloca() const;
243
244 /// \brief Emits atomic load as a libcall.
245 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
246 llvm::AtomicOrdering AO, bool IsVolatile);
247 /// \brief Emits atomic load as LLVM instruction.
248 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
249 /// \brief Emits atomic compare-and-exchange op as a libcall.
250 llvm::Value *EmitAtomicCompareExchangeLibcall(
251 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
252 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
253 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
254 /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
255 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
256 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
257 llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
258 llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
259 bool IsWeak = false);
260 /// \brief Emit atomic update as libcalls.
261 void
262 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
263 const llvm::function_ref<RValue(RValue)> &UpdateOp,
264 bool IsVolatile);
265 /// \brief Emit atomic update as LLVM instructions.
266 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
267 const llvm::function_ref<RValue(RValue)> &UpdateOp,
268 bool IsVolatile);
269 /// \brief Emit atomic update as libcalls.
270 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
271 bool IsVolatile);
272 /// \brief Emit atomic update as LLVM instructions.
273 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
274 bool IsVolatile);
275 };
276 }
277
278 AtomicExpr::AtomicOrderingKind
translateAtomicOrdering(const llvm::AtomicOrdering AO)279 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
280 switch (AO) {
281 case llvm::Unordered:
282 case llvm::NotAtomic:
283 case llvm::Monotonic:
284 return AtomicExpr::AO_ABI_memory_order_relaxed;
285 case llvm::Acquire:
286 return AtomicExpr::AO_ABI_memory_order_acquire;
287 case llvm::Release:
288 return AtomicExpr::AO_ABI_memory_order_release;
289 case llvm::AcquireRelease:
290 return AtomicExpr::AO_ABI_memory_order_acq_rel;
291 case llvm::SequentiallyConsistent:
292 return AtomicExpr::AO_ABI_memory_order_seq_cst;
293 }
294 llvm_unreachable("Unhandled AtomicOrdering");
295 }
296
CreateTempAlloca() const297 llvm::Value *AtomicInfo::CreateTempAlloca() const {
298 auto *TempAlloca = CGF.CreateMemTemp(
299 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
300 : AtomicTy,
301 "atomic-temp");
302 TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
303 // Cast to pointer to value type for bitfields.
304 if (LVal.isBitField())
305 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
306 TempAlloca, getAtomicAddress()->getType());
307 return TempAlloca;
308 }
309
emitAtomicLibcall(CodeGenFunction & CGF,StringRef fnName,QualType resultType,CallArgList & args)310 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
311 StringRef fnName,
312 QualType resultType,
313 CallArgList &args) {
314 const CGFunctionInfo &fnInfo =
315 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
316 FunctionType::ExtInfo(), RequiredArgs::All);
317 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
318 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
319 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
320 }
321
322 /// Does a store of the given IR type modify the full expected width?
isFullSizeType(CodeGenModule & CGM,llvm::Type * type,uint64_t expectedSize)323 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
324 uint64_t expectedSize) {
325 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
326 }
327
328 /// Does the atomic type require memsetting to zero before initialization?
329 ///
330 /// The IR type is provided as a way of making certain queries faster.
requiresMemSetZero(llvm::Type * type) const331 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
332 // If the atomic type has size padding, we definitely need a memset.
333 if (hasPadding()) return true;
334
335 // Otherwise, do some simple heuristics to try to avoid it:
336 switch (getEvaluationKind()) {
337 // For scalars and complexes, check whether the store size of the
338 // type uses the full size.
339 case TEK_Scalar:
340 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
341 case TEK_Complex:
342 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
343 AtomicSizeInBits / 2);
344
345 // Padding in structs has an undefined bit pattern. User beware.
346 case TEK_Aggregate:
347 return false;
348 }
349 llvm_unreachable("bad evaluation kind");
350 }
351
emitMemSetZeroIfNecessary() const352 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
353 assert(LVal.isSimple());
354 llvm::Value *addr = LVal.getAddress();
355 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
356 return false;
357
358 CGF.Builder.CreateMemSet(
359 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
360 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
361 LVal.getAlignment().getQuantity());
362 return true;
363 }
364
emitAtomicCmpXchg(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,llvm::Value * Dest,llvm::Value * Ptr,llvm::Value * Val1,llvm::Value * Val2,uint64_t Size,unsigned Align,llvm::AtomicOrdering SuccessOrder,llvm::AtomicOrdering FailureOrder)365 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
366 llvm::Value *Dest, llvm::Value *Ptr,
367 llvm::Value *Val1, llvm::Value *Val2,
368 uint64_t Size, unsigned Align,
369 llvm::AtomicOrdering SuccessOrder,
370 llvm::AtomicOrdering FailureOrder) {
371 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
372 llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
373 Expected->setAlignment(Align);
374 llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
375 Desired->setAlignment(Align);
376
377 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
378 Ptr, Expected, Desired, SuccessOrder, FailureOrder);
379 Pair->setVolatile(E->isVolatile());
380 Pair->setWeak(IsWeak);
381
382 // Cmp holds the result of the compare-exchange operation: true on success,
383 // false on failure.
384 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
385 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
386
387 // This basic block is used to hold the store instruction if the operation
388 // failed.
389 llvm::BasicBlock *StoreExpectedBB =
390 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
391
392 // This basic block is the exit point of the operation, we should end up
393 // here regardless of whether or not the operation succeeded.
394 llvm::BasicBlock *ContinueBB =
395 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
396
397 // Update Expected if Expected isn't equal to Old, otherwise branch to the
398 // exit point.
399 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
400
401 CGF.Builder.SetInsertPoint(StoreExpectedBB);
402 // Update the memory at Expected with Old's value.
403 llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
404 StoreExpected->setAlignment(Align);
405 // Finally, branch to the exit point.
406 CGF.Builder.CreateBr(ContinueBB);
407
408 CGF.Builder.SetInsertPoint(ContinueBB);
409 // Update the memory at Dest with Cmp's value.
410 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
411 return;
412 }
413
414 /// Given an ordering required on success, emit all possible cmpxchg
415 /// instructions to cope with the provided (but possibly only dynamically known)
416 /// FailureOrder.
emitAtomicCmpXchgFailureSet(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,llvm::Value * Dest,llvm::Value * Ptr,llvm::Value * Val1,llvm::Value * Val2,llvm::Value * FailureOrderVal,uint64_t Size,unsigned Align,llvm::AtomicOrdering SuccessOrder)417 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
418 bool IsWeak, llvm::Value *Dest,
419 llvm::Value *Ptr, llvm::Value *Val1,
420 llvm::Value *Val2,
421 llvm::Value *FailureOrderVal,
422 uint64_t Size, unsigned Align,
423 llvm::AtomicOrdering SuccessOrder) {
424 llvm::AtomicOrdering FailureOrder;
425 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
426 switch (FO->getSExtValue()) {
427 default:
428 FailureOrder = llvm::Monotonic;
429 break;
430 case AtomicExpr::AO_ABI_memory_order_consume:
431 case AtomicExpr::AO_ABI_memory_order_acquire:
432 FailureOrder = llvm::Acquire;
433 break;
434 case AtomicExpr::AO_ABI_memory_order_seq_cst:
435 FailureOrder = llvm::SequentiallyConsistent;
436 break;
437 }
438 if (FailureOrder >= SuccessOrder) {
439 // Don't assert on undefined behaviour.
440 FailureOrder =
441 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
442 }
443 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
444 SuccessOrder, FailureOrder);
445 return;
446 }
447
448 // Create all the relevant BB's
449 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
450 *SeqCstBB = nullptr;
451 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
452 if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
453 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
454 if (SuccessOrder == llvm::SequentiallyConsistent)
455 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
456
457 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
458
459 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
460
461 // Emit all the different atomics
462
463 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
464 // doesn't matter unless someone is crazy enough to use something that
465 // doesn't fold to a constant for the ordering.
466 CGF.Builder.SetInsertPoint(MonotonicBB);
467 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
468 Size, Align, SuccessOrder, llvm::Monotonic);
469 CGF.Builder.CreateBr(ContBB);
470
471 if (AcquireBB) {
472 CGF.Builder.SetInsertPoint(AcquireBB);
473 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
474 Size, Align, SuccessOrder, llvm::Acquire);
475 CGF.Builder.CreateBr(ContBB);
476 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
477 AcquireBB);
478 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
479 AcquireBB);
480 }
481 if (SeqCstBB) {
482 CGF.Builder.SetInsertPoint(SeqCstBB);
483 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
484 Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
485 CGF.Builder.CreateBr(ContBB);
486 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
487 SeqCstBB);
488 }
489
490 CGF.Builder.SetInsertPoint(ContBB);
491 }
492
EmitAtomicOp(CodeGenFunction & CGF,AtomicExpr * E,llvm::Value * Dest,llvm::Value * Ptr,llvm::Value * Val1,llvm::Value * Val2,llvm::Value * IsWeak,llvm::Value * FailureOrder,uint64_t Size,unsigned Align,llvm::AtomicOrdering Order)493 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
494 llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
495 llvm::Value *IsWeak, llvm::Value *FailureOrder,
496 uint64_t Size, unsigned Align,
497 llvm::AtomicOrdering Order) {
498 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
499 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
500
501 switch (E->getOp()) {
502 case AtomicExpr::AO__c11_atomic_init:
503 llvm_unreachable("Already handled!");
504
505 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
506 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
507 FailureOrder, Size, Align, Order);
508 return;
509 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
510 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
511 FailureOrder, Size, Align, Order);
512 return;
513 case AtomicExpr::AO__atomic_compare_exchange:
514 case AtomicExpr::AO__atomic_compare_exchange_n: {
515 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
516 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
517 Val1, Val2, FailureOrder, Size, Align, Order);
518 } else {
519 // Create all the relevant BB's
520 llvm::BasicBlock *StrongBB =
521 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
522 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
523 llvm::BasicBlock *ContBB =
524 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
525
526 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
527 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
528
529 CGF.Builder.SetInsertPoint(StrongBB);
530 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
531 FailureOrder, Size, Align, Order);
532 CGF.Builder.CreateBr(ContBB);
533
534 CGF.Builder.SetInsertPoint(WeakBB);
535 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
536 FailureOrder, Size, Align, Order);
537 CGF.Builder.CreateBr(ContBB);
538
539 CGF.Builder.SetInsertPoint(ContBB);
540 }
541 return;
542 }
543 case AtomicExpr::AO__c11_atomic_load:
544 case AtomicExpr::AO__atomic_load_n:
545 case AtomicExpr::AO__atomic_load: {
546 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
547 Load->setAtomic(Order);
548 Load->setAlignment(Size);
549 Load->setVolatile(E->isVolatile());
550 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
551 StoreDest->setAlignment(Align);
552 return;
553 }
554
555 case AtomicExpr::AO__c11_atomic_store:
556 case AtomicExpr::AO__atomic_store:
557 case AtomicExpr::AO__atomic_store_n: {
558 assert(!Dest && "Store does not return a value");
559 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
560 LoadVal1->setAlignment(Align);
561 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
562 Store->setAtomic(Order);
563 Store->setAlignment(Size);
564 Store->setVolatile(E->isVolatile());
565 return;
566 }
567
568 case AtomicExpr::AO__c11_atomic_exchange:
569 case AtomicExpr::AO__atomic_exchange_n:
570 case AtomicExpr::AO__atomic_exchange:
571 Op = llvm::AtomicRMWInst::Xchg;
572 break;
573
574 case AtomicExpr::AO__atomic_add_fetch:
575 PostOp = llvm::Instruction::Add;
576 // Fall through.
577 case AtomicExpr::AO__c11_atomic_fetch_add:
578 case AtomicExpr::AO__atomic_fetch_add:
579 Op = llvm::AtomicRMWInst::Add;
580 break;
581
582 case AtomicExpr::AO__atomic_sub_fetch:
583 PostOp = llvm::Instruction::Sub;
584 // Fall through.
585 case AtomicExpr::AO__c11_atomic_fetch_sub:
586 case AtomicExpr::AO__atomic_fetch_sub:
587 Op = llvm::AtomicRMWInst::Sub;
588 break;
589
590 case AtomicExpr::AO__atomic_and_fetch:
591 PostOp = llvm::Instruction::And;
592 // Fall through.
593 case AtomicExpr::AO__c11_atomic_fetch_and:
594 case AtomicExpr::AO__atomic_fetch_and:
595 Op = llvm::AtomicRMWInst::And;
596 break;
597
598 case AtomicExpr::AO__atomic_or_fetch:
599 PostOp = llvm::Instruction::Or;
600 // Fall through.
601 case AtomicExpr::AO__c11_atomic_fetch_or:
602 case AtomicExpr::AO__atomic_fetch_or:
603 Op = llvm::AtomicRMWInst::Or;
604 break;
605
606 case AtomicExpr::AO__atomic_xor_fetch:
607 PostOp = llvm::Instruction::Xor;
608 // Fall through.
609 case AtomicExpr::AO__c11_atomic_fetch_xor:
610 case AtomicExpr::AO__atomic_fetch_xor:
611 Op = llvm::AtomicRMWInst::Xor;
612 break;
613
614 case AtomicExpr::AO__atomic_nand_fetch:
615 PostOp = llvm::Instruction::And;
616 // Fall through.
617 case AtomicExpr::AO__atomic_fetch_nand:
618 Op = llvm::AtomicRMWInst::Nand;
619 break;
620 }
621
622 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
623 LoadVal1->setAlignment(Align);
624 llvm::AtomicRMWInst *RMWI =
625 CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
626 RMWI->setVolatile(E->isVolatile());
627
628 // For __atomic_*_fetch operations, perform the operation again to
629 // determine the value which was written.
630 llvm::Value *Result = RMWI;
631 if (PostOp)
632 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
633 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
634 Result = CGF.Builder.CreateNot(Result);
635 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
636 StoreDest->setAlignment(Align);
637 }
638
639 // This function emits any expression (scalar, complex, or aggregate)
640 // into a temporary alloca.
641 static llvm::Value *
EmitValToTemp(CodeGenFunction & CGF,Expr * E)642 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
643 llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
644 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
645 /*Init*/ true);
646 return DeclPtr;
647 }
648
649 static void
AddDirectArgument(CodeGenFunction & CGF,CallArgList & Args,bool UseOptimizedLibcall,llvm::Value * Val,QualType ValTy,SourceLocation Loc,CharUnits SizeInChars)650 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
651 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
652 SourceLocation Loc, CharUnits SizeInChars) {
653 if (UseOptimizedLibcall) {
654 // Load value and pass it to the function directly.
655 unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
656 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
657 ValTy =
658 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
659 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
660 SizeInBits)->getPointerTo();
661 Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
662 Align, CGF.getContext().getPointerType(ValTy),
663 Loc);
664 // Coerce the value into an appropriately sized integer type.
665 Args.add(RValue::get(Val), ValTy);
666 } else {
667 // Non-optimized functions always take a reference.
668 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
669 CGF.getContext().VoidPtrTy);
670 }
671 }
672
EmitAtomicExpr(AtomicExpr * E,llvm::Value * Dest)673 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
674 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
675 QualType MemTy = AtomicTy;
676 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
677 MemTy = AT->getValueType();
678 CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
679 uint64_t Size = sizeChars.getQuantity();
680 CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
681 unsigned Align = alignChars.getQuantity();
682 unsigned MaxInlineWidthInBits =
683 getTarget().getMaxAtomicInlineWidth();
684 bool UseLibcall = (Size != Align ||
685 getContext().toBits(sizeChars) > MaxInlineWidthInBits);
686
687 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
688 *Val2 = nullptr;
689 llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
690
691 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
692 assert(!Dest && "Init does not return a value");
693 LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
694 EmitAtomicInit(E->getVal1(), lvalue);
695 return RValue::get(nullptr);
696 }
697
698 llvm::Value *Order = EmitScalarExpr(E->getOrder());
699
700 switch (E->getOp()) {
701 case AtomicExpr::AO__c11_atomic_init:
702 llvm_unreachable("Already handled above with EmitAtomicInit!");
703
704 case AtomicExpr::AO__c11_atomic_load:
705 case AtomicExpr::AO__atomic_load_n:
706 break;
707
708 case AtomicExpr::AO__atomic_load:
709 Dest = EmitScalarExpr(E->getVal1());
710 break;
711
712 case AtomicExpr::AO__atomic_store:
713 Val1 = EmitScalarExpr(E->getVal1());
714 break;
715
716 case AtomicExpr::AO__atomic_exchange:
717 Val1 = EmitScalarExpr(E->getVal1());
718 Dest = EmitScalarExpr(E->getVal2());
719 break;
720
721 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
722 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
723 case AtomicExpr::AO__atomic_compare_exchange_n:
724 case AtomicExpr::AO__atomic_compare_exchange:
725 Val1 = EmitScalarExpr(E->getVal1());
726 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
727 Val2 = EmitScalarExpr(E->getVal2());
728 else
729 Val2 = EmitValToTemp(*this, E->getVal2());
730 OrderFail = EmitScalarExpr(E->getOrderFail());
731 if (E->getNumSubExprs() == 6)
732 IsWeak = EmitScalarExpr(E->getWeak());
733 break;
734
735 case AtomicExpr::AO__c11_atomic_fetch_add:
736 case AtomicExpr::AO__c11_atomic_fetch_sub:
737 if (MemTy->isPointerType()) {
738 // For pointer arithmetic, we're required to do a bit of math:
739 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
740 // ... but only for the C11 builtins. The GNU builtins expect the
741 // user to multiply by sizeof(T).
742 QualType Val1Ty = E->getVal1()->getType();
743 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
744 CharUnits PointeeIncAmt =
745 getContext().getTypeSizeInChars(MemTy->getPointeeType());
746 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
747 Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
748 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
749 break;
750 }
751 // Fall through.
752 case AtomicExpr::AO__atomic_fetch_add:
753 case AtomicExpr::AO__atomic_fetch_sub:
754 case AtomicExpr::AO__atomic_add_fetch:
755 case AtomicExpr::AO__atomic_sub_fetch:
756 case AtomicExpr::AO__c11_atomic_store:
757 case AtomicExpr::AO__c11_atomic_exchange:
758 case AtomicExpr::AO__atomic_store_n:
759 case AtomicExpr::AO__atomic_exchange_n:
760 case AtomicExpr::AO__c11_atomic_fetch_and:
761 case AtomicExpr::AO__c11_atomic_fetch_or:
762 case AtomicExpr::AO__c11_atomic_fetch_xor:
763 case AtomicExpr::AO__atomic_fetch_and:
764 case AtomicExpr::AO__atomic_fetch_or:
765 case AtomicExpr::AO__atomic_fetch_xor:
766 case AtomicExpr::AO__atomic_fetch_nand:
767 case AtomicExpr::AO__atomic_and_fetch:
768 case AtomicExpr::AO__atomic_or_fetch:
769 case AtomicExpr::AO__atomic_xor_fetch:
770 case AtomicExpr::AO__atomic_nand_fetch:
771 Val1 = EmitValToTemp(*this, E->getVal1());
772 break;
773 }
774
775 QualType RValTy = E->getType().getUnqualifiedType();
776
777 auto GetDest = [&] {
778 if (!RValTy->isVoidType() && !Dest) {
779 Dest = CreateMemTemp(RValTy, ".atomicdst");
780 }
781 return Dest;
782 };
783
784 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
785 if (UseLibcall) {
786 bool UseOptimizedLibcall = false;
787 switch (E->getOp()) {
788 case AtomicExpr::AO__c11_atomic_init:
789 llvm_unreachable("Already handled above with EmitAtomicInit!");
790
791 case AtomicExpr::AO__c11_atomic_fetch_add:
792 case AtomicExpr::AO__atomic_fetch_add:
793 case AtomicExpr::AO__c11_atomic_fetch_and:
794 case AtomicExpr::AO__atomic_fetch_and:
795 case AtomicExpr::AO__c11_atomic_fetch_or:
796 case AtomicExpr::AO__atomic_fetch_or:
797 case AtomicExpr::AO__atomic_fetch_nand:
798 case AtomicExpr::AO__c11_atomic_fetch_sub:
799 case AtomicExpr::AO__atomic_fetch_sub:
800 case AtomicExpr::AO__c11_atomic_fetch_xor:
801 case AtomicExpr::AO__atomic_fetch_xor:
802 case AtomicExpr::AO__atomic_add_fetch:
803 case AtomicExpr::AO__atomic_and_fetch:
804 case AtomicExpr::AO__atomic_nand_fetch:
805 case AtomicExpr::AO__atomic_or_fetch:
806 case AtomicExpr::AO__atomic_sub_fetch:
807 case AtomicExpr::AO__atomic_xor_fetch:
808 // For these, only library calls for certain sizes exist.
809 UseOptimizedLibcall = true;
810 break;
811
812 case AtomicExpr::AO__c11_atomic_load:
813 case AtomicExpr::AO__c11_atomic_store:
814 case AtomicExpr::AO__c11_atomic_exchange:
815 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
816 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
817 case AtomicExpr::AO__atomic_load_n:
818 case AtomicExpr::AO__atomic_load:
819 case AtomicExpr::AO__atomic_store_n:
820 case AtomicExpr::AO__atomic_store:
821 case AtomicExpr::AO__atomic_exchange_n:
822 case AtomicExpr::AO__atomic_exchange:
823 case AtomicExpr::AO__atomic_compare_exchange_n:
824 case AtomicExpr::AO__atomic_compare_exchange:
825 // Only use optimized library calls for sizes for which they exist.
826 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
827 UseOptimizedLibcall = true;
828 break;
829 }
830
831 CallArgList Args;
832 if (!UseOptimizedLibcall) {
833 // For non-optimized library calls, the size is the first parameter
834 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
835 getContext().getSizeType());
836 }
837 // Atomic address is the first or second parameter
838 Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
839
840 std::string LibCallName;
841 QualType LoweredMemTy =
842 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
843 QualType RetTy;
844 bool HaveRetTy = false;
845 switch (E->getOp()) {
846 case AtomicExpr::AO__c11_atomic_init:
847 llvm_unreachable("Already handled!");
848
849 // There is only one libcall for compare an exchange, because there is no
850 // optimisation benefit possible from a libcall version of a weak compare
851 // and exchange.
852 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
853 // void *desired, int success, int failure)
854 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
855 // int success, int failure)
856 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
857 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
858 case AtomicExpr::AO__atomic_compare_exchange:
859 case AtomicExpr::AO__atomic_compare_exchange_n:
860 LibCallName = "__atomic_compare_exchange";
861 RetTy = getContext().BoolTy;
862 HaveRetTy = true;
863 Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
864 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
865 E->getExprLoc(), sizeChars);
866 Args.add(RValue::get(Order), getContext().IntTy);
867 Order = OrderFail;
868 break;
869 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
870 // int order)
871 // T __atomic_exchange_N(T *mem, T val, int order)
872 case AtomicExpr::AO__c11_atomic_exchange:
873 case AtomicExpr::AO__atomic_exchange_n:
874 case AtomicExpr::AO__atomic_exchange:
875 LibCallName = "__atomic_exchange";
876 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
877 E->getExprLoc(), sizeChars);
878 break;
879 // void __atomic_store(size_t size, void *mem, void *val, int order)
880 // void __atomic_store_N(T *mem, T val, int order)
881 case AtomicExpr::AO__c11_atomic_store:
882 case AtomicExpr::AO__atomic_store:
883 case AtomicExpr::AO__atomic_store_n:
884 LibCallName = "__atomic_store";
885 RetTy = getContext().VoidTy;
886 HaveRetTy = true;
887 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
888 E->getExprLoc(), sizeChars);
889 break;
890 // void __atomic_load(size_t size, void *mem, void *return, int order)
891 // T __atomic_load_N(T *mem, int order)
892 case AtomicExpr::AO__c11_atomic_load:
893 case AtomicExpr::AO__atomic_load:
894 case AtomicExpr::AO__atomic_load_n:
895 LibCallName = "__atomic_load";
896 break;
897 // T __atomic_fetch_add_N(T *mem, T val, int order)
898 case AtomicExpr::AO__c11_atomic_fetch_add:
899 case AtomicExpr::AO__atomic_fetch_add:
900 LibCallName = "__atomic_fetch_add";
901 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
902 E->getExprLoc(), sizeChars);
903 break;
904 // T __atomic_fetch_and_N(T *mem, T val, int order)
905 case AtomicExpr::AO__c11_atomic_fetch_and:
906 case AtomicExpr::AO__atomic_fetch_and:
907 LibCallName = "__atomic_fetch_and";
908 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
909 E->getExprLoc(), sizeChars);
910 break;
911 // T __atomic_fetch_or_N(T *mem, T val, int order)
912 case AtomicExpr::AO__c11_atomic_fetch_or:
913 case AtomicExpr::AO__atomic_fetch_or:
914 LibCallName = "__atomic_fetch_or";
915 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
916 E->getExprLoc(), sizeChars);
917 break;
918 // T __atomic_fetch_sub_N(T *mem, T val, int order)
919 case AtomicExpr::AO__c11_atomic_fetch_sub:
920 case AtomicExpr::AO__atomic_fetch_sub:
921 LibCallName = "__atomic_fetch_sub";
922 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
923 E->getExprLoc(), sizeChars);
924 break;
925 // T __atomic_fetch_xor_N(T *mem, T val, int order)
926 case AtomicExpr::AO__c11_atomic_fetch_xor:
927 case AtomicExpr::AO__atomic_fetch_xor:
928 LibCallName = "__atomic_fetch_xor";
929 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
930 E->getExprLoc(), sizeChars);
931 break;
932 // T __atomic_fetch_nand_N(T *mem, T val, int order)
933 case AtomicExpr::AO__atomic_fetch_nand:
934 LibCallName = "__atomic_fetch_nand";
935 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
936 E->getExprLoc(), sizeChars);
937 break;
938
939 // T __atomic_add_fetch_N(T *mem, T val, int order)
940 case AtomicExpr::AO__atomic_add_fetch:
941 LibCallName = "__atomic_add_fetch";
942 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
943 E->getExprLoc(), sizeChars);
944 break;
945 // T __atomic_and_fetch_N(T *mem, T val, int order)
946 case AtomicExpr::AO__atomic_and_fetch:
947 LibCallName = "__atomic_and_fetch";
948 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
949 E->getExprLoc(), sizeChars);
950 break;
951 // T __atomic_or_fetch_N(T *mem, T val, int order)
952 case AtomicExpr::AO__atomic_or_fetch:
953 LibCallName = "__atomic_or_fetch";
954 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
955 E->getExprLoc(), sizeChars);
956 break;
957 // T __atomic_sub_fetch_N(T *mem, T val, int order)
958 case AtomicExpr::AO__atomic_sub_fetch:
959 LibCallName = "__atomic_sub_fetch";
960 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
961 E->getExprLoc(), sizeChars);
962 break;
963 // T __atomic_xor_fetch_N(T *mem, T val, int order)
964 case AtomicExpr::AO__atomic_xor_fetch:
965 LibCallName = "__atomic_xor_fetch";
966 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
967 E->getExprLoc(), sizeChars);
968 break;
969 // T __atomic_nand_fetch_N(T *mem, T val, int order)
970 case AtomicExpr::AO__atomic_nand_fetch:
971 LibCallName = "__atomic_nand_fetch";
972 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
973 E->getExprLoc(), sizeChars);
974 break;
975 }
976
977 // Optimized functions have the size in their name.
978 if (UseOptimizedLibcall)
979 LibCallName += "_" + llvm::utostr(Size);
980 // By default, assume we return a value of the atomic type.
981 if (!HaveRetTy) {
982 if (UseOptimizedLibcall) {
983 // Value is returned directly.
984 // The function returns an appropriately sized integer type.
985 RetTy = getContext().getIntTypeForBitwidth(
986 getContext().toBits(sizeChars), /*Signed=*/false);
987 } else {
988 // Value is returned through parameter before the order.
989 RetTy = getContext().VoidTy;
990 Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
991 }
992 }
993 // order is always the last parameter
994 Args.add(RValue::get(Order),
995 getContext().IntTy);
996
997 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
998 // The value is returned directly from the libcall.
999 if (HaveRetTy && !RetTy->isVoidType())
1000 return Res;
1001 // The value is returned via an explicit out param.
1002 if (RetTy->isVoidType())
1003 return RValue::get(nullptr);
1004 // The value is returned directly for optimized libcalls but the caller is
1005 // expected an out-param.
1006 if (UseOptimizedLibcall) {
1007 llvm::Value *ResVal = Res.getScalarVal();
1008 llvm::StoreInst *StoreDest = Builder.CreateStore(
1009 ResVal,
1010 Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
1011 StoreDest->setAlignment(Align);
1012 }
1013 return convertTempToRValue(Dest, RValTy, E->getExprLoc());
1014 }
1015
1016 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1017 E->getOp() == AtomicExpr::AO__atomic_store ||
1018 E->getOp() == AtomicExpr::AO__atomic_store_n;
1019 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1020 E->getOp() == AtomicExpr::AO__atomic_load ||
1021 E->getOp() == AtomicExpr::AO__atomic_load_n;
1022
1023 llvm::Type *ITy =
1024 llvm::IntegerType::get(getLLVMContext(), Size * 8);
1025 llvm::Value *OrigDest = GetDest();
1026 Ptr = Builder.CreateBitCast(
1027 Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
1028 if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
1029 if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
1030 if (Dest && !E->isCmpXChg())
1031 Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
1032
1033 if (isa<llvm::ConstantInt>(Order)) {
1034 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1035 switch (ord) {
1036 case AtomicExpr::AO_ABI_memory_order_relaxed:
1037 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1038 Size, Align, llvm::Monotonic);
1039 break;
1040 case AtomicExpr::AO_ABI_memory_order_consume:
1041 case AtomicExpr::AO_ABI_memory_order_acquire:
1042 if (IsStore)
1043 break; // Avoid crashing on code with undefined behavior
1044 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1045 Size, Align, llvm::Acquire);
1046 break;
1047 case AtomicExpr::AO_ABI_memory_order_release:
1048 if (IsLoad)
1049 break; // Avoid crashing on code with undefined behavior
1050 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1051 Size, Align, llvm::Release);
1052 break;
1053 case AtomicExpr::AO_ABI_memory_order_acq_rel:
1054 if (IsLoad || IsStore)
1055 break; // Avoid crashing on code with undefined behavior
1056 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1057 Size, Align, llvm::AcquireRelease);
1058 break;
1059 case AtomicExpr::AO_ABI_memory_order_seq_cst:
1060 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1061 Size, Align, llvm::SequentiallyConsistent);
1062 break;
1063 default: // invalid order
1064 // We should not ever get here normally, but it's hard to
1065 // enforce that in general.
1066 break;
1067 }
1068 if (RValTy->isVoidType())
1069 return RValue::get(nullptr);
1070 return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1071 }
1072
1073 // Long case, when Order isn't obviously constant.
1074
1075 // Create all the relevant BB's
1076 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1077 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1078 *SeqCstBB = nullptr;
1079 MonotonicBB = createBasicBlock("monotonic", CurFn);
1080 if (!IsStore)
1081 AcquireBB = createBasicBlock("acquire", CurFn);
1082 if (!IsLoad)
1083 ReleaseBB = createBasicBlock("release", CurFn);
1084 if (!IsLoad && !IsStore)
1085 AcqRelBB = createBasicBlock("acqrel", CurFn);
1086 SeqCstBB = createBasicBlock("seqcst", CurFn);
1087 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1088
1089 // Create the switch for the split
1090 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1091 // doesn't matter unless someone is crazy enough to use something that
1092 // doesn't fold to a constant for the ordering.
1093 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1094 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1095
1096 // Emit all the different atomics
1097 Builder.SetInsertPoint(MonotonicBB);
1098 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1099 Size, Align, llvm::Monotonic);
1100 Builder.CreateBr(ContBB);
1101 if (!IsStore) {
1102 Builder.SetInsertPoint(AcquireBB);
1103 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1104 Size, Align, llvm::Acquire);
1105 Builder.CreateBr(ContBB);
1106 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1107 AcquireBB);
1108 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1109 AcquireBB);
1110 }
1111 if (!IsLoad) {
1112 Builder.SetInsertPoint(ReleaseBB);
1113 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1114 Size, Align, llvm::Release);
1115 Builder.CreateBr(ContBB);
1116 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1117 ReleaseBB);
1118 }
1119 if (!IsLoad && !IsStore) {
1120 Builder.SetInsertPoint(AcqRelBB);
1121 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1122 Size, Align, llvm::AcquireRelease);
1123 Builder.CreateBr(ContBB);
1124 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1125 AcqRelBB);
1126 }
1127 Builder.SetInsertPoint(SeqCstBB);
1128 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1129 Size, Align, llvm::SequentiallyConsistent);
1130 Builder.CreateBr(ContBB);
1131 SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1132 SeqCstBB);
1133
1134 // Cleanup and return
1135 Builder.SetInsertPoint(ContBB);
1136 if (RValTy->isVoidType())
1137 return RValue::get(nullptr);
1138 return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
1139 }
1140
emitCastToAtomicIntPointer(llvm::Value * addr) const1141 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
1142 unsigned addrspace =
1143 cast<llvm::PointerType>(addr->getType())->getAddressSpace();
1144 llvm::IntegerType *ty =
1145 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1146 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1147 }
1148
convertTempToRValue(llvm::Value * addr,AggValueSlot resultSlot,SourceLocation loc,bool AsValue) const1149 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
1150 AggValueSlot resultSlot,
1151 SourceLocation loc, bool AsValue) const {
1152 if (LVal.isSimple()) {
1153 if (EvaluationKind == TEK_Aggregate)
1154 return resultSlot.asRValue();
1155
1156 // Drill into the padding structure if we have one.
1157 if (hasPadding())
1158 addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
1159
1160 // Otherwise, just convert the temporary to an r-value using the
1161 // normal conversion routine.
1162 return CGF.convertTempToRValue(addr, getValueType(), loc);
1163 }
1164 if (!AsValue)
1165 // Get RValue from temp memory as atomic for non-simple lvalues
1166 return RValue::get(
1167 CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
1168 if (LVal.isBitField())
1169 return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
1170 addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
1171 if (LVal.isVectorElt())
1172 return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
1173 LVal.getType(),
1174 LVal.getAlignment()),
1175 loc);
1176 assert(LVal.isExtVectorElt());
1177 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1178 addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
1179 }
1180
ConvertIntToValueOrAtomic(llvm::Value * IntVal,AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue) const1181 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1182 AggValueSlot ResultSlot,
1183 SourceLocation Loc,
1184 bool AsValue) const {
1185 // Try not to in some easy cases.
1186 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1187 if (getEvaluationKind() == TEK_Scalar &&
1188 (((!LVal.isBitField() ||
1189 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1190 !hasPadding()) ||
1191 !AsValue)) {
1192 auto *ValTy = AsValue
1193 ? CGF.ConvertTypeForMem(ValueTy)
1194 : getAtomicAddress()->getType()->getPointerElementType();
1195 if (ValTy->isIntegerTy()) {
1196 assert(IntVal->getType() == ValTy && "Different integer types.");
1197 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1198 } else if (ValTy->isPointerTy())
1199 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1200 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1201 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1202 }
1203
1204 // Create a temporary. This needs to be big enough to hold the
1205 // atomic integer.
1206 llvm::Value *Temp;
1207 bool TempIsVolatile = false;
1208 CharUnits TempAlignment;
1209 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1210 assert(!ResultSlot.isIgnored());
1211 Temp = ResultSlot.getAddr();
1212 TempAlignment = getValueAlignment();
1213 TempIsVolatile = ResultSlot.isVolatile();
1214 } else {
1215 Temp = CreateTempAlloca();
1216 TempAlignment = getAtomicAlignment();
1217 }
1218
1219 // Slam the integer into the temporary.
1220 llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
1221 CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
1222 ->setVolatile(TempIsVolatile);
1223
1224 return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
1225 }
1226
EmitAtomicLoadLibcall(llvm::Value * AddForLoaded,llvm::AtomicOrdering AO,bool)1227 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1228 llvm::AtomicOrdering AO, bool) {
1229 // void __atomic_load(size_t size, void *mem, void *return, int order);
1230 CallArgList Args;
1231 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1232 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1233 CGF.getContext().VoidPtrTy);
1234 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1235 CGF.getContext().VoidPtrTy);
1236 Args.add(RValue::get(
1237 llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1238 CGF.getContext().IntTy);
1239 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1240 }
1241
EmitAtomicLoadOp(llvm::AtomicOrdering AO,bool IsVolatile)1242 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1243 bool IsVolatile) {
1244 // Okay, we're doing this natively.
1245 llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1246 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1247 Load->setAtomic(AO);
1248
1249 // Other decoration.
1250 Load->setAlignment(getAtomicAlignment().getQuantity());
1251 if (IsVolatile)
1252 Load->setVolatile(true);
1253 if (LVal.getTBAAInfo())
1254 CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
1255 return Load;
1256 }
1257
1258 /// An LValue is a candidate for having its loads and stores be made atomic if
1259 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1260 /// performing such an operation can be performed without a libcall.
LValueIsSuitableForInlineAtomic(LValue LV)1261 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1262 AtomicInfo AI(*this, LV);
1263 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1264 // An atomic is inline if we don't need to use a libcall.
1265 bool AtomicIsInline = !AI.shouldUseLibcall();
1266 return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1267 }
1268
1269 /// An type is a candidate for having its loads and stores be made atomic if
1270 /// we are operating under /volatile:ms *and* we know the access is volatile and
1271 /// performing such an operation can be performed without a libcall.
typeIsSuitableForInlineAtomic(QualType Ty,bool IsVolatile) const1272 bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
1273 bool IsVolatile) const {
1274 // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1275 bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1276 getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1277 return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1278 }
1279
EmitAtomicLoad(LValue LV,SourceLocation SL,AggValueSlot Slot)1280 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1281 AggValueSlot Slot) {
1282 llvm::AtomicOrdering AO;
1283 bool IsVolatile = LV.isVolatileQualified();
1284 if (LV.getType()->isAtomicType()) {
1285 AO = llvm::SequentiallyConsistent;
1286 } else {
1287 AO = llvm::Acquire;
1288 IsVolatile = true;
1289 }
1290 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1291 }
1292
EmitAtomicLoad(AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue,llvm::AtomicOrdering AO,bool IsVolatile)1293 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1294 bool AsValue, llvm::AtomicOrdering AO,
1295 bool IsVolatile) {
1296 // Check whether we should use a library call.
1297 if (shouldUseLibcall()) {
1298 llvm::Value *TempAddr;
1299 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1300 assert(getEvaluationKind() == TEK_Aggregate);
1301 TempAddr = ResultSlot.getAddr();
1302 } else
1303 TempAddr = CreateTempAlloca();
1304
1305 EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
1306
1307 // Okay, turn that back into the original value or whole atomic (for
1308 // non-simple lvalues) type.
1309 return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1310 }
1311
1312 // Okay, we're doing this natively.
1313 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1314
1315 // If we're ignoring an aggregate return, don't do anything.
1316 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1317 return RValue::getAggregate(nullptr, false);
1318
1319 // Okay, turn that back into the original value or atomic (for non-simple
1320 // lvalues) type.
1321 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1322 }
1323
1324 /// Emit a load from an l-value of atomic type. Note that the r-value
1325 /// we produce is an r-value of the atomic *value* type.
EmitAtomicLoad(LValue src,SourceLocation loc,llvm::AtomicOrdering AO,bool IsVolatile,AggValueSlot resultSlot)1326 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1327 llvm::AtomicOrdering AO, bool IsVolatile,
1328 AggValueSlot resultSlot) {
1329 AtomicInfo Atomics(*this, src);
1330 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1331 IsVolatile);
1332 }
1333
1334 /// Copy an r-value into memory as part of storing to an atomic type.
1335 /// This needs to create a bit-pattern suitable for atomic operations.
emitCopyIntoMemory(RValue rvalue) const1336 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1337 assert(LVal.isSimple());
1338 // If we have an r-value, the rvalue should be of the atomic type,
1339 // which means that the caller is responsible for having zeroed
1340 // any padding. Just do an aggregate copy of that type.
1341 if (rvalue.isAggregate()) {
1342 CGF.EmitAggregateCopy(getAtomicAddress(),
1343 rvalue.getAggregateAddr(),
1344 getAtomicType(),
1345 (rvalue.isVolatileQualified()
1346 || LVal.isVolatileQualified()),
1347 LVal.getAlignment());
1348 return;
1349 }
1350
1351 // Okay, otherwise we're copying stuff.
1352
1353 // Zero out the buffer if necessary.
1354 emitMemSetZeroIfNecessary();
1355
1356 // Drill past the padding if present.
1357 LValue TempLVal = projectValue();
1358
1359 // Okay, store the rvalue in.
1360 if (rvalue.isScalar()) {
1361 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1362 } else {
1363 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1364 }
1365 }
1366
1367
1368 /// Materialize an r-value into memory for the purposes of storing it
1369 /// to an atomic type.
materializeRValue(RValue rvalue) const1370 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
1371 // Aggregate r-values are already in memory, and EmitAtomicStore
1372 // requires them to be values of the atomic type.
1373 if (rvalue.isAggregate())
1374 return rvalue.getAggregateAddr();
1375
1376 // Otherwise, make a temporary and materialize into it.
1377 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
1378 getAtomicAlignment());
1379 AtomicInfo Atomics(CGF, TempLV);
1380 Atomics.emitCopyIntoMemory(rvalue);
1381 return TempLV.getAddress();
1382 }
1383
convertRValueToInt(RValue RVal) const1384 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1385 // If we've got a scalar value of the right size, try to avoid going
1386 // through memory.
1387 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1388 llvm::Value *Value = RVal.getScalarVal();
1389 if (isa<llvm::IntegerType>(Value->getType()))
1390 return CGF.EmitToMemory(Value, ValueTy);
1391 else {
1392 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1393 CGF.getLLVMContext(),
1394 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1395 if (isa<llvm::PointerType>(Value->getType()))
1396 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1397 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1398 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1399 }
1400 }
1401 // Otherwise, we need to go through memory.
1402 // Put the r-value in memory.
1403 llvm::Value *Addr = materializeRValue(RVal);
1404
1405 // Cast the temporary to the atomic int type and pull a value out.
1406 Addr = emitCastToAtomicIntPointer(Addr);
1407 return CGF.Builder.CreateAlignedLoad(Addr,
1408 getAtomicAlignment().getQuantity());
1409 }
1410
EmitAtomicCompareExchangeOp(llvm::Value * ExpectedVal,llvm::Value * DesiredVal,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1411 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1412 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1413 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1414 // Do the atomic store.
1415 auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
1416 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
1417 Success, Failure);
1418 // Other decoration.
1419 Inst->setVolatile(LVal.isVolatileQualified());
1420 Inst->setWeak(IsWeak);
1421
1422 // Okay, turn that back into the original value type.
1423 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1424 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1425 return std::make_pair(PreviousVal, SuccessFailureVal);
1426 }
1427
1428 llvm::Value *
EmitAtomicCompareExchangeLibcall(llvm::Value * ExpectedAddr,llvm::Value * DesiredAddr,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure)1429 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1430 llvm::Value *DesiredAddr,
1431 llvm::AtomicOrdering Success,
1432 llvm::AtomicOrdering Failure) {
1433 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1434 // void *desired, int success, int failure);
1435 CallArgList Args;
1436 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1437 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
1438 CGF.getContext().VoidPtrTy);
1439 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1440 CGF.getContext().VoidPtrTy);
1441 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1442 CGF.getContext().VoidPtrTy);
1443 Args.add(RValue::get(llvm::ConstantInt::get(
1444 CGF.IntTy, translateAtomicOrdering(Success))),
1445 CGF.getContext().IntTy);
1446 Args.add(RValue::get(llvm::ConstantInt::get(
1447 CGF.IntTy, translateAtomicOrdering(Failure))),
1448 CGF.getContext().IntTy);
1449 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1450 CGF.getContext().BoolTy, Args);
1451
1452 return SuccessFailureRVal.getScalarVal();
1453 }
1454
EmitAtomicCompareExchange(RValue Expected,RValue Desired,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1455 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1456 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1457 llvm::AtomicOrdering Failure, bool IsWeak) {
1458 if (Failure >= Success)
1459 // Don't assert on undefined behavior.
1460 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1461
1462 // Check whether we should use a library call.
1463 if (shouldUseLibcall()) {
1464 // Produce a source address.
1465 auto *ExpectedAddr = materializeRValue(Expected);
1466 auto *DesiredAddr = materializeRValue(Desired);
1467 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr,
1468 Success, Failure);
1469 return std::make_pair(
1470 convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1471 SourceLocation(), /*AsValue=*/false),
1472 Res);
1473 }
1474
1475 // If we've got a scalar value of the right size, try to avoid going
1476 // through memory.
1477 auto *ExpectedVal = convertRValueToInt(Expected);
1478 auto *DesiredVal = convertRValueToInt(Desired);
1479 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1480 Failure, IsWeak);
1481 return std::make_pair(
1482 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1483 SourceLocation(), /*AsValue=*/false),
1484 Res.second);
1485 }
1486
1487 static void
EmitAtomicUpdateValue(CodeGenFunction & CGF,AtomicInfo & Atomics,RValue OldRVal,const llvm::function_ref<RValue (RValue)> & UpdateOp,llvm::Value * DesiredAddr)1488 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1489 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1490 llvm::Value *DesiredAddr) {
1491 llvm::Value *Ptr = nullptr;
1492 LValue UpdateLVal;
1493 RValue UpRVal;
1494 LValue AtomicLVal = Atomics.getAtomicLValue();
1495 LValue DesiredLVal;
1496 if (AtomicLVal.isSimple()) {
1497 UpRVal = OldRVal;
1498 DesiredLVal =
1499 LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(),
1500 AtomicLVal.getAlignment(), CGF.CGM.getContext());
1501 } else {
1502 // Build new lvalue for temp address
1503 Ptr = Atomics.materializeRValue(OldRVal);
1504 if (AtomicLVal.isBitField()) {
1505 UpdateLVal =
1506 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1507 AtomicLVal.getType(), AtomicLVal.getAlignment());
1508 DesiredLVal =
1509 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1510 AtomicLVal.getType(), AtomicLVal.getAlignment());
1511 } else if (AtomicLVal.isVectorElt()) {
1512 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1513 AtomicLVal.getType(),
1514 AtomicLVal.getAlignment());
1515 DesiredLVal = LValue::MakeVectorElt(
1516 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1517 AtomicLVal.getAlignment());
1518 } else {
1519 assert(AtomicLVal.isExtVectorElt());
1520 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1521 AtomicLVal.getType(),
1522 AtomicLVal.getAlignment());
1523 DesiredLVal = LValue::MakeExtVectorElt(
1524 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1525 AtomicLVal.getAlignment());
1526 }
1527 UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1528 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1529 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1530 }
1531 // Store new value in the corresponding memory area
1532 RValue NewRVal = UpdateOp(UpRVal);
1533 if (NewRVal.isScalar()) {
1534 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1535 } else {
1536 assert(NewRVal.isComplex());
1537 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1538 /*isInit=*/false);
1539 }
1540 }
1541
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1542 void AtomicInfo::EmitAtomicUpdateLibcall(
1543 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1544 bool IsVolatile) {
1545 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1546
1547 llvm::Value *ExpectedAddr = CreateTempAlloca();
1548
1549 EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
1550 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1551 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1552 CGF.EmitBlock(ContBB);
1553 auto *DesiredAddr = CreateTempAlloca();
1554 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1555 requiresMemSetZero(
1556 getAtomicAddress()->getType()->getPointerElementType())) {
1557 auto *OldVal = CGF.Builder.CreateAlignedLoad(
1558 ExpectedAddr, getAtomicAlignment().getQuantity());
1559 CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
1560 getAtomicAlignment().getQuantity());
1561 }
1562 auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1563 SourceLocation(), /*AsValue=*/false);
1564 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1565 auto *Res =
1566 EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
1567 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1568 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1569 }
1570
EmitAtomicUpdateOp(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1571 void AtomicInfo::EmitAtomicUpdateOp(
1572 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1573 bool IsVolatile) {
1574 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1575
1576 // Do the atomic load.
1577 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1578 // For non-simple lvalues perform compare-and-swap procedure.
1579 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1580 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1581 auto *CurBB = CGF.Builder.GetInsertBlock();
1582 CGF.EmitBlock(ContBB);
1583 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1584 /*NumReservedValues=*/2);
1585 PHI->addIncoming(OldVal, CurBB);
1586 auto *NewAtomicAddr = CreateTempAlloca();
1587 auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1588 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1589 requiresMemSetZero(
1590 getAtomicAddress()->getType()->getPointerElementType())) {
1591 CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
1592 getAtomicAlignment().getQuantity());
1593 }
1594 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1595 SourceLocation(), /*AsValue=*/false);
1596 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1597 auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
1598 NewAtomicIntAddr, getAtomicAlignment().getQuantity());
1599 // Try to write new value using cmpxchg operation
1600 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1601 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1602 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1603 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1604 }
1605
EmitAtomicUpdateValue(CodeGenFunction & CGF,AtomicInfo & Atomics,RValue UpdateRVal,llvm::Value * DesiredAddr)1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1607 RValue UpdateRVal, llvm::Value *DesiredAddr) {
1608 LValue AtomicLVal = Atomics.getAtomicLValue();
1609 LValue DesiredLVal;
1610 // Build new lvalue for temp address
1611 if (AtomicLVal.isBitField()) {
1612 DesiredLVal =
1613 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1614 AtomicLVal.getType(), AtomicLVal.getAlignment());
1615 } else if (AtomicLVal.isVectorElt()) {
1616 DesiredLVal =
1617 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1618 AtomicLVal.getType(), AtomicLVal.getAlignment());
1619 } else {
1620 assert(AtomicLVal.isExtVectorElt());
1621 DesiredLVal = LValue::MakeExtVectorElt(
1622 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1623 AtomicLVal.getAlignment());
1624 }
1625 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1626 // Store new value in the corresponding memory area
1627 assert(UpdateRVal.isScalar());
1628 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1629 }
1630
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1631 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1632 RValue UpdateRVal, bool IsVolatile) {
1633 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1634
1635 llvm::Value *ExpectedAddr = CreateTempAlloca();
1636
1637 EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
1638 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1639 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1640 CGF.EmitBlock(ContBB);
1641 auto *DesiredAddr = CreateTempAlloca();
1642 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1643 requiresMemSetZero(
1644 getAtomicAddress()->getType()->getPointerElementType())) {
1645 auto *OldVal = CGF.Builder.CreateAlignedLoad(
1646 ExpectedAddr, getAtomicAlignment().getQuantity());
1647 CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
1648 getAtomicAlignment().getQuantity());
1649 }
1650 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1651 auto *Res =
1652 EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
1653 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1654 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1655 }
1656
EmitAtomicUpdateOp(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1657 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1658 bool IsVolatile) {
1659 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1660
1661 // Do the atomic load.
1662 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1663 // For non-simple lvalues perform compare-and-swap procedure.
1664 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1665 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1666 auto *CurBB = CGF.Builder.GetInsertBlock();
1667 CGF.EmitBlock(ContBB);
1668 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1669 /*NumReservedValues=*/2);
1670 PHI->addIncoming(OldVal, CurBB);
1671 auto *NewAtomicAddr = CreateTempAlloca();
1672 auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1673 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1674 requiresMemSetZero(
1675 getAtomicAddress()->getType()->getPointerElementType())) {
1676 CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
1677 getAtomicAlignment().getQuantity());
1678 }
1679 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1680 auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
1681 NewAtomicIntAddr, getAtomicAlignment().getQuantity());
1682 // Try to write new value using cmpxchg operation
1683 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1684 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1685 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1686 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1687 }
1688
EmitAtomicUpdate(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1689 void AtomicInfo::EmitAtomicUpdate(
1690 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1691 bool IsVolatile) {
1692 if (shouldUseLibcall()) {
1693 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1694 } else {
1695 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1696 }
1697 }
1698
EmitAtomicUpdate(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1699 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1700 bool IsVolatile) {
1701 if (shouldUseLibcall()) {
1702 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1703 } else {
1704 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1705 }
1706 }
1707
EmitAtomicStore(RValue rvalue,LValue lvalue,bool isInit)1708 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1709 bool isInit) {
1710 bool IsVolatile = lvalue.isVolatileQualified();
1711 llvm::AtomicOrdering AO;
1712 if (lvalue.getType()->isAtomicType()) {
1713 AO = llvm::SequentiallyConsistent;
1714 } else {
1715 AO = llvm::Release;
1716 IsVolatile = true;
1717 }
1718 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1719 }
1720
1721 /// Emit a store to an l-value of atomic type.
1722 ///
1723 /// Note that the r-value is expected to be an r-value *of the atomic
1724 /// type*; this means that for aggregate r-values, it should include
1725 /// storage for any padding that was necessary.
EmitAtomicStore(RValue rvalue,LValue dest,llvm::AtomicOrdering AO,bool IsVolatile,bool isInit)1726 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1727 llvm::AtomicOrdering AO, bool IsVolatile,
1728 bool isInit) {
1729 // If this is an aggregate r-value, it should agree in type except
1730 // maybe for address-space qualification.
1731 assert(!rvalue.isAggregate() ||
1732 rvalue.getAggregateAddr()->getType()->getPointerElementType()
1733 == dest.getAddress()->getType()->getPointerElementType());
1734
1735 AtomicInfo atomics(*this, dest);
1736 LValue LVal = atomics.getAtomicLValue();
1737
1738 // If this is an initialization, just put the value there normally.
1739 if (LVal.isSimple()) {
1740 if (isInit) {
1741 atomics.emitCopyIntoMemory(rvalue);
1742 return;
1743 }
1744
1745 // Check whether we should use a library call.
1746 if (atomics.shouldUseLibcall()) {
1747 // Produce a source address.
1748 llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
1749
1750 // void __atomic_store(size_t size, void *mem, void *val, int order)
1751 CallArgList args;
1752 args.add(RValue::get(atomics.getAtomicSizeValue()),
1753 getContext().getSizeType());
1754 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
1755 getContext().VoidPtrTy);
1756 args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
1757 args.add(RValue::get(llvm::ConstantInt::get(
1758 IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1759 getContext().IntTy);
1760 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1761 return;
1762 }
1763
1764 // Okay, we're doing this natively.
1765 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1766
1767 // Do the atomic store.
1768 llvm::Value *addr =
1769 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1770 intValue = Builder.CreateIntCast(
1771 intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
1772 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1773
1774 // Initializations don't need to be atomic.
1775 if (!isInit)
1776 store->setAtomic(AO);
1777
1778 // Other decoration.
1779 store->setAlignment(dest.getAlignment().getQuantity());
1780 if (IsVolatile)
1781 store->setVolatile(true);
1782 if (dest.getTBAAInfo())
1783 CGM.DecorateInstruction(store, dest.getTBAAInfo());
1784 return;
1785 }
1786
1787 // Emit simple atomic update operation.
1788 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1789 }
1790
1791 /// Emit a compare-and-exchange op for atomic type.
1792 ///
EmitAtomicCompareExchange(LValue Obj,RValue Expected,RValue Desired,SourceLocation Loc,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak,AggValueSlot Slot)1793 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1794 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1795 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1796 AggValueSlot Slot) {
1797 // If this is an aggregate r-value, it should agree in type except
1798 // maybe for address-space qualification.
1799 assert(!Expected.isAggregate() ||
1800 Expected.getAggregateAddr()->getType()->getPointerElementType() ==
1801 Obj.getAddress()->getType()->getPointerElementType());
1802 assert(!Desired.isAggregate() ||
1803 Desired.getAggregateAddr()->getType()->getPointerElementType() ==
1804 Obj.getAddress()->getType()->getPointerElementType());
1805 AtomicInfo Atomics(*this, Obj);
1806
1807 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1808 IsWeak);
1809 }
1810
EmitAtomicUpdate(LValue LVal,llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1811 void CodeGenFunction::EmitAtomicUpdate(
1812 LValue LVal, llvm::AtomicOrdering AO,
1813 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1814 AtomicInfo Atomics(*this, LVal);
1815 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1816 }
1817
EmitAtomicInit(Expr * init,LValue dest)1818 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1819 AtomicInfo atomics(*this, dest);
1820
1821 switch (atomics.getEvaluationKind()) {
1822 case TEK_Scalar: {
1823 llvm::Value *value = EmitScalarExpr(init);
1824 atomics.emitCopyIntoMemory(RValue::get(value));
1825 return;
1826 }
1827
1828 case TEK_Complex: {
1829 ComplexPairTy value = EmitComplexExpr(init);
1830 atomics.emitCopyIntoMemory(RValue::getComplex(value));
1831 return;
1832 }
1833
1834 case TEK_Aggregate: {
1835 // Fix up the destination if the initializer isn't an expression
1836 // of atomic type.
1837 bool Zeroed = false;
1838 if (!init->getType()->isAtomicType()) {
1839 Zeroed = atomics.emitMemSetZeroIfNecessary();
1840 dest = atomics.projectValue();
1841 }
1842
1843 // Evaluate the expression directly into the destination.
1844 AggValueSlot slot = AggValueSlot::forLValue(dest,
1845 AggValueSlot::IsNotDestructed,
1846 AggValueSlot::DoesNotNeedGCBarriers,
1847 AggValueSlot::IsNotAliased,
1848 Zeroed ? AggValueSlot::IsZeroed :
1849 AggValueSlot::IsNotZeroed);
1850
1851 EmitAggExpr(init, slot);
1852 return;
1853 }
1854 }
1855 llvm_unreachable("bad evaluation kind");
1856 }
1857