1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|*                     The LLVM Compiler Infrastructure
4|*
5|* This file is distributed under the University of Illinois Open Source
6|* License. See LICENSE.TXT for details.
7|*
8\*===----------------------------------------------------------------------===*/
9/*
10 * This is the master file that defines all the data structure, signature,
11 * constant literals that are shared across profiling runtime library,
12 * compiler (instrumentation), and host tools (reader/writer). The entities
13 * defined in this file affect the profile runtime ABI, the raw profile format,
14 * or both.
15 *
16 * The file has two identical copies. The master copy lives in LLVM and
17 * the other one  sits in compiler-rt/lib/profile directory. To make changes
18 * in this file, first modify the master copy and copy it over to compiler-rt.
19 * Testing of any change in this file can start only after the two copies are
20 * synced up.
21 *
22 * The first part of the file includes macros that defines types, names, and
23 * initializers for the member fields of the core data structures. The field
24 * declarations for one structure is enabled by defining the field activation
25 * macro associated with that structure. Only one field activation record
26 * can be defined at one time and the rest definitions will be filtered out by
27 * the preprocessor.
28 *
29 * Examples of how the template is used to instantiate structure definition:
30 * 1. To declare a structure:
31 *
32 * struct ProfData {
33 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
34 *    Type Name;
35 * #include "llvm/ProfileData/InstrProfData.inc"
36 * };
37 *
38 * 2. To construct LLVM type arrays for the struct type:
39 *
40 * Type *DataTypes[] = {
41 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
42 *   LLVMType,
43 * #include "llvm/ProfileData/InstrProfData.inc"
44 * };
45 *
46 * 4. To construct constant array for the initializers:
47 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
48 *   Initializer,
49 * Constant *ConstantVals[] = {
50 * #include "llvm/ProfileData/InstrProfData.inc"
51 * };
52 *
53 *
54 * The second part of the file includes definitions all other entities that
55 * are related to runtime ABI and format. When no field activation macro is
56 * defined, this file can be included to introduce the definitions.
57 *
58\*===----------------------------------------------------------------------===*/
59
60/* INSTR_PROF_DATA start. */
61/* Definition of member fields of the per-function control structure. */
62#ifndef INSTR_PROF_DATA
63#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
64#else
65#define INSTR_PROF_DATA_DEFINED
66#endif
67
68INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
69                ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
70                NamePtr->getType()->getPointerElementType()->getArrayNumElements()))
71INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
72                ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
73INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
74                ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
75                Inc->getHash()->getZExtValue()))
76INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \
77                ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx)))
78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
79                ConstantExpr::getBitCast(CounterPtr, \
80                llvm::Type::getInt64PtrTy(Ctx)))
81INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
82                FunctionAddr)
83INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
84                ConstantPointerNull::get(Int8PtrTy))
85INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
86                ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
87#undef INSTR_PROF_DATA
88/* INSTR_PROF_DATA end. */
89
90/* INSTR_PROF_RAW_HEADER  start */
91/* Definition of member fields of the raw profile header data structure. */
92#ifndef INSTR_PROF_RAW_HEADER
93#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
94#else
95#define INSTR_PROF_DATA_DEFINED
96#endif
97INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
98INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
99INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
100INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
101INSTR_PROF_RAW_HEADER(uint64_t, NamesSize,  NamesSize)
102INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
103INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
104INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
105INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize)
106INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin)
107#undef INSTR_PROF_RAW_HEADER
108/* INSTR_PROF_RAW_HEADER  end */
109
110/* VALUE_PROF_FUNC_PARAM start */
111/* Definition of parameter types of the runtime API used to do value profiling
112 * for a given value site.
113 */
114#ifndef VALUE_PROF_FUNC_PARAM
115#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
116#define INSTR_PROF_COMMA
117#else
118#define INSTR_PROF_DATA_DEFINED
119#define INSTR_PROF_COMMA ,
120#endif
121VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
122                      INSTR_PROF_COMMA
123VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
124VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
125#undef VALUE_PROF_FUNC_PARAM
126#undef INSTR_PROF_COMMA
127/* VALUE_PROF_FUNC_PARAM end */
128
129/* VALUE_PROF_KIND start */
130#ifndef VALUE_PROF_KIND
131#define VALUE_PROF_KIND(Enumerator, Value)
132#else
133#define INSTR_PROF_DATA_DEFINED
134#endif
135VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0)
136/* These two kinds must be the last to be
137 * declared. This is to make sure the string
138 * array created with the template can be
139 * indexed with the kind value.
140 */
141VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget)
142VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget)
143
144#undef VALUE_PROF_KIND
145/* VALUE_PROF_KIND end */
146
147/* COVMAP_FUNC_RECORD start */
148/* Definition of member fields of the function record structure in coverage
149 * map.
150 */
151#ifndef COVMAP_FUNC_RECORD
152#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
153#else
154#define INSTR_PROF_DATA_DEFINED
155#endif
156COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
157                   NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
158                   llvm::Type::getInt8PtrTy(Ctx)))
159COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
160                   llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
161                   NameValue.size()))
162COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
163                   llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
164                   CoverageMapping.size()))
165COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
166                   llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash))
167#undef COVMAP_FUNC_RECORD
168/* COVMAP_FUNC_RECORD end.  */
169
170/* COVMAP_HEADER start */
171/* Definition of member fields of coverage map header.
172 */
173#ifndef COVMAP_HEADER
174#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
175#else
176#define INSTR_PROF_DATA_DEFINED
177#endif
178COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
179              llvm::ConstantInt::get(Int32Ty,  FunctionRecords.size()))
180COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
181              llvm::ConstantInt::get(Int32Ty, FilenamesSize))
182COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
183              llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
184COVMAP_HEADER(uint32_t, Int32Ty, Version, \
185              llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
186#undef COVMAP_HEADER
187/* COVMAP_HEADER end.  */
188
189
190#ifdef INSTR_PROF_VALUE_PROF_DATA
191#define INSTR_PROF_DATA_DEFINED
192
193#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
194/*!
195 * This is the header of the data structure that defines the on-disk
196 * layout of the value profile data of a particular kind for one function.
197 */
198typedef struct ValueProfRecord {
199  /* The kind of the value profile record. */
200  uint32_t Kind;
201  /*
202   * The number of value profile sites. It is guaranteed to be non-zero;
203   * otherwise the record for this kind won't be emitted.
204   */
205  uint32_t NumValueSites;
206  /*
207   * The first element of the array that stores the number of profiled
208   * values for each value site. The size of the array is NumValueSites.
209   * Since NumValueSites is greater than zero, there is at least one
210   * element in the array.
211   */
212  uint8_t SiteCountArray[1];
213
214  /*
215   * The fake declaration is for documentation purpose only.
216   * Align the start of next field to be on 8 byte boundaries.
217  uint8_t Padding[X];
218   */
219
220  /* The array of value profile data. The size of the array is the sum
221   * of all elements in SiteCountArray[].
222  InstrProfValueData ValueData[];
223   */
224
225#ifdef __cplusplus
226  /*!
227   * \brief Return the number of value sites.
228   */
229  uint32_t getNumValueSites() const { return NumValueSites; }
230  /*!
231   * \brief Read data from this record and save it to Record.
232   */
233  void deserializeTo(InstrProfRecord &Record,
234                     InstrProfRecord::ValueMapType *VMap);
235  /*
236   * In-place byte swap:
237   * Do byte swap for this instance. \c Old is the original order before
238   * the swap, and \c New is the New byte order.
239   */
240  void swapBytes(support::endianness Old, support::endianness New);
241#endif
242} ValueProfRecord;
243
244/*!
245 * Per-function header/control data structure for value profiling
246 * data in indexed format.
247 */
248typedef struct ValueProfData {
249  /*
250   * Total size in bytes including this field. It must be a multiple
251   * of sizeof(uint64_t).
252   */
253  uint32_t TotalSize;
254  /*
255   *The number of value profile kinds that has value profile data.
256   * In this implementation, a value profile kind is considered to
257   * have profile data if the number of value profile sites for the
258   * kind is not zero. More aggressively, the implementation can
259   * choose to check the actual data value: if none of the value sites
260   * has any profiled values, the kind can be skipped.
261   */
262  uint32_t NumValueKinds;
263
264  /*
265   * Following are a sequence of variable length records. The prefix/header
266   * of each record is defined by ValueProfRecord type. The number of
267   * records is NumValueKinds.
268   * ValueProfRecord Record_1;
269   * ValueProfRecord Record_N;
270   */
271
272#if __cplusplus
273  /*!
274   * Return the total size in bytes of the on-disk value profile data
275   * given the data stored in Record.
276   */
277  static uint32_t getSize(const InstrProfRecord &Record);
278  /*!
279   * Return a pointer to \c ValueProfData instance ready to be streamed.
280   */
281  static std::unique_ptr<ValueProfData>
282  serializeFrom(const InstrProfRecord &Record);
283  /*!
284   * Check the integrity of the record. Return the error code when
285   * an error is detected, otherwise return instrprof_error::success.
286   */
287  instrprof_error checkIntegrity();
288  /*!
289   * Return a pointer to \c ValueProfileData instance ready to be read.
290   * All data in the instance are properly byte swapped. The input
291   * data is assumed to be in little endian order.
292   */
293  static ErrorOr<std::unique_ptr<ValueProfData>>
294  getValueProfData(const unsigned char *SrcBuffer,
295                   const unsigned char *const SrcBufferEnd,
296                   support::endianness SrcDataEndianness);
297  /*!
298   * Swap byte order from \c Endianness order to host byte order.
299   */
300  void swapBytesToHost(support::endianness Endianness);
301  /*!
302   * Swap byte order from host byte order to \c Endianness order.
303   */
304  void swapBytesFromHost(support::endianness Endianness);
305  /*!
306   * Return the total size of \c ValueProfileData.
307   */
308  uint32_t getSize() const { return TotalSize; }
309  /*!
310   * Read data from this data and save it to \c Record.
311   */
312  void deserializeTo(InstrProfRecord &Record,
313                     InstrProfRecord::ValueMapType *VMap);
314  void operator delete(void *ptr) { ::operator delete(ptr); }
315#endif
316} ValueProfData;
317
318/*
319 * The closure is designed to abstact away two types of value profile data:
320 * - InstrProfRecord which is the primary data structure used to
321 *   represent profile data in host tools (reader, writer, and profile-use)
322 * - value profile runtime data structure suitable to be used by C
323 *   runtime library.
324 *
325 * Both sources of data need to serialize to disk/memory-buffer in common
326 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
327 * writer to share the same format and code with indexed profile writer.
328 *
329 * For documentation of the member methods below, refer to corresponding methods
330 * in class InstrProfRecord.
331 */
332typedef struct ValueProfRecordClosure {
333  const void *Record;
334  uint32_t (*GetNumValueKinds)(const void *Record);
335  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
336  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
337  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
338
339  /*
340   * After extracting the value profile data from the value profile record,
341   * this method is used to map the in-memory value to on-disk value. If
342   * the method is null, value will be written out untranslated.
343   */
344  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
345  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
346                          uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t));
347  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
348} ValueProfRecordClosure;
349
350/*
351 * A wrapper struct that represents value profile runtime data.
352 * Like InstrProfRecord class which is used by profiling host tools,
353 * ValueProfRuntimeRecord also implements the abstract intefaces defined in
354 * ValueProfRecordClosure so that the runtime data can be serialized using
355 * shared C implementation. In this structure, NumValueSites and Nodes
356 * members are the primary fields while other fields hold the derived
357 * information for fast implementation of closure interfaces.
358 */
359typedef struct ValueProfRuntimeRecord {
360  /* Number of sites for each value profile kind.  */
361  const uint16_t *NumValueSites;
362  /* An array of linked-list headers. The size of of the array is the
363   * total number of value profile sites : sum(NumValueSites[*])). Each
364   * linked-list stores the values profiled for a value profile site. */
365  ValueProfNode **Nodes;
366
367  /* Total number of value profile kinds which have at least one
368   *  value profile sites. */
369  uint32_t NumValueKinds;
370  /* An array recording the number of values tracked at each site.
371   * The size of the array is TotalNumValueSites. */
372  uint8_t *SiteCountArray[IPVK_Last + 1];
373  ValueProfNode **NodesKind[IPVK_Last + 1];
374} ValueProfRuntimeRecord;
375
376/* Forward declarations of C interfaces.  */
377int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
378                                     const uint16_t *NumValueSites,
379                                     ValueProfNode **Nodes);
380void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord);
381uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record);
382ValueProfData *
383serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
384                             ValueProfData *Dst);
385uint32_t getNumValueKindsRT(const void *R);
386
387#undef INSTR_PROF_VALUE_PROF_DATA
388#endif  /* INSTR_PROF_VALUE_PROF_DATA */
389
390
391#ifdef INSTR_PROF_COMMON_API_IMPL
392#define INSTR_PROF_DATA_DEFINED
393#ifdef __cplusplus
394#define INSTR_PROF_INLINE inline
395#define INSTR_PROF_NULLPTR nullptr
396#else
397#define INSTR_PROF_INLINE
398#define INSTR_PROF_NULLPTR NULL
399#endif
400
401#ifndef offsetof
402#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
403#endif
404
405/*!
406 * \brief Return the \c ValueProfRecord header size including the
407 * padding bytes.
408 */
409INSTR_PROF_INLINE
410uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
411  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
412                  sizeof(uint8_t) * NumValueSites;
413  /* Round the size to multiple of 8 bytes. */
414  Size = (Size + 7) & ~7;
415  return Size;
416}
417
418/*!
419 * \brief Return the total size of the value profile record including the
420 * header and the value data.
421 */
422INSTR_PROF_INLINE
423uint32_t getValueProfRecordSize(uint32_t NumValueSites,
424                                uint32_t NumValueData) {
425  return getValueProfRecordHeaderSize(NumValueSites) +
426         sizeof(InstrProfValueData) * NumValueData;
427}
428
429/*!
430 * \brief Return the pointer to the start of value data array.
431 */
432INSTR_PROF_INLINE
433InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
434  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
435                                                   This->NumValueSites));
436}
437
438/*!
439 * \brief Return the total number of value data for \c This record.
440 */
441INSTR_PROF_INLINE
442uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
443  uint32_t NumValueData = 0;
444  uint32_t I;
445  for (I = 0; I < This->NumValueSites; I++)
446    NumValueData += This->SiteCountArray[I];
447  return NumValueData;
448}
449
450/*!
451 * \brief Use this method to advance to the next \c This \c ValueProfRecord.
452 */
453INSTR_PROF_INLINE
454ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
455  uint32_t NumValueData = getValueProfRecordNumValueData(This);
456  return (ValueProfRecord *)((char *)This +
457                             getValueProfRecordSize(This->NumValueSites,
458                                                    NumValueData));
459}
460
461/*!
462 * \brief Return the first \c ValueProfRecord instance.
463 */
464INSTR_PROF_INLINE
465ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
466  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
467}
468
469/* Closure based interfaces.  */
470
471/*!
472 * Return the total size in bytes of the on-disk value profile data
473 * given the data stored in Record.
474 */
475uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) {
476  uint32_t Kind;
477  uint32_t TotalSize = sizeof(ValueProfData);
478  const void *Record = Closure->Record;
479  uint32_t NumValueKinds = Closure->GetNumValueKinds(Record);
480  if (NumValueKinds == 0)
481    return TotalSize;
482
483  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
484    uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
485    if (!NumValueSites)
486      continue;
487    TotalSize += getValueProfRecordSize(NumValueSites,
488                                        Closure->GetNumValueData(Record, Kind));
489  }
490  return TotalSize;
491}
492
493/*!
494 * Extract value profile data of a function for the profile kind \c ValueKind
495 * from the \c Closure and serialize the data into \c This record instance.
496 */
497void serializeValueProfRecordFrom(ValueProfRecord *This,
498                                  ValueProfRecordClosure *Closure,
499                                  uint32_t ValueKind, uint32_t NumValueSites) {
500  uint32_t S;
501  const void *Record = Closure->Record;
502  This->Kind = ValueKind;
503  This->NumValueSites = NumValueSites;
504  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
505
506  for (S = 0; S < NumValueSites; S++) {
507    uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
508    This->SiteCountArray[S] = ND;
509    Closure->GetValueForSite(Record, DstVD, ValueKind, S,
510                             Closure->RemapValueData);
511    DstVD += ND;
512  }
513}
514
515/*!
516 * Extract value profile data of a function  from the \c Closure
517 * and serialize the data into \c DstData if it is not NULL or heap
518 * memory allocated by the \c Closure's allocator method.
519 */
520ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
521                                          ValueProfData *DstData) {
522  uint32_t Kind;
523  uint32_t TotalSize = getValueProfDataSize(Closure);
524
525  ValueProfData *VPD =
526      DstData ? DstData : Closure->AllocValueProfData(TotalSize);
527
528  VPD->TotalSize = TotalSize;
529  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
530  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
531  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
532    uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
533    if (!NumValueSites)
534      continue;
535    serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
536    VR = getValueProfRecordNext(VR);
537  }
538  return VPD;
539}
540
541/*
542 * The value profiler runtime library stores the value profile data
543 * for a given function in \c NumValueSites and \c Nodes structures.
544 * \c ValueProfRuntimeRecord class is used to encapsulate the runtime
545 * profile data and provides fast interfaces to retrieve the profile
546 * information. This interface is used to initialize the runtime record
547 * and pre-compute the information needed for efficient implementation
548 * of callbacks required by ValueProfRecordClosure class.
549 */
550int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
551                                     const uint16_t *NumValueSites,
552                                     ValueProfNode **Nodes) {
553  unsigned I, J, S = 0, NumValueKinds = 0;
554  RuntimeRecord->NumValueSites = NumValueSites;
555  RuntimeRecord->Nodes = Nodes;
556  for (I = 0; I <= IPVK_Last; I++) {
557    uint16_t N = NumValueSites[I];
558    if (!N) {
559      RuntimeRecord->SiteCountArray[I] = INSTR_PROF_NULLPTR;
560      continue;
561    }
562    NumValueKinds++;
563    RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1);
564    if (!RuntimeRecord->SiteCountArray[I])
565      return 1;
566    RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : INSTR_PROF_NULLPTR;
567    for (J = 0; J < N; J++) {
568      /* Compute value count for each site. */
569      uint32_t C = 0;
570      ValueProfNode *Site =
571          Nodes ? RuntimeRecord->NodesKind[I][J] : INSTR_PROF_NULLPTR;
572      while (Site) {
573        C++;
574        Site = Site->Next;
575      }
576      if (C > UCHAR_MAX)
577        C = UCHAR_MAX;
578      RuntimeRecord->SiteCountArray[I][J] = C;
579    }
580    S += N;
581  }
582  RuntimeRecord->NumValueKinds = NumValueKinds;
583  return 0;
584}
585
586void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) {
587  unsigned I;
588  for (I = 0; I <= IPVK_Last; I++) {
589    if (RuntimeRecord->SiteCountArray[I])
590      free(RuntimeRecord->SiteCountArray[I]);
591  }
592}
593
594/* ValueProfRecordClosure Interface implementation for
595 * ValueProfDataRuntimeRecord.  */
596uint32_t getNumValueKindsRT(const void *R) {
597  return ((const ValueProfRuntimeRecord *)R)->NumValueKinds;
598}
599
600uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
601  return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK];
602}
603
604uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) {
605  const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
606  return Record->SiteCountArray[VK][S];
607}
608
609uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
610  unsigned I, S = 0;
611  const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
612  if (Record->SiteCountArray[VK] == INSTR_PROF_NULLPTR)
613    return 0;
614  for (I = 0; I < Record->NumValueSites[VK]; I++)
615    S += Record->SiteCountArray[VK][I];
616  return S;
617}
618
619void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK,
620                       uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) {
621  unsigned I, N = 0;
622  const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
623  N = getNumValueDataForSiteRT(R, VK, S);
624  if (N == 0)
625    return;
626  ValueProfNode *VNode = Record->NodesKind[VK][S];
627  for (I = 0; I < N; I++) {
628    Dst[I] = VNode->VData;
629    VNode = VNode->Next;
630  }
631}
632
633ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) {
634  return (ValueProfData *)calloc(TotalSizeInBytes, 1);
635}
636
637static ValueProfRecordClosure RTRecordClosure = {
638    INSTR_PROF_NULLPTR, getNumValueKindsRT,       getNumValueSitesRT,
639    getNumValueDataRT,  getNumValueDataForSiteRT, INSTR_PROF_NULLPTR,
640    getValueForSiteRT,  allocValueProfDataRT};
641
642/*
643 * Return the size of ValueProfData structure to store data
644 * recorded in the runtime record.
645 */
646uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) {
647  RTRecordClosure.Record = Record;
648  return getValueProfDataSize(&RTRecordClosure);
649}
650
651/*
652 * Return a ValueProfData instance that stores the data collected
653 * from runtime. If \c DstData is provided by the caller, the value
654 * profile data will be store in *DstData and DstData is returned,
655 * otherwise the method will allocate space for the value data and
656 * return pointer to the newly allocated space.
657 */
658ValueProfData *
659serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
660                             ValueProfData *DstData) {
661  RTRecordClosure.Record = Record;
662  return serializeValueProfDataFrom(&RTRecordClosure, DstData);
663}
664
665#undef INSTR_PROF_COMMON_API_IMPL
666#endif /* INSTR_PROF_COMMON_API_IMPL */
667
668/*============================================================================*/
669
670#ifndef INSTR_PROF_DATA_DEFINED
671
672#ifndef INSTR_PROF_DATA_INC
673#define INSTR_PROF_DATA_INC
674
675/* Helper macros.  */
676#define INSTR_PROF_SIMPLE_QUOTE(x) #x
677#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
678#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
679#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
680
681/* Magic number to detect file format and endianness.
682 * Use 255 at one end, since no UTF-8 file can use that character.  Avoid 0,
683 * so that utilities, like strings, don't grab it as a string.  129 is also
684 * invalid UTF-8, and high enough to be interesting.
685 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
686 * for 32-bit platforms.
687 */
688#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
689       (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 |  \
690        (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
691#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
692       (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 |  \
693        (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
694
695/* Raw profile format version. */
696#define INSTR_PROF_RAW_VERSION 2
697#define INSTR_PROF_INDEX_VERSION 3
698#define INSTR_PROF_COVMAP_VERSION 0
699
700/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
701 * version for other variants of profile. We set the lowest bit of the upper 8
702 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
703 * generated profile, and 0 if this is a Clang FE generated profile.
704*/
705#define VARIANT_MASKS_ALL 0xff00000000000000ULL
706#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
707
708/* Runtime section names and name strings.  */
709#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data
710#define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names
711#define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts
712#define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap
713
714#define INSTR_PROF_DATA_SECT_NAME_STR                                          \
715  INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME)
716#define INSTR_PROF_NAME_SECT_NAME_STR                                          \
717  INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME)
718#define INSTR_PROF_CNTS_SECT_NAME_STR                                          \
719  INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
720#define INSTR_PROF_COVMAP_SECT_NAME_STR                                        \
721  INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME)
722
723/* Macros to define start/stop section symbol for a given
724 * section on Linux. For instance
725 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
726 * expand to __start___llvm_prof_data
727 */
728#define INSTR_PROF_SECT_START(Sect) \
729        INSTR_PROF_CONCAT(__start_,Sect)
730#define INSTR_PROF_SECT_STOP(Sect) \
731        INSTR_PROF_CONCAT(__stop_,Sect)
732
733/* Value Profiling API linkage name.  */
734#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
735#define INSTR_PROF_VALUE_PROF_FUNC_STR \
736        INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
737
738/* InstrProfile per-function control data alignment.  */
739#define INSTR_PROF_DATA_ALIGNMENT 8
740
741/* The data structure that represents a tracked value by the
742 * value profiler.
743 */
744typedef struct InstrProfValueData {
745  /* Profiled value. */
746  uint64_t Value;
747  /* Number of times the value appears in the training run. */
748  uint64_t Count;
749} InstrProfValueData;
750
751/* This is an internal data structure used by value profiler. It
752 * is defined here to allow serialization code sharing by LLVM
753 * to be used in unit test.
754 */
755typedef struct ValueProfNode {
756  InstrProfValueData VData;
757  struct ValueProfNode *Next;
758} ValueProfNode;
759
760#endif /* INSTR_PROF_DATA_INC */
761
762#else
763#undef INSTR_PROF_DATA_DEFINED
764#endif
765