LLVM LLD linker enabling in edk2 build - shijunjing/edk2 GitHub Wiki

LLD is a linker from the LLVM project. That is a drop-in replacement for system linkers and runs much faster than them. According to the lld introduction, the lld is more than twice faster than the gold linker which is current default linker in Clang38 toolchain. Detail introduction can been found here: "https://lld.llvm.org/":https://lld.llvm.org/

I'm working on the build performance improvement, and I can use compiler cache method to accelerate the compile time, but the compiler cache cannot accelerate the link time speed. So, I'm interested in whether the lld can help us to accelerate the link time This task is to enable the lld, which is llvm native linker, in the Clang38 (and even GCC) toolchain.

  1. Build release version LLVM/Clang with lld
$ git clone  https://github.com/llvm-project/llvm.git
$ cd ~/llvm/tools/
$ git clone https://github.com/llvm-mirror/clang.git
$ cd ~/llvm/tools
$ git clone https://github.com/llvm-mirror/lld.git
$ cd ~/llvm/projects/
$ git clone https://github.com/llvm-mirror/compiler-rt.git
$ cd ~
$ wget https://ftp.gnu.org/gnu/binutils/binutils-2.31.tar.gz
$ tar -xvf binutils-2.31.tar.gz
$ mkdir releasebuild
$ cd releasebuild

Build with distcc distributed compilation disabled:

cmake ../llvm -G "Unix Makefiles" -DCMAKE_BUILD_TYPE="Release" -DLLVM_TARGETS_TO_BUILD="X86" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_COMPILER="/home/jshi19/wksp_efi/distcc/distcc_ln/g++" -DCMAKE_C_COMPILER="/home/jshi19/wksp_efi/distcc/distcc_ln/gcc" -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_BINUTILS_INCDIR=/home/jshi19/llvm/binutils-2.31/include -DLLVM_INSTALL_UTILS=ON -DLLVM_ENABLE_ASSERTIONS=ON -DCMAKE_C_FLAGS=-DLLVM_ENABLE_DUMP -DCMAKE_CXX_FLAGS=-DLLVM_ENABLE_DUMP -DCMAKE_INSTALL_PREFIX=../releaseinstall/ 
$ make -j $(getconf _NPROCESSORS_ONLN)

Build with distcc distributed compilation enabled:

$ cmake ../llvm -G "Unix Makefiles" -DCMAKE_BUILD_TYPE="Release" -DLLVM_TARGETS_TO_BUILD="X86" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_COMPILER="/home/jshi19/wksp_efi/distcc/distcc_ln/g++" -DCMAKE_C_COMPILER="/home/jshi19/wksp_efi/distcc/distcc_ln/gcc" -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_BINUTILS_INCDIR=/home/jshi19/llvm/binutils-2.31/include -DLLVM_INSTALL_UTILS=ON -DLLVM_ENABLE_ASSERTIONS=ON -DCMAKE_C_FLAGS=-DLLVM_ENABLE_DUMP -DCMAKE_CXX_FLAGS=-DLLVM_ENABLE_DUMP -DCMAKE_INSTALL_PREFIX=../releaseinstall/ 
$ make -j 20
  1. If you want, you can build and install GNU Binutils 2.31 as below steps (this step is not necessary at all) in Ubuntu:
$sudo apt-get install bison
$sudo apt-get install flex
Install other necessary binutils build tools if missing
$ wget https://ftp.gnu.org/gnu/binutils/binutils-2.31.tar.gz
$ tar -xvf binutils-2.31.tar.gz
$ mkdir binutils-2.31build
$ cd binutils-2.31build
$ ../binutils-2.31/configure --enable-gold --enable-plugins --disable-werror --prefix=/usr
$ make -j 4
$ sudo make install

Status: The lld 7 doesn't support to discard the .dynamic sections, which is required by /edk2/BaseTools/Scripts/GccBase.lds, and have below link failure:

ld.lld: error: discarding .dynsym section is not allowed
ld.lld: error: discarding .dynstr section is not allowed
ld.lld: error: discarding .dynamic section is not allowed
clang-7: error: linker command failed with exit code 1 (use -v to see invocation)

But latest lld trunk happen to support it in this month, see below code review. Have verified the lld latest version in trunk can solve the .dynamic sections discard problem. "https://reviews.llvm.org/D55211":https://reviews.llvm.org/D55211

Enabling branch: "https://github.com/shijunjing/edk2/tree/lld":https://github.com/shijunjing/edk2/tree/lld "https://github.com/lgao4/edk2.git":https://github.com/lgao4/edk2.git branch clang7win

There are two options to enable the lld in windows:

  1. Windows CLANG.exe + LLD COFF sublinker lld-link.exe, which is CLANG7WIN tool chain in branch clang7win
  2. Windows CLANG.exe + LLD COFF sublinker ld.lld.exe, which is CLANG38 tool chain in branch clang7win

Build and debug commands to reproduce the crash in Linux: Windows:

D:\Project\Edk2Git\workspace\lgao4\edk2>set CLANG38_BIN=D:\Project\LLVM\LLVM_Rel_20190121\bin\
D:\Project\Edk2Git\workspace\lgao4\edk2>build -p OvmfPkg\OvmfPkgX64.dsc -a X64 -t CLANG38 -b DEBUG
D:\Project\Edk2Git\workspace\lgao4\edk2>"C:\Program Files\qemu\qemu-system-x86_64.exe"  -pflash Build\OvmfX64\DEBUG_CLANG38\FV\OVMF.fd -debugcon file:serial.log -global isa-debugcon.iobase=0x402

Linux:

  1. Apply below patch to add function pointer list code:
+++ b/MdeModulePkg/Application/HelloWorld/HelloWorld.c
@@ -28,6 +28,35 @@
 //
 GLOBAL_REMOVE_IF_UNREFERENCED EFI_STRING_ID mStringHelpTokenId = STRING_TOKEN (STR_HELLO_WORLD_HELP_INFORMATION);
 
+
+typedef
+CHAR16 *
+(* TEST_FUNCTION) (
+  IN EFI_HANDLE          Handle
+  );
+
+CHAR16 *
+TestFunction1 (
+  IN EFI_HANDLE                Handle
+  )
+{
+  return NULL;
+}
+
+CHAR16 *
+TestFunction2 (
+  IN EFI_HANDLE                Handle
+  )
+{
+  return NULL;
+}
+
+TEST_FUNCTION mTestFunctionList[] = {
+  TestFunction1,
+  TestFunction2
+};
+
+
 /**
   The user Entry Point for Application. The user code starts with this function
   as the real entry point for the application.
@@ -46,10 +75,17 @@ UefiMain (
   IN EFI_SYSTEM_TABLE  *SystemTable
   )
 {
-  UINT32 Index;
+  volatile UINT32   Index;
 
   Index = 0;
 
+  Print (L"TestFunction1= 0x%x\n", TestFunction1);
+  Print (L"mTestFunctionList[0]= 0x%x\n", mTestFunctionList[0]);
+  Print (L"mTestFunctionList[Index]= 0x%x\n", mTestFunctionList[Index]);
+
+  Print (L"To call function list\n");
+  mTestFunctionList[Index] (ImageHandle);
+  Print (L"Return from function list\n");
   //
   // Three PCD type (FeatureFlag, UINT32 and String) are used as the sample.
   //
  1. Build the HelloWorld and run
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ export CLANG38_BIN=~/llvm/releaseinstall/bin/
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ build -p OvmfPkg/OvmfPkgX64.dsc -a X64 -t GCC5 -b NOOPT
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ build -p OvmfPkg/OvmfPkgX64.dsc -a X64 -t CLANG38 -b NOOPT -m MdeModulePkg/Application/HelloWorld/HelloWorld.inf
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ cp Build/OvmfX64/NOOPT_GCC5/X64/HelloWorld.efi Build/rootfs/HelloWorld.gcc5.efi
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ cp Build/OvmfX64/NOOPT_CLANG38/X64/HelloWorld.efi Build/rootfs/
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ qemu-system-x86_64 -pflash Build/OvmfX64/NOOPT_GCC5/FV/OVMF.fd -debugcon file:serial.log -global isa-debugcon.iobase=0x402 -nographic -net none -hda fat:Build/rootfs/

FS0:\> HelloWorld.gcc5.efi
TestFunction1= 0x6822240
mTestFunctionList[0]= 0x6822240
mTestFunctionList[Index]= 0x6822240
To call function list
Return from function list
UEFI Hello World!
FS0:\> HelloWorld.efi
TestFunction1= 0x68238B7
mTestFunctionList[0]= 0x68238B7
mTestFunctionList[Index]= 0x6822000
To call function list
!!!! X64 Exception Type - 06(#UD - Invalid Opcode)  CPU Apic ID - 00000000 !!!!
RIP  - 00000000068220B2, CS  - 0000000000000038, RFLAGS - 0000000000000282
RAX  - 00000000000000E0, RCX - 0000000007B79BE0, RDX - 0000000000000007
RBX  - 0000000000000000, RSP - 0000000007EA6590, RBP - 0000000006B1D998
RSI  - 00000000068245C0, RDI - 0000000006B1DC18
R8   - 0000000006B1C000, R9  - 0000000000001000, R10 - 0000000006823708
R11  - 0000000000000000, R12 - 0000000007EA66C8, R13 - 0000000007EA66B8
R14  - 000000000674B201, R15 - 0000000007EA6828
DS   - 0000000000000030, ES  - 0000000000000030, FS  - 0000000000000030
GS   - 0000000000000030, SS  - 0000000000000030
CR0  - 0000000080010033, CR2 - 0000000000000000, CR3 - 0000000007C01000
CR4  - 0000000000000668, CR8 - 0000000000000000
DR0  - 0000000000000000, DR1 - 0000000000000000, DR2 - 0000000000000000
DR3  - 0000000000000000, DR6 - 00000000FFFF0FF0, DR7 - 0000000000000400
GDTR - 0000000007BEE698 0000000000000047, LDTR - 0000000000000000
IDTR - 00000000072E2018 0000000000000FFF,   TR - 0000000000000000
FXSAVE_STATE - 0000000007EA61F0
!!!! Find image based on IP(0x68220B2) /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/DEBUG_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll (ImageBase=0000000006822000, EntryPoint=000000000682357C) !!!!
QEMU 2.11.1 monitor - type 'help' for more information

You can see the HelloWorld.efi mTestFunctionList[Index]= 0x6822000 is not correct, which means a relocation is not correctly resolved.

  1. Read the HelloWorld relocation type as below, you will see there are many R_X86_64_PLT32 are emitted which might not be correctly converted by BaseTools\Source\C\GenFw\Elf64Convert.c.
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -r Build/OvmfX64/DEBUG_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.debug
... ...
Relocation section '.rela.text' at offset 0x2968 contains 204 entries:
  Offset          Info           Type           Sym. Value    Sym. Name + Addend
00000000036c  000000000002 R_X86_64_PC32                        -4
000000000373  000000000002 R_X86_64_PC32                        -4
00000000037b  004600000004 R_X86_64_PLT32    0000000000000240 .text + 12b8
0000000003b1  000000000002 R_X86_64_PC32                        -4
0000000003b8  000000000002 R_X86_64_PC32                        -4
0000000003c2  004600000004 R_X86_64_PLT32    0000000000000240 .text + 12b8
00000000045d  000000000002 R_X86_64_PC32                        -4
000000000464  000000000002 R_X86_64_PC32                        -4
0000000004b4  004600000004 R_X86_64_PLT32    0000000000000240 .text + 1aa
0000000004c3  000000000002 R_X86_64_PC32                        -4
0000000004ca  000000000002 R_X86_64_PC32                        -4
... ..

If you compare GCC5 built relocations, there is also R_X86_64_PLT32 emitted, but very few. Need continue to investigate the R_X86_64_PLT32 covert rule.

jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -r Build/OvmfX64/DEBUG_GCC5/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.debug

Relocation section '.rela.text' at offset 0xc8b0 contains 147 entries:
  Offset          Info           Type           Sym. Value    Sym. Name + Addend
000000000248  000200000002 R_X86_64_PC32     0000000000002140 .data + 154
000000000259  000200000002 R_X86_64_PC32     0000000000002140 .data + 154
000000000261  000200000002 R_X86_64_PC32     0000000000002140 .data + 154
000000000267  000200000002 R_X86_64_PC32     0000000000002140 .data + 154
0000000002d8  002b00000002 R_X86_64_PC32     00000000000016a6 .LC0 - 4
0000000002ea  000100000002 R_X86_64_PC32     0000000000000240 .text + f64
0000000002f2  000100000002 R_X86_64_PC32     0000000000000240 .text + 2
000000000306  007500000004 R_X86_64_PLT32    000000000000163b IoWriteFifo8 - 4
000000000337  000100000002 R_X86_64_PC32     0000000000000240 .text + 2b
000000000347  002c00000002 R_X86_64_PC32     00000000000016ba .LC1 - 4
00000000034e  002d00000002 R_X86_64_PC32     00000000000016dd .LC2 - 4
  • Investigations on 20190126
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -r /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/NOOPT_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll | grep mTestFunctionList
00000000319f  00d900000002 R_X86_64_PC32     0000000000005040 mTestFunctionList - 4
0000000031c0  00d900000002 R_X86_64_PC32     0000000000005040 mTestFunctionList - 4
0000000031f5  00d900000002 R_X86_64_PC32     0000000000005040 mTestFunctionList - 4

The mTestFunctionList three relocations look correct, and they all point to the relocation whose offset is 0x5040 through the symbol item 0x00d9 which is Num 217. You can check the symbol Num 217, readelf find the 0x5040 offset value through the Num 217 in symbol table.

jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -s /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/NOOPT_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll
   217: 0000000000005040    16 OBJECT  LOCAL  HIDDEN     2 mTestFunctionList

And the offset 0x5040 relocation is TestFunction1:
jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -r /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/NOOPT_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll
000000005040  00d600000001 R_X86_64_64       0000000000003130 TestFunction1 + 0
000000005048  00d700000001 R_X86_64_64       0000000000003150 TestFunction2 + 0
And you can see the TestFunction1 symbol as the Num 0xd6/214
   214: 0000000000003130    17 FUNC    LOCAL  HIDDEN     1 TestFunction1
   215: 0000000000003150    17 FUNC    LOCAL  HIDDEN     1 TestFunction2

OK, the TestFunction1 is a function which has the offset 0x3130 in the object file. You can check the TestFunction1 real offset like this:

jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ objdump -S /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/NOOPT_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll
CHAR16 *
TestFunction1 (
  IN EFI_HANDLE                Handle
  )
{
    3130:       55                      push   %rbp
    3131:       48 89 e5                mov    %rsp,%rbp
    3134:       50                      push   %rax
    3135:       31 c0                   xor    %eax,%eax
    3137:       48 89 7d f8             mov    %rdi,-0x8(%rbp)
  return NULL;
    313b:       48 83 c4 08             add    $0x8,%rsp
    313f:       5d                      pop    %rbp
    3140:       c3                      retq
    3141:       cc                      int3
    3142:       cc                      int3

OK, it is correct! Current problem is "GenFw" not correctly convert the R_X86_64_64 0x5040 relocation in PE/COFF, need continue to investigate.

other useful command: We can check the '.data' content like this:

jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -S /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/NOOPT_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll
There are 16 section headers, starting at offset 0x5ee80:
Section Headers:
  [Nr] Name              Type             Address           Offset
       Size              EntSize          Flags  Link  Info  Align
  [ 0]                   NULL             0000000000000000  00000000
       0000000000000000  0000000000000000           0     0     0
  [ 1] .text             PROGBITS         0000000000000240  000001c0
       0000000000004c88  0000000000000000 WAXMS       0     0     64
  [ 2] .data             PROGBITS         0000000000004f00  00004e80
       0000000000000190  0000000000000000  WA       0     0     64
readelf: Warning: [ 3]: Link field (0) should index a symtab section.
  [ 3] .rela             RELA             0000000000000000  00005010
       00000000000003d8  0000000000000018           0     0     8
  [ 4] .debug_str        PROGBITS         0000000000000000  000053e8
       00000000000194c4  0000000000000001  MS       0     0     1
... ...

From the section info we know the '.data' section Num is 2. OK, then

jshi19@ub2-uefi-b01:~/wksp_efi/lgao4/edk2$ readelf -x2 /home/jshi19/wksp_efi/lgao4/edk2/Build/OvmfX64/NOOPT_CLANG38/X64/MdeModulePkg/Application/HelloWorld/HelloWorld/DEBUG/HelloWorld.dll
Hex dump of section '.data':
 NOTE: This section has relocations against it, but these have NOT been applied to this dump.
  0x00004f00 00000000 00000000 00000000 00000000 ................
  0x00004f10 00000000 00000000 00000000 00000000 ................
  0x00004f20 00000000 00000000 00000000 00000000 ................
  0x00004f30 00000000 00000000 00000000 00000000 ................
  0x00004f40 00000000 00000000 00000000 00000000 ................
  0x00004f50 00000000 00000000 00000000 00000000 ................
  0x00004f60 00000000 00000000 00000000 00000000 ................
  0x00004f70 00000000 00000000 00000000 00000000 ................
  0x00004f80 00000000 00000000 00000000 00000000 ................
  0x00004f90 00000000 00000000 00000000 00000000 ................
  0x00004fa0 00000000 00000000 00000000 00000000 ................
  0x00004fb0 00000000 00000000 00000000 00000000 ................
  0x00004fc0 00000000 00000000 00000000 00000000 ................
  0x00004fd0 00000000 00000000 00000000 00000000 ................
  0x00004fe0 00000000 00000000 00000000 00000000 ................
  0x00004ff0 00000000 00000000 00000000 00000000 ................
  0x00005000 00000000 00000000 00000000 00000000 ................
  0x00005010 00000000 00000000 00000000 00000000 ................
  0x00005020 00000000 00000000 00000000 00000000 ................
  0x00005030 00000000 00000000 00000000 00000000 ................
  0x00005040 00000000 00000000 00000000 00000000 ................
  0x00005050 4ebe7903 06d77d43 b037edb8 2fb772a4 N.y...}C.7../.r.
  0x00005060 00000000 00000000 00000000 00000000 ................
  0x00005070 00000000 00000000 00000000 00000000 ................
  0x00005080 00000000 00000000 00000000 00000000 ................

Caution! the offset 0x5040 value is 0, which looks wrong!!! It should contain the offset of TestFunction1, which is 0x3130 ?

The -Wl,-pie option cause the lld don't write the R_X86_64_64 values in ".data" section. If I change the -Wl,-pie to -fpie in *_CLANG38_X64_DLINK_FLAGS, the R_X86_64_64 vales appear in ".data" section, and the OvmfPkgX64 NOOPT target works and can boot to shell! But for DEBUG target, the lld will emit a R_X86_64_32S relocation item which is not supported by current Elf64convert. See R_X86_64_32S comments between lines 1199 ~ 1220 in edk2\BaseTools\Source\C\GenFw\Elf64Convert.c

OK, next work should be to investigate why the lld don't write the R_X86_64_64 values in ".data" section if -Wl,-pie option is used.

Status in 20190129:

Good news. I’ve enabled the ld.lld and the Clang38 toolchain can work in both linux windows now! All edk2 OvmfPkgX64, OvmfPkgIa32X64 and OvmfPkgIa32 platforms can pass build in window and successfully boot Qemu now. Need to add --apply-dynamic-relocs to force ld.lld write relocation. Below is the enabling patch. Details see below mail discussion: http://lists.llvm.org/pipermail/llvm-dev/2019-January/129679.html

From e1bf50c14d2175579c962d793623dd24cd3d919e Mon Sep 17 00:00:00 2001
From: "Shi, Steven" <[email protected]>
Date: Tue, 29 Jan 2019 14:36:47 +0800
Subject: [PATCH] Set --apply-dynamic-relocs to force ld.lld write relocation
 addend

One difference between ld.lld and ld.bfd is that ld.lld defaults to
--no-apply-dynamic-relocs (do not write the relocation addend) and
ld.bfd defaults to --apply-dynamic-relocs (write the relocation addend).
Without --apply-dynamic-relocs, the lld will write wrong 0 symbol value
in .data section if enable -pie. Details see below mail discussion:
http://lists.llvm.org/pipermail/llvm-dev/2019-January/129679.html
---
 BaseTools/Conf/tools_def.template | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template
index 017e476b96..db8854b592 100755
--- a/BaseTools/Conf/tools_def.template
+++ b/BaseTools/Conf/tools_def.template
@@ -5367,15 +5367,15 @@ NOOPT_CLANG38_IA32_DLINK2_FLAGS     = DEF(GCC5_IA32_DLINK2_FLAGS) -O0 -fuse-ld=l
 *_CLANG38_X64_VFRPP_FLAGS          = DEF(GCC_VFRPP_FLAGS) DEF(CLANG38_X64_TARGET)
 
 DEBUG_CLANG38_X64_CC_FLAGS         = DEF(CLANG38_ALL_CC_FLAGS) -m64 "-DEFIAPI=__attribute__((ms_abi))" -mno-red-zone -mcmodel=small -fpie -Oz -flto DEF(CLANG38_X64_TARGET) -g
-DEBUG_CLANG38_X64_DLINK_FLAGS      = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Wl,-O3 -Wl,-melf_x86_64 -Wl,--oformat,elf64-x86-64 -Wl,-pie -mcmodel=small
+DEBUG_CLANG38_X64_DLINK_FLAGS      = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Wl,-O3 -Wl,-melf_x86_64 -Wl,--oformat,elf64-x86-64 -Wl,-pie -mcmodel=small -Wl,--apply-dynamic-relocs
 DEBUG_CLANG38_X64_DLINK2_FLAGS     = DEF(GCC5_X64_DLINK2_FLAGS) -O3 -fuse-ld=lld
 
 RELEASE_CLANG38_X64_CC_FLAGS       = DEF(CLANG38_ALL_CC_FLAGS) -m64 "-DEFIAPI=__attribute__((ms_abi))" -mno-red-zone -mcmodel=small -fpie -Oz -flto DEF(CLANG38_X64_TARGET)
-RELEASE_CLANG38_X64_DLINK_FLAGS    = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Wl,-O3 -Wl,-melf_x86_64 -Wl,--oformat,elf64-x86-64 -Wl,-pie -mcmodel=small
+RELEASE_CLANG38_X64_DLINK_FLAGS    = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Wl,-O3 -Wl,-melf_x86_64 -Wl,--oformat,elf64-x86-64 -Wl,-pie -mcmodel=small -Wl,--apply-dynamic-relocs
 RELEASE_CLANG38_X64_DLINK2_FLAGS   = DEF(GCC5_X64_DLINK2_FLAGS) -O3 -fuse-ld=lld
 
 NOOPT_CLANG38_X64_CC_FLAGS         = DEF(CLANG38_ALL_CC_FLAGS) -m64 "-DEFIAPI=__attribute__((ms_abi))" -mno-red-zone -mcmodel=small -fpie -O0 DEF(CLANG38_X64_TARGET) -g
-NOOPT_CLANG38_X64_DLINK_FLAGS      = DEF(GCC5_IA32_X64_DLINK_FLAGS) -Wl,-O0 -Wl,-melf_x86_64 -Wl,--oformat,elf64-x86-64 -Wl,-pie -mcmodel=small
+NOOPT_CLANG38_X64_DLINK_FLAGS      = DEF(GCC5_IA32_X64_DLINK_FLAGS) -Wl,-O0 -Wl,-melf_x86_64 -Wl,--oformat,elf64-x86-64 -Wl,-pie -mcmodel=small -Wl,--apply-dynamic-relocs
 NOOPT_CLANG38_X64_DLINK2_FLAGS     = DEF(GCC5_X64_DLINK2_FLAGS) -O0 -fuse-ld=lld
 
 ##################
-- 
2.17.1