.PHONY: all clean clean-profiles clean-pgo-profiles clean-bolt-profiles restore-originals delete-originals \
	print-profile-artifacts top stage-path

# Abort recipes on the first failed command, including inside loops.
.SHELLFLAGS := -ec

STAGE0_BUILD:=$(CURDIR)/toolchain
STAGE0_TARGET_BUILD:=$(CURDIR)/toolchain-target
STAGE1_BUILD:=$(CURDIR)/pgo-instrumented.build
STAGE2_BUILD:=$(CURDIR)/optimized.build

STAGE0_BINARIES:=$(STAGE0_BUILD)/usr/bin/
STAGE0_TOOLS:=$(STAGE0_BUILD)/usr/tools/

BOLT_PROFILE_DIR:=$(CURDIR)/profiles-bolt
PGO_PROFILE_DIR:=$(CURDIR)/profiles
PGO_PROFILE_FILE:=$(PGO_PROFILE_DIR)/merged.prof
PGO_PROFRAW_FILES:=$(wildcard $(PGO_PROFILE_DIR)/*.profraw)
BOLT_PROFRAW_FILES:=$(wildcard $(BOLT_PROFILE_DIR)/*-prof.[0-9]*.fdata)
JULIA_ROOT:=$(CURDIR)/../..
CONTRIB_DIR_REL:=contrib/$(notdir $(CURDIR))

# Ask the regular build system about the platform it would build for, so that
# the staged builds agree with it. BUILD_MACHINE is the target triple recorded
# in the system image (Sys.MACHINE); the stage-0 clang spells it differently
# than the default compiler (e.g. aarch64-apple-darwin vs arm64-apple-darwin),
# and a plain `make binary-dist` of the optimized build would then consider
# build_h.jl stale and rebuild the system image.
# Preserve command-line variables, but omit flags such as -n from the probe.
JULIA_BUILD_VARS:=$(shell MAKEFLAGS='-- $(subst ','"'"',$(MAKEOVERRIDES))' $(MAKE) -s --no-print-directory -C $(JULIA_ROOT) print-OS print-ARCH print-BINARY print-EXE print-BUILD_MACHINE print-COMPRESS_DEBUG_SECTIONS print-XC_HOST print-BUILD_OS print-MARCH print-MCPU print-JULIA_CPU_TARGET)
HOST_OS:=$(patsubst OS=%,%,$(filter OS=%,$(JULIA_BUILD_VARS)))
HOST_ARCH:=$(patsubst ARCH=%,%,$(filter ARCH=%,$(JULIA_BUILD_VARS)))
HOST_BINARY:=$(patsubst BINARY=%,%,$(filter BINARY=%,$(JULIA_BUILD_VARS)))
EXE:=$(patsubst EXE=%,%,$(filter EXE=%,$(JULIA_BUILD_VARS)))
BUILD_MACHINE:=$(patsubst BUILD_MACHINE=%,%,$(filter BUILD_MACHINE=%,$(JULIA_BUILD_VARS)))
COMPRESS_DEBUG_SECTIONS:=$(patsubst COMPRESS_DEBUG_SECTIONS=%,%,$(filter COMPRESS_DEBUG_SECTIONS=%,$(JULIA_BUILD_VARS)))
XC_HOST:=$(patsubst XC_HOST=%,%,$(filter XC_HOST=%,$(JULIA_BUILD_VARS)))
BUILD_OS:=$(patsubst BUILD_OS=%,%,$(filter BUILD_OS=%,$(JULIA_BUILD_VARS)))
MARCH:=$(patsubst MARCH=%,%,$(filter MARCH=%,$(JULIA_BUILD_VARS)))
MCPU:=$(patsubst MCPU=%,%,$(filter MCPU=%,$(JULIA_BUILD_VARS)))
# Stage 1 only exists to profile the compiler, so it only needs the first CPU
# target. That one is the baseline, which on riscv64 must include the runtime's
# floating-point ABI. Stage 2 uses all of JULIA_CPU_TARGET.
STAGE1_CPU_TARGET ?= $(firstword $(subst ;, ,$(patsubst JULIA_CPU_TARGET=%,%,$(filter JULIA_CPU_TARGET=%,$(JULIA_BUILD_VARS)))))
ifeq (,$(HOST_OS))
$(error Could not determine the build platform: `$(MAKE) -C $(JULIA_ROOT) print-OS` \
        failed. Run it to see why; a XC_HOST without a matching OS is one way.)
endif

# Which optimizations to apply. PGO and ThinLTO work wherever the stage-0
# toolchain does. BOLT rewrites ELF binaries only, and only for x86-64 and
# AArch64, so it defaults to off everywhere else; set USE_BOLT explicitly to
# override. USE_PGO and USE_LTO are mostly there to bisect build problems.
USE_PGO ?= 1
USE_LTO ?= 1
ifeq ($(HOST_OS),Linux)
ifneq (,$(filter $(HOST_ARCH),x86_64 aarch64))
USE_BOLT ?= 1
endif
endif
USE_BOLT ?= 0

# Backends of the stage-0 toolchain when cross-compiling.
LLVM_BACKEND_x86_64 := X86
LLVM_BACKEND_i686 := X86
LLVM_BACKEND_aarch64 := AArch64
LLVM_BACKEND_armv7l := ARM
LLVM_BACKEND_powerpc64le := PowerPC
LLVM_BACKEND_riscv64 := RISCV
STAGE0_LLVM_TARGETS ?= host;$(LLVM_BACKEND_$(HOST_ARCH))
# compiler-rt's name for the target architecture, where it differs.
COMPILER_RT_ARCH_i686 := i386
COMPILER_RT_ARCH_armv7l := armhf
COMPILER_RT_ARCH := $(or $(COMPILER_RT_ARCH_$(HOST_ARCH)),$(HOST_ARCH))

# Each concurrent ThinLTO backend holds a module and the PGO profile.
# Limit their number on 32-bit targets so wide hosts do not exhaust the
# linker's address space. Empty leaves the linker's default.
ifeq ($(HOST_BINARY),32)
LTO_JOBS ?= 8
endif
LTO_JOBS ?=

# Settings taken from https://github.com/rust-lang/rust/blob/master/src/tools/opt-dist/src/bolt.rs
BOLT_ARGS :=
# Reorder basic blocks within functions
BOLT_ARGS += -reorder-blocks=ext-tsp
# Reorder functions within the binary
BOLT_ARGS += -reorder-functions=cdsort
# Split function code into hot and cold regions
BOLT_ARGS += -split-functions
# Split as many basic blocks as possible
BOLT_ARGS += -split-all-cold
# Move jump tables to a separate section (only supported on x86-64).
ifeq ($(HOST_ARCH),x86_64)
BOLT_ARGS += -jump-tables=move
endif
# Use regular size pages for code alignment
BOLT_ARGS += -no-huge-pages
# Preserve function addresses, including those obtained through dlsym.
BOLT_ARGS += -icf=none
# Cache-directed splitting is incompatible with AArch64's default code model.
# On x86-64, apply it below except for libjulia-internal until
# https://github.com/llvm/llvm-project/issues/89508 is fixed.
ifeq ($(HOST_ARCH),x86_64)
BOLT_SPLIT_STRATEGY := -split-strategy=cdsplit
endif
# Update DWARF debug info in the final binary
BOLT_ARGS += -update-debug-sections
# Print optimization statistics
BOLT_ARGS += -dyno-stats
# BOLT doesn't fully support computed gotos, https://github.com/llvm/llvm-project/issues/89117
# Use escaped regex as the name BOLT recognises is often a bit different, e.g. apply_cl/1(*2)
# This doesn't actually seem to do anything, the actual mitigation is not using --use-old-text
# which we do in the bolt target
BOLT_ARGS += -skip-funcs=.\*apply_cl.\*

# BOLT needs relocations in the libraries it rewrites. Only pass this when BOLT
# is enabled: --emit-relocs is an ELF linker flag.
# -fno-reorder-blocks-and-partition is needed on gcc >= 8.
ifeq ($(USE_BOLT),1)
BOLT_FLAGS := $\
	"BOLT_CFLAGS_GCC+=-fno-reorder-blocks-and-partition" $\
	"BOLT_LDFLAGS=-Wl,--emit-relocs"
else
BOLT_FLAGS :=
endif

LLVM_BOLT:=$(STAGE0_BINARIES)llvm-bolt
LLVM_MERGEFDATA:=$(STAGE0_BINARIES)merge-fdata
LLVM_CXXFILT:=$(STAGE0_TOOLS)llvm-cxxfilt
LLVM_PROFDATA:=$(STAGE0_TOOLS)llvm-profdata
LLVM_OBJCOPY:=$(STAGE0_TOOLS)llvm-objcopy

ifeq ($(USE_BOLT),1)
# Stop stage 2 after the libraries that BOLT rewrites, so that building the
# system image and the stdlib package images afterwards exercises the
# instrumented libraries and produces the BOLT profile.
STAGE2_TARGETS := julia-src-release julia-symlink julia-libccalltest \
				  julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest
# If you add new files to optimize, you need to add BOLT_LDFLAGS and BOLT_CFLAGS to the build of your new file.
# Resolve these symlinks in the recipes, after stage 2 creates them.
SYMLINKS_TO_OPTIMIZE := libLLVM.so libjulia-internal.so libjulia-codegen.so
else
# Without BOLT, stage 2 is the complete build.
STAGE2_TARGETS :=
SYMLINKS_TO_OPTIMIZE :=
endif

AFTER_INSTRUMENT_MESSAGE:='Run `make bolt-train` to build the rest of Julia with the $\
	instrumented libraries. You can then optionally collect more profiling data by running Julia $\
	with an appropriate workload; if you wish, run `make clean-bolt-profiles` before doing so to remove $\
	any profiling data generated by `make bolt-train`. You should end up with some data in $\
	$(BOLT_PROFILE_DIR). Afterwards run `make bolt`.'

# When building a single libLLVM.so we need to increase -vp-counters-per-site
# significantly
COUNTERS_PER_SITE:=6
# Note: profile counters are not atomic by default, https://discourse.llvm.org/t/profile-guided-optimization-pgo-related-questions-and-suggestions/75232/5

ifeq ($(USE_LTO),1)
LTO_FLAGS:=-flto=thin
else
LTO_FLAGS:=
endif

ifeq ($(USE_PGO),1)
PGO_GENERATE_FLAGS:=-fprofile-generate=$(PGO_PROFILE_DIR)
PGO_COUNTER_FLAGS:=-Xclang -mllvm -Xclang -vp-counters-per-site=$(COUNTERS_PER_SITE)
# Configure probes can collide with profiled functions such as main. Suppress
# missing/stale-profile warnings so CMake's -Werror checks test the requested
# option, not the profile's coverage. Otherwise LLVM can silently drop -fPIC
# and -fno-semantic-interposition after a profile hash mismatch.
PGO_USE_FLAGS:=-fprofile-use=$(PGO_PROFILE_FILE)
PGO_WARNING_FLAGS:=-Wno-backend-plugin -Wno-profile-instr-unprofiled $\
	-Wno-profile-instr-out-of-date
else
PGO_GENERATE_FLAGS:=
PGO_COUNTER_FLAGS:=
PGO_USE_FLAGS:=
PGO_WARNING_FLAGS:=
endif

AFTER_STAGE1_MESSAGE:='You can now optionally collect more profiling data for use in PGO by running Julia $\
	with an appropriate workload. If you wish, run `make clean-pgo-profiles` before doing so to remove any profiling data $\
	generated by building Julia. The profiles are written to $(PGO_PROFILE_DIR). $\
	Note that running extensive scripts may result in counter overflows, which can be detected by running $\
	`make top`. Afterwards run `make stage2`.'

# Native programs need Windows paths in LLVM_PROFILE_FILE. Keep compiler
# arguments in MSYS2 form so its argument conversion does not mangle them.
ifeq ($(HOST_OS),WINNT)
native_path = $(shell cygpath -m '$(1)')
else
native_path = $(1)
endif

# Platform hooks for toolchain setup, stage flags and DLL lookup.
STAGE0_EXTRA_TARGETS:=
STAGE0_EXTRA_FLAGS:=
# Turn [cd]tors into init/fini_array sections in libclang_rt, since lld doesn't
# do that, and otherwise the profile constructor is not executed.
STAGE0_POST = find $(STAGE0_BUILD) -name 'libclang_rt.profile-*.a' -exec $(LLVM_OBJCOPY) --rename-section .ctors=.init_array --rename-section .dtors=.fini_array {} +
STAGE1_PLATFORM_FLAGS:=
# Which C/C++ runtime headers and libraries to build against.
SYSROOT_FLAGS:=
# The target, which the regular build passes through CC and CXX as well. When
# cross-compiling, the default configuration files below are for the build
# machine, so skip them.
TARGET_FLAGS:=$(if $(XC_HOST),--no-default-config --target=$(XC_HOST)) $(if $(MARCH),-march=$(MARCH)) $(if $(MCPU),-mcpu=$(MCPU))
# When cross-compiling, tools for the build machine get the stage flags too,
# which only clang accepts.
ifneq (,$(XC_HOST))
CROSS_FLAGS:="HOSTCC=$(STAGE0_TOOLS)clang" "HOSTCXX=$(STAGE0_TOOLS)clang++"
# Those tools, and the stage-0 compiler-rt, need the build machine's GCC
# installation and sysroot, for the reasons given with SYSROOT_FLAGS below.
# HOSTCC has to be a plain command, so the stage-0 clang gets them from its
# default configuration files.
STAGE0_CONFIG_DIR:=$(STAGE0_BUILD)/clang-config
BUILD_CC:=$(patsubst HOSTCC=%,%,$(shell MAKEFLAGS='-- $(subst ','"'"',$(MAKEOVERRIDES))' $(MAKE) -s --no-print-directory -C $(JULIA_ROOT) print-HOSTCC))
ifeq (,$(if $(BUILD_CC),$(shell $(BUILD_CC) -dM -E -x c /dev/null 2>/dev/null | grep __clang__),clang))
BUILD_GCC_INSTALL_DIR:=$(patsubst %/,%,$(dir $(wildcard $(shell LANG=C $(BUILD_CC) -print-libgcc-file-name 2>/dev/null))))
BUILD_SYSROOT:=$(shell $(BUILD_CC) -print-sysroot 2>/dev/null)
endif
BUILD_SYSROOT_FLAGS:=$(if $(BUILD_GCC_INSTALL_DIR),--gcc-install-dir=$(BUILD_GCC_INSTALL_DIR) $(if $(BUILD_SYSROOT),--sysroot=$(BUILD_SYSROOT)))
endif
# How a stage's own shared libraries are made findable by the tools built
# during it. 1: that stage's build directory.
STAGE_PATH =
# What STAGE_PATH needs set up before a stage.
STAGE_PATH_SETUP =

# How to drive the toolchain on this platform. Everything else below is
# platform independent.
ifeq ($(HOST_OS),Darwin)
# The stage-0 clang does not come with Xcode, so tell it where the SDK is.
export SDKROOT ?= $(shell xcrun --show-sdk-path)
# Link with Xcode's ld64: the stage-0 ld64.lld cannot always read the text stubs
# of newer SDKs. ld64 has to use the stage-0 libLTO, since Xcode's cannot read
# the stage-0 bitcode, and clang only passes it along when it knows the linker
# version. Select ld64 explicitly for links driven by clang as well as make.
LINKER:=$(shell xcrun --find ld)
LINKER_FLAGS:=--ld-path=$(LINKER) -Wl,-lto_library,$(STAGE0_BUILD)/usr/lib/libLTO.dylib
LINK_ONLY_FLAGS:=
# For the same reason, LLVM's static libraries cannot be created with Xcode's
# libtool, which does not index the stage-0 bitcode.
PLATFORM_FLAGS:="LLVM_LIBTOOL=$(STAGE0_TOOLS)llvm-libtool-darwin"
else ifeq ($(HOST_OS),WINNT)
# Julia invokes LD directly; ld.lld selects lld's MinGW driver.
LINKER:=$(STAGE0_TOOLS)ld.lld$(EXE)
# MSYS2's GCC links winpthread through its spec file, which is what its libstdc++
# expects; clang only does so for -pthread.
LINKER_FLAGS:=-fuse-ld=lld -pthread
# The ELF-only flags do not apply to PE/COFF.
LINK_ONLY_FLAGS:=
# Apply the export map through a COFF definition for lld's MinGW driver.
# Keep the caller's linker flags, but not the driver flags added by this flow.
PLATFORM_FLAGS:="WIN_LD_USE_DEF=1" "LINK_LDFLAGS=$(LDFLAGS)"
# BinaryBuilder's tools need its compiler support libraries (MSYS2's
# libstdc++ lacks some of their imports), plus libLLVM's compression DLLs.
# Place these beside the tools before running any of them. Skip the ELF
# constructor fixup: COFF's profile runtime registers through .CRT$XCU.
STAGE0_EXTRA_TARGETS:=install-zlib install-csl
STAGE0_EXTRA_FLAGS:=USE_BINARYBUILDER_CSL=1
STAGE0_POST = cp $(STAGE0_BUILD)/usr/bin/*.dll $(STAGE0_TOOLS)
# Those direct linker invocations also miss the profile runtime that
# -fprofile-generate would otherwise have contributed to the link.
STAGE1_PLATFORM_FLAGS = "WIN_LD_EXTRA_LIBS=$(wildcard $(STAGE0_BUILD)/usr/lib/clang/*/lib/windows/libclang_rt.profile-*.a)"
# Let newly built tools such as llvm-tblgen find the stage's zstd/zlib DLLs.
# Append so the sysroot's compiler support libraries retain PATH precedence.
STAGE_PATH = PATH="$$PATH:$(1)/usr/bin"
else
LINKER:=$(STAGE0_TOOLS)ld.lld$(EXE)
LINKER_FLAGS:=-fuse-ld=lld
# Julia's version script names symbols that not every configuration defines.
LINK_ONLY_FLAGS:=-Wl,--undefined-version
ifneq (,$(LTO_JOBS))
LINK_ONLY_FLAGS+=-Wl,--thinlto-jobs=$(LTO_JOBS)
endif
PLATFORM_FLAGS:=
# Build against the GCC installation and sysroot of the regular build's C
# compiler, i.e. the same libstdc++ headers and C library. Left to itself, clang
# picks the newest GCC under /usr instead, and whatever C library that one uses:
# BinaryBuilder's build images keep their GCC with an old glibc in /usr/local,
# next to a newer distribution toolchain that the binaries would then require.
# Both can also be given explicitly, e.g. for a toolchain that cannot run here.
ifeq (,$(TOOLCHAIN_GCC_INSTALL_DIR))
HOST_CC:=$(patsubst CC=%,%,$(shell MAKEFLAGS='-- $(subst ','"'"',$(MAKEOVERRIDES))' $(MAKE) -s --no-print-directory -C $(JULIA_ROOT) print-CC))
ifeq (,$(shell $(HOST_CC) -dM -E -x c /dev/null 2>/dev/null | grep __clang__))
TOOLCHAIN_GCC_INSTALL_DIR:=$(patsubst %/,%,$(dir $(wildcard $(shell LANG=C $(HOST_CC) -print-libgcc-file-name 2>/dev/null))))
TOOLCHAIN_SYSROOT:=$(shell $(HOST_CC) -print-sysroot 2>/dev/null)
endif
endif
ifneq (,$(TOOLCHAIN_GCC_INSTALL_DIR))
SYSROOT_FLAGS:=--gcc-install-dir=$(TOOLCHAIN_GCC_INSTALL_DIR) $(if $(TOOLCHAIN_SYSROOT),--sysroot=$(TOOLCHAIN_SYSROOT))
# That libstdc++ can be newer than the build machine's, which the tools built
# during a stage, e.g. llvm-tblgen, then need to run. Expose that library on its
# own: it can live in a system library directory (e.g. with a distribution GCC),
# whose other libraries would then take precedence over the stage's own, such as
# an older libzstd over the one Julia's zstd executable requires.
STAGE_LIBSTDCXX = $(realpath $(wildcard $(shell $(STAGE0_TOOLS)clang++ $(TARGET_FLAGS) $(SYSROOT_FLAGS) -print-file-name=libstdc++.so.6)))
STAGE_LIBSTDCXX_DIR:=$(STAGE0_BUILD)/libstdcxx
STAGE_PATH_SETUP = rm -rf $(STAGE_LIBSTDCXX_DIR)$(if $(STAGE_LIBSTDCXX), && mkdir -p $(STAGE_LIBSTDCXX_DIR) && ln -s $(STAGE_LIBSTDCXX) $(STAGE_LIBSTDCXX_DIR)/libstdc++.so.6)
STAGE_PATH = LD_LIBRARY_PATH="$(STAGE_LIBSTDCXX_DIR)$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}"
endif
endif

# The toolchain flags go into CC/CXX rather than *FLAGS, because not every part of
# the build uses those: the loader in cli/, for one, compiles and links with
# JCFLAGS/JLDFLAGS only.
TOOLCHAIN_FLAGS = $\
	"CC=$(strip $(STAGE0_TOOLS)clang$(EXE) $(TARGET_FLAGS) $(SYSROOT_FLAGS))" $\
	"BUILD_MACHINE=$(BUILD_MACHINE)" $\
	"CXX=$(strip $(STAGE0_TOOLS)clang++$(EXE) $(TARGET_FLAGS) $(SYSROOT_FLAGS))" $\
	"LD=$(LINKER)" $\
	"AR=$(STAGE0_TOOLS)llvm-ar$(EXE)" $\
	"RANLIB=$(STAGE0_TOOLS)llvm-ranlib$(EXE)" $\
	"CFLAGS+=$(STAGE_CFLAGS)" $\
	"CXXFLAGS+=$(STAGE_CXXFLAGS)" $\
	"LDFLAGS+=$(LINKER_FLAGS) $(LINK_ONLY_FLAGS) $(STAGE_LDFLAGS)" $\
	$(CROSS_FLAGS) $(PLATFORM_FLAGS)

# Shown as `+opt` after the version in the banner and `versioninfo()` of the optimized
# build. Pass the same value to any later make in the stage-2 build (e.g. binary-dist),
# or build_h.jl changes and the system image is rebuilt.
JULIA_BUILD_VARIANT ?= opt
STAGE2_FLAGS = "JULIA_BUILD_VARIANT=$(JULIA_BUILD_VARIANT)"

# Profile data worth keeping, as globs relative to the Julia source root; CI
# archives these so that a build can be reproduced or a regression bisected.
PROFILE_ARTIFACTS :=
ifeq ($(USE_PGO),1)
PROFILE_ARTIFACTS += $(CONTRIB_DIR_REL)/$(notdir $(PGO_PROFILE_DIR))/$(notdir $(PGO_PROFILE_FILE))
endif
ifeq ($(USE_BOLT),1)
PROFILE_ARTIFACTS += $(CONTRIB_DIR_REL)/$(notdir $(BOLT_PROFILE_DIR))/*.merged.fdata
endif

# The whole flow, in one invocation. Every step below is reachable on its own
# too, e.g. to collect extra profile data between stages. The stages announce
# themselves with a line that buildkite renders as a collapsible log section.
ifeq ($(USE_BOLT),1)
all: bolt
else
all: stage2
endif

$(STAGE0_BUILD) $(STAGE0_TARGET_BUILD) $(STAGE1_BUILD) $(STAGE2_BUILD):
	$(MAKE) -C $(JULIA_ROOT) O=$@ configure

# Stage 0: download the toolchain.
ifeq (,$(XC_HOST))
stage0: export USE_BINARYBUILDER_LLVM=1
stage0: | $(STAGE0_BUILD)
	@echo "--- Stage 0: toolchain"
	# The toolchain links against zstd, which macOS does not provide.
	$(MAKE) -C $(STAGE0_BUILD)/deps $(STAGE0_EXTRA_FLAGS) install-clang install-llvm install-lld install-llvm-tools install-zstd $(STAGE0_EXTRA_TARGETS)
else
# When cross-compiling, the downloaded toolchain only runs on the target, and
# the build machine's lacks the target's backend. Build one from source
# instead, and add the target's profile runtime.
stage0: | $(STAGE0_BUILD) $(STAGE0_TARGET_BUILD)
	@echo "--- Stage 0: toolchain"
	mkdir -p $(STAGE0_CONFIG_DIR)
	echo '$(BUILD_SYSROOT_FLAGS)' > $(STAGE0_CONFIG_DIR)/clang.cfg
	echo '$(BUILD_SYSROOT_FLAGS)' > $(STAGE0_CONFIG_DIR)/clang++.cfg
	$(MAKE) -C $(STAGE0_BUILD)/deps XC_HOST= OS=$(BUILD_OS) ARCH= MARCH= MCPU= \
		USE_BINARYBUILDER_LLVM=0 BUILD_LLVM_CLANG=1 BUILD_LLD=1 \
		CLANG_CONFIG_FILE_SYSTEM_DIR=$(STAGE0_CONFIG_DIR) \
		"LLVM_TARGETS=$(STAGE0_LLVM_TARGETS)" "LLVM_ENABLE_PROJECTS=clang;lld" \
		install-llvm install-zstd
	$(MAKE) -C $(STAGE0_TARGET_BUILD)/deps USE_BINARYBUILDER_LLVM=1 install-clang
	mkdir -p $$($(STAGE0_TOOLS)clang -print-resource-dir)/lib/linux
	cp $(STAGE0_TARGET_BUILD)/usr/lib/clang/*/lib/linux/libclang_rt.profile-$(COMPILER_RT_ARCH).a $$($(STAGE0_TOOLS)clang -print-resource-dir)/lib/linux/
endif
ifeq ($(USE_BOLT),1)
	$(MAKE) -C $(STAGE0_BUILD)/deps install-BOLT
endif
	$(STAGE0_POST)
	touch $@

# Order-only for the stages below: refreshed on every run, without making them
# out of date.
stage-path: stage0
	$(STAGE_PATH_SETUP)

# Stage 1: an instrumented Julia. Building its system image and package images
# is the workload that the PGO profile is collected from.
stage1: STAGE_CFLAGS:=$(PGO_GENERATE_FLAGS) $(PGO_COUNTER_FLAGS)
stage1: STAGE_CXXFLAGS:=$(PGO_GENERATE_FLAGS) $(PGO_COUNTER_FLAGS)
stage1: STAGE_LDFLAGS:=$(LTO_FLAGS) $(PGO_GENERATE_FLAGS)
stage1: export USE_BINARYBUILDER_LLVM=0
stage1: stage0 | $(STAGE1_BUILD) stage-path
	@echo "--- Stage 1: instrumented build"
	# Building the dependencies runs instrumented build tools, e.g. llvm-tblgen,
	# which share code with libLLVM but not Julia's workload. Keep their
	# profiles out of the profile directory.
	$(call STAGE_PATH,$(STAGE1_BUILD)) LLVM_PROFILE_FILE=$(call native_path,$(STAGE1_BUILD))/deps-profiles/%m.profraw $(MAKE) -C $(STAGE1_BUILD) $(TOOLCHAIN_FLAGS) $(STAGE1_PLATFORM_FLAGS) JULIA_CPU_TARGET=$(STAGE1_CPU_TARGET) julia-deps
	$(call STAGE_PATH,$(STAGE1_BUILD)) $(MAKE) -C $(STAGE1_BUILD) $(TOOLCHAIN_FLAGS) $(STAGE1_PLATFORM_FLAGS) JULIA_CPU_TARGET=$(STAGE1_CPU_TARGET)
	# Remove the empty profile emitted by the instrumented launcher.
	rm -f $(JULIA_ROOT)/base/default.profraw
	touch $@
	@echo $(AFTER_STAGE1_MESSAGE)

# Merge whatever profiles are around. The glob is expanded by the shell, not by
# make, so that this also works in the same invocation that runs stage 1.
$(PGO_PROFILE_FILE): stage1 $(PGO_PROFRAW_FILES)
	$(LLVM_PROFDATA) merge -output=$@ $(PGO_PROFILE_DIR)/*.profraw

# show top 50 functions
top: $(PGO_PROFILE_FILE)
	$(LLVM_PROFDATA) show --topn=50 $< | $(LLVM_CXXFILT)

# Stage 2: the optimized Julia. With BOLT this stops after the libraries that
# get rewritten (see STAGE2_TARGETS); bolt-train builds the rest.
#
# Emit bitcode and ensure configure checks link it with the stage-0 LTO support.
# The linker flags only matter when linking, so compile-only invocations need
# -Qunused-arguments or -Werror'd compile probes fail (e.g. LLVM's -fPIC check,
# which would silently disable PIC and break the libLLVM link).
stage2 bolt-train: STAGE_CFLAGS:=$(PGO_USE_FLAGS) $(PGO_WARNING_FLAGS) $(LTO_FLAGS) $(LINKER_FLAGS) -Qunused-arguments
stage2 bolt-train: STAGE_CXXFLAGS:=$(PGO_USE_FLAGS) $(PGO_WARNING_FLAGS) $(LTO_FLAGS) $(LINKER_FLAGS) -Qunused-arguments
stage2 bolt-train: STAGE_LDFLAGS:=$(LTO_FLAGS) $(PGO_USE_FLAGS)
stage2 bolt-train: export USE_BINARYBUILDER_LLVM=0
ifeq ($(USE_PGO),1)
stage2: $(PGO_PROFILE_FILE)
else
stage2: stage0
endif
stage2: | $(STAGE2_BUILD) stage-path
	@echo "--- Stage 2: optimized build"
	$(call STAGE_PATH,$(STAGE2_BUILD)) $(MAKE) -C $(STAGE2_BUILD) $(TOOLCHAIN_FLAGS) $(STAGE2_FLAGS) $(BOLT_FLAGS) $(STAGE2_TARGETS)
	touch $@

# Keep a copy of the libraries as linked, since both the instrumentation and
# the final rewrite below start from them.
bolt-originals: stage2
	for link in $(SYMLINKS_TO_OPTIMIZE); do \
		file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \
		abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \
		cp -a $$abs_file "$$abs_file.original"; \
	done
	touch $@

# Match the page alignment of the final rewrite. Preserve library mtimes so
# that make does not rebuild their dependents solely due to instrumentation.
bolt-instrument: bolt-originals
	@echo "--- BOLT: instrument"
	mkdir -p $(BOLT_PROFILE_DIR)
	for link in $(SYMLINKS_TO_OPTIMIZE); do \
		file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \
		abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \
		$(LLVM_BOLT) "$$abs_file.original" -o $$abs_file --instrument --instrumentation-file-append-pid --instrumentation-file="$(BOLT_PROFILE_DIR)/$$file-prof" -no-huge-pages; \
		touch -r "$$abs_file.original" $$abs_file; \
		printf "\n"; \
	done
	touch $@
	@echo $(AFTER_INSTRUMENT_MESSAGE)

# Build the rest of Julia against the instrumented libraries: the system image
# and the stdlib package images are the BOLT training workload.
bolt-train: bolt-instrument | stage-path
	@echo "--- BOLT: training build"
	# Keep $(BOLT_FLAGS) here: anything relinked during this stage must carry
	# --emit-relocs, or the later BOLT rewrite of it fails.
	$(call STAGE_PATH,$(STAGE2_BUILD)) $(MAKE) -C $(STAGE2_BUILD) $(TOOLCHAIN_FLAGS) $(STAGE2_FLAGS) $(BOLT_FLAGS)
	touch $@

bolt-merge: bolt-train $(BOLT_PROFRAW_FILES)
	for link in $(SYMLINKS_TO_OPTIMIZE); do \
		file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \
		$(LLVM_MERGEFDATA) "$(BOLT_PROFILE_DIR)/$$file-prof."[0-9]*.fdata > "$(BOLT_PROFILE_DIR)/$$file-prof.merged.fdata"; \
	done
	touch $@

# Reuse old text segments to reduce library size, except for libjulia-internal:
# BOLT does not fully support its computed gotos (llvm/llvm-project#89117).
# Compress the debug info here, as `make install` leaves BOLT-rewritten libraries
# alone, and with llvm-objcopy: GNU objcopy breaks the segment layout of BOLT's
# output. Preserve mtimes so the rewrite does not trigger dependent builds.
bolt: bolt-merge
	@echo "--- BOLT: rewrite the libraries"
	for link in $(SYMLINKS_TO_OPTIMIZE); do \
		file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \
		abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \
		if [ "$$link" = libjulia-internal.so ]; then extra_args=; else extra_args="--use-old-text $(BOLT_SPLIT_STRATEGY)"; fi; \
		$(LLVM_BOLT) "$$abs_file.original" -data "$(BOLT_PROFILE_DIR)/$$file-prof.merged.fdata" -o $$abs_file $(BOLT_ARGS) $$extra_args; \
		$(if $(filter-out none,$(COMPRESS_DEBUG_SECTIONS)),$(LLVM_OBJCOPY) --compress-debug-sections=$(COMPRESS_DEBUG_SECTIONS) $$abs_file;) \
		touch -r "$$abs_file.original" $$abs_file; \
	done
	touch $@

# The pre-BOLT copies are only needed to redo a rewrite; drop them before
# packaging the build, or restore them to undo the rewrite.
restore-originals:
	for link in $(SYMLINKS_TO_OPTIMIZE); do \
		file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \
		abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \
		cp -pP "$$abs_file.original" $$abs_file; \
	done
	rm -f bolt

delete-originals:
	for link in $(SYMLINKS_TO_OPTIMIZE); do \
		file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \
		rm -f "$(STAGE2_BUILD)/usr/lib/$$file.original"; \
	done

print-profile-artifacts:
	@echo '$(PROFILE_ARTIFACTS)'

clean-profiles: clean-pgo-profiles clean-bolt-profiles

clean-pgo-profiles:
	rm -rf $(PGO_PROFILE_DIR)

clean-bolt-profiles:
	rm -rf $(BOLT_PROFILE_DIR)
	mkdir -p $(BOLT_PROFILE_DIR)
	rm -f bolt-merge bolt

# Only the stamps: this makes the next build redo the stages incrementally.
clean:
	rm -f stage0 stage1 stage2 bolt bolt-originals bolt-instrument bolt-train bolt-merge
	rm -f $(PGO_PROFILE_FILE)
