Skip to content

Commit 2d1e075

Browse files
borsbors
committed
Auto merge of #96285 - flip1995:pk-vfe, r=nagisa
Introduce `-Zvirtual-function-elimination` codegen flag Fixes #68262 This PR adds a codegen flag `-Zvirtual-function-elimination` to enable the VFE optimization in LLVM. To make this work, additonal information has to be added to vtables ([`!vcall_visibility` metadata](https://llvm.org/docs/TypeMetadata.html#vcall-visibility-metadata) and a `typeid` of the trait). Furthermore, instead of just `load`ing functions, the [`llvm.type.checked.load` intrinsic](https://llvm.org/docs/LangRef.html#llvm-type-checked-load-intrinsic) has to be used to map functions to vtables. For technical details of the changes, see the commit messages. I also tested this flag on https://github.com/tock/tock on different boards to verify that this fixes the issue tock/tock#2594. This flag is able to improve the size of the resulting binary by about 8k-9k bytes by removing the unused debug print functions. [Rendered documentation update](https://github.com/flip1995/rust/blob/pk-vfe/src/doc/rustc/src/codegen-options/index.md#virtual-function-elimination)
2 parents 1f34da9 + 195f208 commit 2d1e075

File tree

20 files changed

+431
-32
lines changed

20 files changed

+431
-32
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3695,6 +3695,7 @@ dependencies = [
36953695
"rustc_serialize",
36963696
"rustc_session",
36973697
"rustc_span",
3698+
"rustc_symbol_mangling",
36983699
"rustc_target",
36993700
"smallvec",
37003701
"tracing",

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,16 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
356356
self.context.new_rvalue_from_int(self.int_type, 0)
357357
}
358358

359+
fn type_checked_load(
360+
&mut self,
361+
_llvtable: Self::Value,
362+
_vtable_byte_offset: u64,
363+
_typeid: Self::Value,
364+
) -> Self::Value {
365+
// Unsupported.
366+
self.context.new_rvalue_from_int(self.int_type, 0)
367+
}
368+
359369
fn va_start(&mut self, _va_list: RValue<'gcc>) -> RValue<'gcc> {
360370
unimplemented!();
361371
}

‎compiler/rustc_codegen_llvm/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ rustc-demangle = "0.1.21"
1919
rustc_arena = { path = "../rustc_arena" }
2020
rustc_attr = { path = "../rustc_attr" }
2121
rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
22+
rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
2223
rustc_data_structures = { path = "../rustc_data_structures" }
2324
rustc_errors = { path = "../rustc_errors" }
2425
rustc_fs_util = { path = "../rustc_fs_util" }

‎compiler/rustc_codegen_llvm/src/back/lto.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -586,9 +586,21 @@ pub(crate) fn run_pass_manager(
586586
// LTO-specific optimization passes that LLVM provides.
587587
//
588588
// This code is based off the code found in llvm's LTO code generator:
589-
// tools/lto/LTOCodeGenerator.cpp
589+
// llvm/lib/LTO/LTOCodeGenerator.cpp
590590
debug!("running the pass manager");
591591
unsafe {
592+
if !llvm::LLVMRustHasModuleFlag(
593+
module.module_llvm.llmod(),
594+
"LTOPostLink".as_ptr().cast(),
595+
11,
596+
) {
597+
llvm::LLVMRustAddModuleFlag(
598+
module.module_llvm.llmod(),
599+
llvm::LLVMModFlagBehavior::Error,
600+
"LTOPostLink\0".as_ptr().cast(),
601+
1,
602+
);
603+
}
592604
if llvm_util::should_use_new_llvm_pass_manager(
593605
&config.new_llvm_pass_manager,
594606
&cgcx.target_arch,

‎compiler/rustc_codegen_llvm/src/context.rs

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,15 @@ pub unsafe fn create_module<'ll>(
326326
)
327327
}
328328

329+
if sess.opts.debugging_opts.virtual_function_elimination {
330+
llvm::LLVMRustAddModuleFlag(
331+
llmod,
332+
llvm::LLVMModFlagBehavior::Error,
333+
"Virtual Function Elim\0".as_ptr().cast(),
334+
1,
335+
);
336+
}
337+
329338
llmod
330339
}
331340

@@ -656,6 +665,7 @@ impl<'ll> CodegenCx<'ll, '_> {
656665
let t_isize = self.type_isize();
657666
let t_f32 = self.type_f32();
658667
let t_f64 = self.type_f64();
668+
let t_metadata = self.type_metadata();
659669

660670
ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
661671
ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
@@ -881,11 +891,12 @@ impl<'ll> CodegenCx<'ll, '_> {
881891
ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void);
882892
}
883893

884-
ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1);
894+
ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1);
895+
ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1});
885896

886897
if self.sess().opts.debuginfo != DebugInfo::None {
887-
ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void);
888-
ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void);
898+
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void);
899+
ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void);
889900
}
890901
None
891902
}

‎compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

Lines changed: 92 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,20 +30,21 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE};
3030
use rustc_index::vec::{Idx, IndexVec};
3131
use rustc_middle::bug;
3232
use rustc_middle::mir::{self, GeneratorLayout};
33-
use rustc_middle::ty::layout::LayoutOf;
34-
use rustc_middle::ty::layout::TyAndLayout;
33+
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
3534
use rustc_middle::ty::subst::GenericArgKind;
36-
use rustc_middle::ty::{self, AdtKind, Instance, ParamEnv, Ty, TyCtxt};
37-
use rustc_session::config::{self, DebugInfo};
35+
use rustc_middle::ty::{
36+
self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility,
37+
};
38+
use rustc_session::config::{self, DebugInfo, Lto};
3839
use rustc_span::symbol::Symbol;
3940
use rustc_span::FileName;
40-
use rustc_span::FileNameDisplayPreference;
41-
use rustc_span::{self, SourceFile};
41+
use rustc_span::{self, FileNameDisplayPreference, SourceFile};
42+
use rustc_symbol_mangling::typeid_for_trait_ref;
4243
use rustc_target::abi::{Align, Size};
4344
use smallvec::smallvec;
4445
use tracing::debug;
4546

46-
use libc::{c_longlong, c_uint};
47+
use libc::{c_char, c_longlong, c_uint};
4748
use std::borrow::Cow;
4849
use std::fmt::{self, Write};
4950
use std::hash::{Hash, Hasher};
@@ -1468,6 +1469,84 @@ fn build_vtable_type_di_node<'ll, 'tcx>(
14681469
.di_node
14691470
}
14701471

1472+
fn vcall_visibility_metadata<'ll, 'tcx>(
1473+
cx: &CodegenCx<'ll, 'tcx>,
1474+
ty: Ty<'tcx>,
1475+
trait_ref: Option<PolyExistentialTraitRef<'tcx>>,
1476+
vtable: &'ll Value,
1477+
) {
1478+
enum VCallVisibility {
1479+
Public = 0,
1480+
LinkageUnit = 1,
1481+
TranslationUnit = 2,
1482+
}
1483+
1484+
let Some(trait_ref) = trait_ref else { return };
1485+
1486+
let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty);
1487+
let trait_ref_self = cx.tcx.erase_regions(trait_ref_self);
1488+
let trait_def_id = trait_ref_self.def_id();
1489+
let trait_vis = cx.tcx.visibility(trait_def_id);
1490+
1491+
let cgus = cx.sess().codegen_units();
1492+
let single_cgu = cgus == 1;
1493+
1494+
let lto = cx.sess().lto();
1495+
1496+
// Since LLVM requires full LTO for the virtual function elimination optimization to apply,
1497+
// only the `Lto::Fat` cases are relevant currently.
1498+
let vcall_visibility = match (lto, trait_vis, single_cgu) {
1499+
// If there is not LTO and the visibility in public, we have to assume that the vtable can
1500+
// be seen from anywhere. With multiple CGUs, the vtable is quasi-public.
1501+
(Lto::No | Lto::ThinLocal, Visibility::Public, _)
1502+
| (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => {
1503+
VCallVisibility::Public
1504+
}
1505+
// With LTO and a quasi-public visibility, the usages of the functions of the vtable are
1506+
// all known by the `LinkageUnit`.
1507+
// FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also
1508+
// supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those.
1509+
(Lto::Fat | Lto::Thin, Visibility::Public, _)
1510+
| (
1511+
Lto::ThinLocal | Lto::Thin | Lto::Fat,
1512+
Visibility::Restricted(_) | Visibility::Invisible,
1513+
false,
1514+
) => VCallVisibility::LinkageUnit,
1515+
// If there is only one CGU, private vtables can only be seen by that CGU/translation unit
1516+
// and therefore we know of all usages of functions in the vtable.
1517+
(_, Visibility::Restricted(_) | Visibility::Invisible, true) => {
1518+
VCallVisibility::TranslationUnit
1519+
}
1520+
};
1521+
1522+
let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref);
1523+
1524+
unsafe {
1525+
let typeid = llvm::LLVMMDStringInContext(
1526+
cx.llcx,
1527+
trait_ref_typeid.as_ptr() as *const c_char,
1528+
trait_ref_typeid.as_bytes().len() as c_uint,
1529+
);
1530+
let v = [cx.const_usize(0), typeid];
1531+
llvm::LLVMRustGlobalAddMetadata(
1532+
vtable,
1533+
llvm::MD_type as c_uint,
1534+
llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext(
1535+
cx.llcx,
1536+
v.as_ptr(),
1537+
v.len() as c_uint,
1538+
)),
1539+
);
1540+
let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64));
1541+
let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1);
1542+
llvm::LLVMGlobalSetMetadata(
1543+
vtable,
1544+
llvm::MetadataType::MD_vcall_visibility as c_uint,
1545+
vcall_visibility_metadata,
1546+
);
1547+
}
1548+
}
1549+
14711550
/// Creates debug information for the given vtable, which is for the
14721551
/// given type.
14731552
///
@@ -1478,6 +1557,12 @@ pub fn create_vtable_di_node<'ll, 'tcx>(
14781557
poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
14791558
vtable: &'ll Value,
14801559
) {
1560+
// FIXME(flip1995): The virtual function elimination optimization only works with full LTO in
1561+
// LLVM at the moment.
1562+
if cx.sess().opts.debugging_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat {
1563+
vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable);
1564+
}
1565+
14811566
if cx.dbg_cx.is_none() {
14821567
return;
14831568
}

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
406406
self.call_intrinsic("llvm.type.test", &[bitcast, typeid])
407407
}
408408

409+
fn type_checked_load(
410+
&mut self,
411+
llvtable: &'ll Value,
412+
vtable_byte_offset: u64,
413+
typeid: &'ll Value,
414+
) -> Self::Value {
415+
let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
416+
self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid])
417+
}
418+
409419
fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
410420
self.call_intrinsic("llvm.va_start", &[va_list])
411421
}

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,7 @@ pub enum MetadataType {
442442
MD_nonnull = 11,
443443
MD_align = 17,
444444
MD_type = 19,
445+
MD_vcall_visibility = 28,
445446
MD_noundef = 29,
446447
}
447448

@@ -1067,6 +1068,7 @@ extern "C" {
10671068
pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value);
10681069
pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value);
10691070
pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
1071+
pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
10701072
pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;
10711073

10721074
// Operations on constants of any type
@@ -1080,6 +1082,11 @@ extern "C" {
10801082
Vals: *const &'a Value,
10811083
Count: c_uint,
10821084
) -> &'a Value;
1085+
pub fn LLVMMDNodeInContext2<'a>(
1086+
C: &'a Context,
1087+
Vals: *const &'a Metadata,
1088+
Count: size_t,
1089+
) -> &'a Metadata;
10831090
pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value);
10841091

10851092
// Operations on scalar constants
@@ -1936,6 +1943,7 @@ extern "C" {
19361943
name: *const c_char,
19371944
value: u32,
19381945
);
1946+
pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool;
19391947

19401948
pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value;
19411949

‎compiler/rustc_codegen_ssa/src/meth.rs

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
use crate::traits::*;
22

3-
use rustc_middle::ty::{self, Ty};
3+
use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt};
4+
use rustc_session::config::Lto;
5+
use rustc_symbol_mangling::typeid_for_trait_ref;
46
use rustc_target::abi::call::FnAbi;
57

68
#[derive(Copy, Clone, Debug)]
@@ -15,20 +17,32 @@ impl<'a, 'tcx> VirtualIndex {
1517
self,
1618
bx: &mut Bx,
1719
llvtable: Bx::Value,
20+
ty: Ty<'tcx>,
1821
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1922
) -> Bx::Value {
2023
// Load the data pointer from the object.
21-
debug!("get_fn({:?}, {:?})", llvtable, self);
22-
24+
debug!("get_fn({llvtable:?}, {ty:?}, {self:?})");
2325
let llty = bx.fn_ptr_backend_type(fn_abi);
2426
let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty));
25-
let ptr_align = bx.tcx().data_layout.pointer_align.abi;
26-
let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]);
27-
let ptr = bx.load(llty, gep, ptr_align);
28-
bx.nonnull_metadata(ptr);
29-
// Vtable loads are invariant.
30-
bx.set_invariant_load(ptr);
31-
ptr
27+
28+
if bx.cx().sess().opts.debugging_opts.virtual_function_elimination
29+
&& bx.cx().sess().lto() == Lto::Fat
30+
{
31+
let typeid =
32+
bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty)));
33+
let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes();
34+
let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
35+
let func = bx.extract_value(type_checked_load, 0);
36+
bx.pointercast(func, llty)
37+
} else {
38+
let ptr_align = bx.tcx().data_layout.pointer_align.abi;
39+
let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]);
40+
let ptr = bx.load(llty, gep, ptr_align);
41+
bx.nonnull_metadata(ptr);
42+
// Vtable loads are invariant.
43+
bx.set_invariant_load(ptr);
44+
ptr
45+
}
3246
}
3347

3448
pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
@@ -50,6 +64,24 @@ impl<'a, 'tcx> VirtualIndex {
5064
}
5165
}
5266

67+
fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
68+
for arg in ty.peel_refs().walk() {
69+
if let GenericArgKind::Type(ty) = arg.unpack() {
70+
if let ty::Dynamic(trait_refs, _) = ty.kind() {
71+
return trait_refs[0].map_bound(|trait_ref| match trait_ref {
72+
ExistentialPredicate::Trait(tr) => tr,
73+
ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx),
74+
ExistentialPredicate::AutoTrait(_) => {
75+
bug!("auto traits don't have functions")
76+
}
77+
});
78+
}
79+
}
80+
}
81+
82+
bug!("expected a `dyn Trait` ty, found {ty:?}")
83+
}
84+
5385
/// Creates a dynamic vtable for the given type and vtable origin.
5486
/// This is used only for objects.
5587
///

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
401401
args = &args[..1];
402402
(
403403
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
404-
.get_fn(&mut bx, vtable, &fn_abi),
404+
.get_fn(&mut bx, vtable, ty, &fn_abi),
405405
fn_abi,
406406
)
407407
}
@@ -819,17 +819,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
819819
// the data pointer as the first argument
820820
match op.val {
821821
Pair(data_ptr, meta) => {
822-
llfn = Some(
823-
meth::VirtualIndex::from_index(idx).get_fn(&mut bx, meta, &fn_abi),
824-
);
822+
llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
823+
&mut bx,
824+
meta,
825+
op.layout.ty,
826+
&fn_abi,
827+
));
825828
llargs.push(data_ptr);
826829
continue 'make_args;
827830
}
828831
other => bug!("expected a Pair, got {:?}", other),
829832
}
830833
} else if let Ref(data_ptr, Some(meta), _) = op.val {
831834
// by-value dynamic dispatch
832-
llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(&mut bx, meta, &fn_abi));
835+
llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
836+
&mut bx,
837+
meta,
838+
op.layout.ty,
839+
&fn_abi,
840+
));
833841
llargs.push(data_ptr);
834842
continue;
835843
} else {

0 commit comments

Comments
 (0)