New upstream version 1.53.0+dfsg1

Cargo.toml

@@ -7,6 +7,7 @@ members = [
   "src/rustdoc-json-types",
   "src/tools/cargotest",
   "src/tools/clippy",
+  "src/tools/clippy/clippy_dev",
   "src/tools/compiletest",
   "src/tools/error_index_generator",
   "src/tools/linkchecker",
@@ -45,6 +46,8 @@ exclude = [
   # not all `Cargo.toml` files are available, so we exclude the `x` binary,
   # so it can be invoked before the current checkout is set up.
   "src/tools/x",
+  # stdarch has its own Cargo workspace
+  "library/stdarch",
 ]
 
 [profile.release.package.compiler_builtins]
@@ -88,6 +91,7 @@ object.debug = 0
 # vendored copy.
 [patch."https://github.com/rust-lang/cargo"]
 cargo = { path = "src/tools/cargo" }
+cargo-util = { path = "src/tools/cargo/crates/cargo-util" }
 
 [patch."https://github.com/rust-lang/rustfmt"]
 # Similar to Cargo above we want the RLS to use a vendored version of `rustfmt`

RELEASES.md

@@ -1,3 +1,205 @@
+Version 1.53.0 (2021-06-17)
+============================
+
+Language
+-----------------------
+- [You can now use unicode for identifiers.][83799] This allows multilingual
+  identifiers but still doesn't allow glyphs that are not considered characters
+  such as `◆` or `🦀`. More specifically you can now use any identifier that
+  matches the UAX #31 "Unicode Identifier and Pattern Syntax" standard. This
+  is the same standard as languages like Python, however Rust uses NFC
+  normalization which may be different from other languages.
+- [You can now specify "or patterns" inside pattern matches.][79278]
+  Previously you could only use `|` (OR) on complete patterns. E.g.
+  ```rust
+  let x = Some(2u8);
+  // Before
+  matches!(x, Some(1) | Some(2));
+  // Now
+  matches!(x, Some(1 | 2));
+  ```
+- [Added the `:pat_param` `macro_rules!` matcher.][83386] This matcher
+  has the same semantics as the `:pat` matcher. This is to allow `:pat`
+  to change semantics to being a pattern fragment in a future edition.
+
+Compiler
+-----------------------
+- [Updated the minimum external LLVM version to LLVM 10.][83387]
+- [Added Tier 3\* support for the `wasm64-unknown-unknown` target.][80525]
+- [Improved debuginfo for closures and async functions on Windows MSVC.][83941]
+
+\* Refer to Rust's [platform support page][platform-support-doc] for more
+information on Rust's tiered platform support.
+
+Libraries
+-----------------------
+- [Abort messages will now forward to `android_set_abort_message` on
+  Android platforms when available.][81469]
+- [`slice::IterMut<'_, T>` now implements `AsRef<[T]>`][82771]
+- [Arrays of any length now implement `IntoIterator`.][84147]
+  Currently calling `.into_iter()` as a method on an array will
+  return `impl Iterator<Item=&T>`, but this may change in a
+  future edition to change `Item` to `T`. Calling `IntoIterator::into_iter`
+  directly on arrays will provide `impl Iterator<Item=T>` as expected.
+- [`leading_zeros`, and `trailing_zeros` are now available on all
+  `NonZero` integer types.][84082]
+- [`{f32, f64}::from_str` now parse and print special values
+  (`NaN`, `-0`) according to IEEE RFC 754.][78618]
+- [You can now index into slices using `(Bound<usize>, Bound<usize>)`.][77704]
+- [Add the `BITS` associated constant to all numeric types.][82565]
+
+Stabilised APIs
+---------------
+- [`AtomicBool::fetch_update`]
+- [`AtomicPtr::fetch_update`]
+- [`BTreeMap::retain`]
+- [`BTreeSet::retain`]
+- [`BufReader::seek_relative`]
+- [`DebugStruct::non_exhaustive`]
+- [`Duration::MAX`]
+- [`Duration::ZERO`]
+- [`Duration::is_zero`]
+- [`Duration::saturating_add`]
+- [`Duration::saturating_mul`]
+- [`Duration::saturating_sub`]
+- [`ErrorKind::Unsupported`]
+- [`Option::insert`]
+- [`Ordering::is_eq`]
+- [`Ordering::is_ge`]
+- [`Ordering::is_gt`]
+- [`Ordering::is_le`]
+- [`Ordering::is_lt`]
+- [`Ordering::is_ne`]
+- [`OsStr::is_ascii`]
+- [`OsStr::make_ascii_lowercase`]
+- [`OsStr::make_ascii_uppercase`]
+- [`OsStr::to_ascii_lowercase`]
+- [`OsStr::to_ascii_uppercase`]
+- [`Peekable::peek_mut`]
+- [`Rc::decrement_strong_count`]
+- [`Rc::increment_strong_count`]
+- [`Vec::extend_from_within`]
+- [`array::from_mut`]
+- [`array::from_ref`]
+- [`char::MAX`]
+- [`char::REPLACEMENT_CHARACTER`]
+- [`char::UNICODE_VERSION`]
+- [`char::decode_utf16`]
+- [`char::from_digit`]
+- [`char::from_u32_unchecked`]
+- [`char::from_u32`]
+- [`cmp::max_by_key`]
+- [`cmp::max_by`]
+- [`cmp::min_by_key`]
+- [`cmp::min_by`]
+- [`f32::is_subnormal`]
+- [`f64::is_subnormal`]
+
+Cargo
+-----------------------
+- [Cargo now supports git repositories where the default `HEAD` branch is not
+  "master".][cargo/9392] This also includes a switch to the version 3 `Cargo.lock` format
+  which can handle default branches correctly.
+- [macOS targets now default to `unpacked` split-debuginfo.][cargo/9298]
+- [The `authors` field is no longer included in `Cargo.toml` for new
+  projects.][cargo/9282]
+
+Rustdoc
+-----------------------
+- [Added the `rustdoc::bare_urls` lint that warns when you have URLs
+  without hyperlinks.][81764]
+
+Compatibility Notes
+-------------------
+- [Implement token-based handling of attributes during expansion][82608]
+- [`Ipv4::from_str` will now reject octal format IP addresses in addition
+  to rejecting hexadecimal IP addresses.][83652] The octal format can lead
+  to confusion and potential security vulnerabilities and [is no
+  longer recommended][ietf6943].
+
+
+Internal Only
+-------------
+These changes provide no direct user facing benefits, but represent significant
+improvements to the internals and overall performance of rustc and
+related tools.
+
+- [Rework the `std::sys::windows::alloc` implementation.][83065]
+- [rustdoc: Don't enter an infer_ctxt in get_blanket_impls for impls that aren't blanket impls.][82864]
+- [rustdoc: Only look at blanket impls in `get_blanket_impls`][83681]
+- [Rework rustdoc const type][82873]
+
+[83386]: https://github.com/rust-lang/rust/pull/83386
+[82771]: https://github.com/rust-lang/rust/pull/82771
+[84147]: https://github.com/rust-lang/rust/pull/84147
+[84082]: https://github.com/rust-lang/rust/pull/84082
+[83799]: https://github.com/rust-lang/rust/pull/83799
+[83681]: https://github.com/rust-lang/rust/pull/83681
+[83652]: https://github.com/rust-lang/rust/pull/83652
+[83387]: https://github.com/rust-lang/rust/pull/83387
+[82873]: https://github.com/rust-lang/rust/pull/82873
+[82864]: https://github.com/rust-lang/rust/pull/82864
+[82608]: https://github.com/rust-lang/rust/pull/82608
+[82565]: https://github.com/rust-lang/rust/pull/82565
+[80525]: https://github.com/rust-lang/rust/pull/80525
+[79278]: https://github.com/rust-lang/rust/pull/79278
+[78618]: https://github.com/rust-lang/rust/pull/78618
+[77704]: https://github.com/rust-lang/rust/pull/77704
+[83941]: https://github.com/rust-lang/rust/pull/83941
+[83065]: https://github.com/rust-lang/rust/pull/83065
+[81764]: https://github.com/rust-lang/rust/pull/81764
+[81469]: https://github.com/rust-lang/rust/pull/81469
+[cargo/9298]: https://github.com/rust-lang/cargo/pull/9298
+[cargo/9282]: https://github.com/rust-lang/cargo/pull/9282
+[cargo/9392]: https://github.com/rust-lang/cargo/pull/9392
+[`char::MAX`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.MAX
+[`char::REPLACEMENT_CHARACTER`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.REPLACEMENT_CHARACTER
+[`char::UNICODE_VERSION`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.UNICODE_VERSION
+[`char::decode_utf16`]: https://doc.rust-lang.org/std/primitive.char.html#method.decode_utf16
+[`char::from_u32`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_u32
+[`char::from_u32_unchecked`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_u32_unchecked
+[`char::from_digit`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_digit
+[`AtomicBool::fetch_update`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicBool.html#method.fetch_update
+[`AtomicPtr::fetch_update`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicPtr.html#method.fetch_update
+[`BTreeMap::retain`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.retain
+[`BTreeSet::retain`]: https://doc.rust-lang.org/std/collections/struct.BTreeSet.html#method.retain
+[`BufReader::seek_relative`]: https://doc.rust-lang.org/std/io/struct.BufReader.html#method.seek_relative
+[`DebugStruct::non_exhaustive`]: https://doc.rust-lang.org/std/fmt/struct.DebugStruct.html#method.finish_non_exhaustive
+[`Duration::MAX`]: https://doc.rust-lang.org/std/time/struct.Duration.html#associatedconstant.MAX
+[`Duration::ZERO`]: https://doc.rust-lang.org/std/time/struct.Duration.html#associatedconstant.ZERO
+[`Duration::is_zero`]: https://doc.rust-lang.org/std/time/struct.Duration.html#method.is_zero
+[`Duration::saturating_add`]: https://doc.rust-lang.org/std/time/struct.Duration.html#method.saturating_add
+[`Duration::saturating_mul`]: https://doc.rust-lang.org/std/time/struct.Duration.html#method.saturating_mul
+[`Duration::saturating_sub`]: https://doc.rust-lang.org/std/time/struct.Duration.html#method.saturating_sub
+[`ErrorKind::Unsupported`]: https://doc.rust-lang.org/std/io/enum.ErrorKind.html#variant.Unsupported
+[`Option::insert`]: https://doc.rust-lang.org/std/option/enum.Option.html#method.insert
+[`Ordering::is_eq`]: https://doc.rust-lang.org/std/cmp/enum.Ordering.html#method.is_eq
+[`Ordering::is_ge`]: https://doc.rust-lang.org/std/cmp/enum.Ordering.html#method.is_ge
+[`Ordering::is_gt`]: https://doc.rust-lang.org/std/cmp/enum.Ordering.html#method.is_gt
+[`Ordering::is_le`]: https://doc.rust-lang.org/std/cmp/enum.Ordering.html#method.is_le
+[`Ordering::is_lt`]: https://doc.rust-lang.org/std/cmp/enum.Ordering.html#method.is_lt
+[`Ordering::is_ne`]: https://doc.rust-lang.org/std/cmp/enum.Ordering.html#method.is_ne
+[`OsStr::eq_ignore_ascii_case`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html#method.eq_ignore_ascii_case
+[`OsStr::is_ascii`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html#method.is_ascii
+[`OsStr::make_ascii_lowercase`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html#method.make_ascii_lowercase
+[`OsStr::make_ascii_uppercase`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html#method.make_ascii_uppercase
+[`OsStr::to_ascii_lowercase`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html#method.to_ascii_lowercase
+[`OsStr::to_ascii_uppercase`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html#method.to_ascii_uppercase
+[`Peekable::peek_mut`]: https://doc.rust-lang.org/std/iter/struct.Peekable.html#method.peek_mut
+[`Rc::decrement_strong_count`]: https://doc.rust-lang.org/std/rc/struct.Rc.html#method.increment_strong_count
+[`Rc::increment_strong_count`]: https://doc.rust-lang.org/std/rc/struct.Rc.html#method.increment_strong_count
+[`Vec::extend_from_within`]: https://doc.rust-lang.org/beta/std/vec/struct.Vec.html#method.extend_from_within
+[`array::from_mut`]: https://doc.rust-lang.org/beta/std/array/fn.from_mut.html
+[`array::from_ref`]: https://doc.rust-lang.org/beta/std/array/fn.from_ref.html
+[`cmp::max_by_key`]: https://doc.rust-lang.org/beta/std/cmp/fn.max_by_key.html
+[`cmp::max_by`]: https://doc.rust-lang.org/beta/std/cmp/fn.max_by.html
+[`cmp::min_by_key`]: https://doc.rust-lang.org/beta/std/cmp/fn.min_by_key.html
+[`cmp::min_by`]: https://doc.rust-lang.org/beta/std/cmp/fn.min_by.html
+[`f32::is_subnormal`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_subnormal
+[`f64::is_subnormal`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_subnormal
+[ietf6943]: https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1
+
+
 Version 1.52.1 (2021-05-10)
 ============================
 
@@ -79,7 +281,7 @@ The following previously stable APIs are now `const`.
 Rustdoc
 -------
 - [Rustdoc lints are now treated as a tool lint, meaning that
-  lints are now prefixed with `rustdoc::` (e.g. `#[warn(rustdoc::non_autolinks)]`).][80527]
+  lints are now prefixed with `rustdoc::` (e.g. `#[warn(rustdoc::broken_intra_doc_links)]`).][80527]
   Using the old style is still allowed, and will become a warning in
   a future release.
 - [Rustdoc now supports argument files.][82261]

compiler/rustc/Cargo.toml

@@ -11,12 +11,16 @@ rustc_driver = { path = "../rustc_driver" }
 # crate is intended to be used by codegen backends, which may not be in-tree.
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 
-[dependencies.jemalloc-sys]
+[dependencies.tikv-jemalloc-sys]
-version = '0.3.0'
+version = '0.4.0'
 optional = true
 features = ['unprefixed_malloc_on_supported_platforms']
 
+[dependencies.tikv-jemallocator]
+version = '0.4.0'
+optional = true
+
 [features]
-jemalloc = ['jemalloc-sys']
+jemalloc = ['tikv-jemalloc-sys', 'tikv-jemallocator']
 llvm = ['rustc_driver/llvm']
 max_level_info = ['rustc_driver/max_level_info']

compiler/rustc/src/main.rs

@@ -1,3 +1,16 @@
+// Configure jemalloc as the `global_allocator` when configured. This is
+// so that we use the sized deallocation apis jemalloc provides
+// (namely `sdallocx`).
+//
+// The symbol overrides documented below are also performed so that we can
+// ensure that we use a consistent allocator across the rustc <-> llvm boundary
+#[cfg(feature = "jemalloc")]
+#[global_allocator]
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+#[cfg(feature = "tikv-jemalloc-sys")]
+use tikv_jemalloc_sys as jemalloc_sys;
+
 fn main() {
     // Pull in jemalloc when enabled.
     //
@@ -7,7 +20,7 @@ fn main() {
     // dynamic libraries. That means to pull in jemalloc we actually need to
     // reference allocation symbols one way or another (as this file is the only
     // object code in the rustc executable).
-    #[cfg(feature = "jemalloc-sys")]
+    #[cfg(feature = "tikv-jemalloc-sys")]
     {
         use std::os::raw::{c_int, c_void};
 

compiler/rustc_apfloat/src/ieee.rs

@@ -2273,6 +2273,7 @@ impl Loss {
 mod sig {
     use super::{limbs_for_bits, ExpInt, Limb, Loss, LIMB_BITS};
     use core::cmp::Ordering;
+    use core::iter;
     use core::mem;
 
     pub(super) fn is_all_zeros(limbs: &[Limb]) -> bool {
@@ -2483,7 +2484,7 @@ mod sig {
     pub(super) fn add(a: &mut [Limb], b: &[Limb], mut c: Limb) -> Limb {
         assert!(c <= 1);
 
-        for (a, &b) in a.iter_mut().zip(b) {
+        for (a, &b) in iter::zip(a, b) {
             let (r, overflow) = a.overflowing_add(b);
             let (r, overflow2) = r.overflowing_add(c);
             *a = r;
@@ -2497,7 +2498,7 @@ mod sig {
     pub(super) fn sub(a: &mut [Limb], b: &[Limb], mut c: Limb) -> Limb {
         assert!(c <= 1);
 
-        for (a, &b) in a.iter_mut().zip(b) {
+        for (a, &b) in iter::zip(a, b) {
             let (r, overflow) = a.overflowing_sub(b);
             let (r, overflow2) = r.overflowing_sub(c);
             *a = r;

compiler/rustc_apfloat/src/lib.rs

@@ -33,8 +33,9 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![no_std]
 #![forbid(unsafe_code)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 
 #[macro_use]
 extern crate alloc;

compiler/rustc_arena/src/lib.rs

@@ -236,26 +236,6 @@ impl<T> TypedArena<T> {
         start_ptr
     }
 
-    /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable
-    /// reference to it. Will panic if passed a zero-sized types.
-    ///
-    /// Panics:
-    ///
-    ///  - Zero-sized types
-    ///  - Zero-length slices
-    #[inline]
-    pub fn alloc_slice(&self, slice: &[T]) -> &mut [T]
-    where
-        T: Copy,
-    {
-        unsafe {
-            let len = slice.len();
-            let start_ptr = self.alloc_raw_slice(len);
-            slice.as_ptr().copy_to_nonoverlapping(start_ptr, len);
-            slice::from_raw_parts_mut(start_ptr, len)
-        }
-    }
-
     #[inline]
     pub fn alloc_from_iter<I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
         assert!(mem::size_of::<T>() != 0);

compiler/rustc_ast/src/ast.rs

@@ -100,6 +100,7 @@ pub struct Path {
 }
 
 impl PartialEq<Symbol> for Path {
+    #[inline]
     fn eq(&self, symbol: &Symbol) -> bool {
         self.segments.len() == 1 && { self.segments[0].ident.name == *symbol }
     }
@@ -762,14 +763,6 @@ pub enum Mutability {
 }
 
 impl Mutability {
-    /// Returns `MutMutable` only if both `self` and `other` are mutable.
-    pub fn and(self, other: Self) -> Self {
-        match self {
-            Mutability::Mut => other,
-            Mutability::Not => Mutability::Not,
-        }
-    }
-
     pub fn invert(self) -> Self {
         match self {
             Mutability::Mut => Mutability::Not,
@@ -1353,7 +1346,7 @@ pub enum ExprKind {
     Field(P<Expr>, Ident),
     /// An indexing operation (e.g., `foo[2]`).
     Index(P<Expr>, P<Expr>),
-    /// A range (e.g., `1..2`, `1..`, `..2`, `1..=2`, `..=2`; and `..` in destructuring assingment).
+    /// A range (e.g., `1..2`, `1..`, `..2`, `1..=2`, `..=2`; and `..` in destructuring assignment).
     Range(Option<P<Expr>>, Option<P<Expr>>, RangeLimits),
     /// An underscore, used in destructuring assignment to ignore a value.
     Underscore,
@@ -1722,13 +1715,6 @@ impl FloatTy {
             FloatTy::F64 => sym::f64,
         }
     }
-
-    pub fn bit_width(self) -> u64 {
-        match self {
-            FloatTy::F32 => 32,
-            FloatTy::F64 => 64,
-        }
-    }
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
@@ -1764,29 +1750,6 @@ impl IntTy {
             IntTy::I128 => sym::i128,
         }
     }
-
-    pub fn bit_width(&self) -> Option<u64> {
-        Some(match *self {
-            IntTy::Isize => return None,
-            IntTy::I8 => 8,
-            IntTy::I16 => 16,
-            IntTy::I32 => 32,
-            IntTy::I64 => 64,
-            IntTy::I128 => 128,
-        })
-    }
-
-    pub fn normalize(&self, target_width: u32) -> Self {
-        match self {
-            IntTy::Isize => match target_width {
-                16 => IntTy::I16,
-                32 => IntTy::I32,
-                64 => IntTy::I64,
-                _ => unreachable!(),
-            },
-            _ => *self,
-        }
-    }
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy, Debug)]
@@ -1822,29 +1785,6 @@ impl UintTy {
             UintTy::U128 => sym::u128,
         }
     }
-
-    pub fn bit_width(&self) -> Option<u64> {
-        Some(match *self {
-            UintTy::Usize => return None,
-            UintTy::U8 => 8,
-            UintTy::U16 => 16,
-            UintTy::U32 => 32,
-            UintTy::U64 => 64,
-            UintTy::U128 => 128,
-        })
-    }
-
-    pub fn normalize(&self, target_width: u32) -> Self {
-        match self {
-            UintTy::Usize => match target_width {
-                16 => UintTy::U16,
-                32 => UintTy::U32,
-                64 => UintTy::U64,
-                _ => unreachable!(),
-            },
-            _ => *self,
-        }
-    }
 }
 
 /// A constraint on an associated type (e.g., `A = Bar` in `Foo<A = Bar>` or
@@ -2059,7 +1999,7 @@ pub enum InlineAsmOperand {
         out_expr: Option<P<Expr>>,
     },
     Const {
-        expr: P<Expr>,
+        anon_const: AnonConst,
     },
     Sym {
         expr: P<Expr>,
@@ -2215,9 +2155,6 @@ pub struct FnDecl {
 }
 
 impl FnDecl {
-    pub fn get_self(&self) -> Option<ExplicitSelf> {
-        self.inputs.get(0).and_then(Param::to_self)
-    }
     pub fn has_self(&self) -> bool {
         self.inputs.get(0).map_or(false, Param::is_self)
     }

compiler/rustc_ast/src/ast_like.rs

@@ -1,20 +1,32 @@
 use super::ptr::P;
+use super::token::Nonterminal;
 use super::tokenstream::LazyTokenStream;
 use super::{Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
-use super::{AssocItem, Expr, ForeignItem, Item, Local};
+use super::{AssocItem, Expr, ForeignItem, Item, Local, MacCallStmt};
 use super::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
 use super::{AttrVec, Attribute, Stmt, StmtKind};
 
+use std::fmt::Debug;
+
 /// An `AstLike` represents an AST node (or some wrapper around
 /// and AST node) which stores some combination of attributes
 /// and tokens.
-pub trait AstLike: Sized {
+pub trait AstLike: Sized + Debug {
+    /// This is `true` if this `AstLike` might support 'custom' (proc-macro) inner
+    /// attributes. Attributes like `#![cfg]` and `#![cfg_attr]` are not
+    /// considered 'custom' attributes
+    ///
+    /// If this is `false`, then this `AstLike` definitely does
+    /// not support 'custom' inner attributes, which enables some optimizations
+    /// during token collection.
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool;
     fn attrs(&self) -> &[Attribute];
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
 }
 
 impl<T: AstLike + 'static> AstLike for P<T> {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
     fn attrs(&self) -> &[Attribute] {
         (**self).attrs()
     }
@@ -26,6 +38,55 @@ impl<T: AstLike + 'static> AstLike for P<T> {
     }
 }
 
+impl AstLike for crate::token::Nonterminal {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
+    fn attrs(&self) -> &[Attribute] {
+        match self {
+            Nonterminal::NtItem(item) => item.attrs(),
+            Nonterminal::NtStmt(stmt) => stmt.attrs(),
+            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.attrs(),
+            Nonterminal::NtPat(_)
+            | Nonterminal::NtTy(_)
+            | Nonterminal::NtMeta(_)
+            | Nonterminal::NtPath(_)
+            | Nonterminal::NtVis(_)
+            | Nonterminal::NtTT(_)
+            | Nonterminal::NtBlock(_)
+            | Nonterminal::NtIdent(..)
+            | Nonterminal::NtLifetime(_) => &[],
+        }
+    }
+    fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
+        match self {
+            Nonterminal::NtItem(item) => item.visit_attrs(f),
+            Nonterminal::NtStmt(stmt) => stmt.visit_attrs(f),
+            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.visit_attrs(f),
+            Nonterminal::NtPat(_)
+            | Nonterminal::NtTy(_)
+            | Nonterminal::NtMeta(_)
+            | Nonterminal::NtPath(_)
+            | Nonterminal::NtVis(_)
+            | Nonterminal::NtTT(_)
+            | Nonterminal::NtBlock(_)
+            | Nonterminal::NtIdent(..)
+            | Nonterminal::NtLifetime(_) => {}
+        }
+    }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        match self {
+            Nonterminal::NtItem(item) => item.tokens_mut(),
+            Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
+            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
+            Nonterminal::NtPat(pat) => pat.tokens_mut(),
+            Nonterminal::NtTy(ty) => ty.tokens_mut(),
+            Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
+            Nonterminal::NtPath(path) => path.tokens_mut(),
+            Nonterminal::NtVis(vis) => vis.tokens_mut(),
+            _ => panic!("Called tokens_mut on {:?}", self),
+        }
+    }
+}
+
 fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
     crate::mut_visit::visit_clobber(attrs, |attrs| {
         let mut vec = attrs.into();
@@ -35,6 +96,10 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
 }
 
 impl AstLike for StmtKind {
+    // This might be an `StmtKind::Item`, which contains
+    // an item that supports inner attrs
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
+
     fn attrs(&self) -> &[Attribute] {
         match self {
             StmtKind::Local(local) => local.attrs(),
@@ -66,6 +131,8 @@ impl AstLike for StmtKind {
 }
 
 impl AstLike for Stmt {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = StmtKind::SUPPORTS_CUSTOM_INNER_ATTRS;
+
     fn attrs(&self) -> &[Attribute] {
         self.kind.attrs()
     }
@@ -79,6 +146,8 @@ impl AstLike for Stmt {
 }
 
 impl AstLike for Attribute {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
     fn attrs(&self) -> &[Attribute] {
         &[]
     }
@@ -94,6 +163,8 @@ impl AstLike for Attribute {
 }
 
 impl<T: AstLike> AstLike for Option<T> {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
+
     fn attrs(&self) -> &[Attribute] {
         self.as_ref().map(|inner| inner.attrs()).unwrap_or(&[])
     }
@@ -127,8 +198,13 @@ impl VecOrAttrVec for AttrVec {
 }
 
 macro_rules! derive_has_tokens_and_attrs {
-    ($($ty:path),*) => { $(
+    (
+        const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs:literal;
+        $($ty:path),*
+    ) => { $(
         impl AstLike for $ty {
+            const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs;
+
             fn attrs(&self) -> &[Attribute] {
                 &self.attrs
             }
@@ -140,6 +216,7 @@ macro_rules! derive_has_tokens_and_attrs {
             fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
                 Some(&mut self.tokens)
             }
+
         }
     )* }
 }
@@ -147,6 +224,8 @@ macro_rules! derive_has_tokens_and_attrs {
 macro_rules! derive_has_attrs_no_tokens {
     ($($ty:path),*) => { $(
         impl AstLike for $ty {
+            const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
             fn attrs(&self) -> &[Attribute] {
                 &self.attrs
             }
@@ -165,12 +244,13 @@ macro_rules! derive_has_attrs_no_tokens {
 macro_rules! derive_has_tokens_no_attrs {
     ($($ty:path),*) => { $(
         impl AstLike for $ty {
+            const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
             fn attrs(&self) -> &[Attribute] {
                 &[]
             }
 
             fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
-
             fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
                 Some(&mut self.tokens)
             }
@@ -178,10 +258,18 @@ macro_rules! derive_has_tokens_no_attrs {
     )* }
 }
 
-// These AST nodes support both inert and active
+// These ast nodes support both active and inert attributes,
-// attributes, so they also have tokens.
+// so they have tokens collected to pass to proc macros
 derive_has_tokens_and_attrs! {
-    Item, Expr, Local, AssocItem, ForeignItem
+    // Both `Item` and `AssocItem` can have bodies, which
+    // can contain inner attributes
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
+    Item, AssocItem, ForeignItem
+}
+
+derive_has_tokens_and_attrs! {
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+    Local, MacCallStmt, Expr
 }
 
 // These ast nodes only support inert attributes, so they don't

compiler/rustc_ast/src/attr/mod.rs

@@ -6,7 +6,9 @@ use crate::ast::{Lit, LitKind};
 use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
 use crate::ast::{Path, PathSegment};
 use crate::token::{self, CommentKind, Token};
-use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree, TreeAndSpacing};
+use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
+use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
+use crate::tokenstream::{LazyTokenStream, TokenStream};
 
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::BytePos;
@@ -100,16 +102,7 @@ impl NestedMetaItem {
         self.meta_item().map_or(false, |meta_item| meta_item.is_word())
     }
 
-    /// Returns `true` if `self` is a `MetaItem` and the meta item is a `ValueString`.
+    /// See [`MetaItem::name_value_literal_span`].
-    pub fn is_value_str(&self) -> bool {
-        self.value_str().is_some()
-    }
-
-    /// Returns `true` if `self` is a `MetaItem` and the meta item is a list.
-    pub fn is_meta_item_list(&self) -> bool {
-        self.meta_item_list().is_some()
-    }
-
     pub fn name_value_literal_span(&self) -> Option<Span> {
         self.meta_item()?.name_value_literal_span()
     }
@@ -165,31 +158,6 @@ impl Attribute {
             false
         }
     }
-
-    pub fn is_meta_item_list(&self) -> bool {
-        self.meta_item_list().is_some()
-    }
-
-    /// Indicates if the attribute is a `ValueString`.
-    pub fn is_value_str(&self) -> bool {
-        self.value_str().is_some()
-    }
-
-    /// This is used in case you want the value span instead of the whole attribute. Example:
-    ///
-    /// ```text
-    /// #[doc(alias = "foo")]
-    /// ```
-    ///
-    /// In here, it'll return a span for `"foo"`.
-    pub fn name_value_literal_span(&self) -> Option<Span> {
-        match self.kind {
-            AttrKind::Normal(ref item, _) => {
-                item.meta(self.span).and_then(|meta| meta.name_value_literal_span())
-            }
-            AttrKind::DocComment(..) => None,
-        }
-    }
 }
 
 impl MetaItem {
@@ -236,10 +204,6 @@ impl MetaItem {
         self.path == name
     }
 
-    pub fn is_value_str(&self) -> bool {
-        self.value_str().is_some()
-    }
-
     /// This is used in case you want the value span instead of the whole attribute. Example:
     ///
     /// ```text
@@ -306,14 +270,18 @@ impl Attribute {
         }
     }
 
-    pub fn tokens(&self) -> TokenStream {
+    pub fn tokens(&self) -> AttrAnnotatedTokenStream {
         match self.kind {
             AttrKind::Normal(_, ref tokens) => tokens
                 .as_ref()
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
                 .create_token_stream(),
-            AttrKind::DocComment(comment_kind, data) => TokenStream::from(TokenTree::Token(
+            AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from((
-                Token::new(token::DocComment(comment_kind, self.style, data), self.span),
+                AttrAnnotatedTokenTree::Token(Token::new(
+                    token::DocComment(comment_kind, self.style, data),
+                    self.span,
+                )),
+                Spacing::Alone,
             )),
         }
     }

compiler/rustc_ast/src/expand/mod.rs

@@ -1,3 +1,3 @@
-//! Definitions shared by macros / syntax extensions and e.g. librustc_middle.
+//! Definitions shared by macros / syntax extensions and e.g. `rustc_middle`.
 
 pub mod allocator;

compiler/rustc_ast/src/lib.rs

@@ -10,13 +10,15 @@
 )]
 #![feature(box_syntax)]
 #![feature(box_patterns)]
-#![feature(const_fn)] // For the `transmute` in `P::new`
+#![cfg_attr(bootstrap, feature(const_fn))] // For the `transmute` in `P::new`
+#![cfg_attr(not(bootstrap), feature(const_fn_unsize))] // For the `transmute` in `P::new`
 #![feature(const_fn_transmute)]
 #![feature(const_panic)]
 #![feature(crate_visibility_modifier)]
+#![feature(iter_zip)]
 #![feature(label_break_value)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 #[macro_use]
@@ -58,7 +60,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
-/// instead of implementing everything in librustc_middle.
+/// instead of implementing everything in `rustc_middle`.
 pub trait HashStableContext: rustc_span::HashStableContext {
     fn hash_attr(&mut self, _: &ast::Attribute, hasher: &mut StableHasher);
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -630,6 +630,33 @@ pub fn noop_flat_map_param<T: MutVisitor>(mut param: Param, vis: &mut T) -> Smal
     smallvec![param]
 }
 
+// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
+pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
+    match tt {
+        AttrAnnotatedTokenTree::Token(token) => {
+            visit_token(token, vis);
+        }
+        AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
+            vis.visit_span(open);
+            vis.visit_span(close);
+            visit_attr_annotated_tts(tts, vis);
+        }
+        AttrAnnotatedTokenTree::Attributes(data) => {
+            for attr in &mut *data.attrs {
+                match &mut attr.kind {
+                    AttrKind::Normal(_, attr_tokens) => {
+                        visit_lazy_tts(attr_tokens, vis);
+                    }
+                    AttrKind::DocComment(..) => {
+                        vis.visit_span(&mut attr.span);
+                    }
+                }
+            }
+            visit_lazy_tts_opt_mut(Some(&mut data.tokens), vis);
+        }
+    }
+}
+
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
     match tt {
@@ -652,16 +679,30 @@ pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T)
     }
 }
 
-pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
+pub fn visit_attr_annotated_tts<T: MutVisitor>(
-    if vis.token_visiting_enabled() {
+    AttrAnnotatedTokenStream(tts): &mut AttrAnnotatedTokenStream,
-        visit_opt(lazy_tts, |lazy_tts| {
+    vis: &mut T,
-            let mut tts = lazy_tts.create_token_stream();
+) {
-            visit_tts(&mut tts, vis);
+    if vis.token_visiting_enabled() && !tts.is_empty() {
-            *lazy_tts = LazyTokenStream::new(tts);
+        let tts = Lrc::make_mut(tts);
-        })
+        visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis));
     }
 }
 
+pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(lazy_tts: Option<&mut LazyTokenStream>, vis: &mut T) {
+    if vis.token_visiting_enabled() {
+        if let Some(lazy_tts) = lazy_tts {
+            let mut tts = lazy_tts.create_token_stream();
+            visit_attr_annotated_tts(&mut tts, vis);
+            *lazy_tts = LazyTokenStream::new(tts);
+        }
+    }
+}
+
+pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
+    visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis);
+}
+
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 // Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
 // In practice the ident part is not actually used by specific visitors right now,
@@ -1252,7 +1293,6 @@ pub fn noop_visit_expr<T: MutVisitor>(
                 match op {
                     InlineAsmOperand::In { expr, .. }
                     | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Const { expr, .. }
                     | InlineAsmOperand::Sym { expr, .. } => vis.visit_expr(expr),
                     InlineAsmOperand::Out { expr, .. } => {
                         if let Some(expr) = expr {
@@ -1265,6 +1305,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
                             vis.visit_expr(out_expr);
                         }
                     }
+                    InlineAsmOperand::Const { anon_const, .. } => vis.visit_anon_const(anon_const),
                 }
             }
         }

compiler/rustc_ast/src/token.rs

@@ -688,16 +688,12 @@ pub enum NonterminalKind {
     Item,
     Block,
     Stmt,
-    Pat2018 {
+    PatParam {
-        /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the
+        /// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the
-        /// edition of the span. This is used for diagnostics.
-        inferred: bool,
-    },
-    Pat2021 {
-        /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the
         /// edition of the span. This is used for diagnostics.
         inferred: bool,
     },
+    PatWithOr,
     Expr,
     Ty,
     Ident,
@@ -722,12 +718,11 @@ impl NonterminalKind {
             sym::stmt => NonterminalKind::Stmt,
             sym::pat => match edition() {
                 Edition::Edition2015 | Edition::Edition2018 => {
-                    NonterminalKind::Pat2018 { inferred: true }
+                    NonterminalKind::PatParam { inferred: true }
                 }
-                Edition::Edition2021 => NonterminalKind::Pat2021 { inferred: true },
+                Edition::Edition2021 => NonterminalKind::PatWithOr,
             },
-            sym::pat2018 => NonterminalKind::Pat2018 { inferred: false },
+            sym::pat_param => NonterminalKind::PatParam { inferred: false },
-            sym::pat2021 => NonterminalKind::Pat2021 { inferred: false },
             sym::expr => NonterminalKind::Expr,
             sym::ty => NonterminalKind::Ty,
             sym::ident => NonterminalKind::Ident,
@@ -745,10 +740,8 @@ impl NonterminalKind {
             NonterminalKind::Item => sym::item,
             NonterminalKind::Block => sym::block,
             NonterminalKind::Stmt => sym::stmt,
-            NonterminalKind::Pat2018 { inferred: false } => sym::pat2018,
+            NonterminalKind::PatParam { inferred: false } => sym::pat_param,
-            NonterminalKind::Pat2021 { inferred: false } => sym::pat2021,
+            NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
-            NonterminalKind::Pat2018 { inferred: true }
-            | NonterminalKind::Pat2021 { inferred: true } => sym::pat,
             NonterminalKind::Expr => sym::expr,
             NonterminalKind::Ty => sym::ty,
             NonterminalKind::Ident => sym::ident,

compiler/rustc_ast/src/tokenstream.rs

@@ -14,6 +14,7 @@
 //! ownership of the original.
 
 use crate::token::{self, DelimToken, Token, TokenKind};
+use crate::AttrVec;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{self, Lrc};
@@ -89,10 +90,6 @@ impl TokenTree {
         }
     }
 
-    pub fn joint(self) -> TokenStream {
-        TokenStream::new(vec![(self, Spacing::Joint)])
-    }
-
     pub fn token(kind: TokenKind, span: Span) -> TokenTree {
         TokenTree::Token(Token::new(kind, span))
     }
@@ -127,11 +124,11 @@ where
 }
 
 pub trait CreateTokenStream: sync::Send + sync::Sync {
-    fn create_token_stream(&self) -> TokenStream;
+    fn create_token_stream(&self) -> AttrAnnotatedTokenStream;
 }
 
-impl CreateTokenStream for TokenStream {
+impl CreateTokenStream for AttrAnnotatedTokenStream {
-    fn create_token_stream(&self) -> TokenStream {
+    fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
         self.clone()
     }
 }
@@ -147,14 +144,14 @@ impl LazyTokenStream {
         LazyTokenStream(Lrc::new(Box::new(inner)))
     }
 
-    pub fn create_token_stream(&self) -> TokenStream {
+    pub fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
         self.0.create_token_stream()
     }
 }
 
 impl fmt::Debug for LazyTokenStream {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Debug::fmt("LazyTokenStream", f)
+        write!(f, "LazyTokenStream({:?})", self.create_token_stream())
     }
 }
 
@@ -177,12 +174,151 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
     }
 }
 
+/// A `AttrAnnotatedTokenStream` is similar to a `TokenStream`, but with extra
+/// information about the tokens for attribute targets. This is used
+/// during expansion to perform early cfg-expansion, and to process attributes
+/// during proc-macro invocations.
+#[derive(Clone, Debug, Default, Encodable, Decodable)]
+pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>);
+
+/// Like `TokenTree`, but for `AttrAnnotatedTokenStream`
+#[derive(Clone, Debug, Encodable, Decodable)]
+pub enum AttrAnnotatedTokenTree {
+    Token(Token),
+    Delimited(DelimSpan, DelimToken, AttrAnnotatedTokenStream),
+    /// Stores the attributes for an attribute target,
+    /// along with the tokens for that attribute target.
+    /// See `AttributesData` for more information
+    Attributes(AttributesData),
+}
+
+impl AttrAnnotatedTokenStream {
+    pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream {
+        AttrAnnotatedTokenStream(Lrc::new(tokens))
+    }
+
+    /// Converts this `AttrAnnotatedTokenStream` to a plain `TokenStream
+    /// During conversion, `AttrAnnotatedTokenTree::Attributes` get 'flattened'
+    /// back to a `TokenStream` of the form `outer_attr attr_target`.
+    /// If there are inner attributes, they are inserted into the proper
+    /// place in the attribute target tokens.
+    pub fn to_tokenstream(&self) -> TokenStream {
+        let trees: Vec<_> = self
+            .0
+            .iter()
+            .flat_map(|tree| match &tree.0 {
+                AttrAnnotatedTokenTree::Token(inner) => {
+                    smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter()
+                }
+                AttrAnnotatedTokenTree::Delimited(span, delim, stream) => smallvec![(
+                    TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),
+                    tree.1,
+                )]
+                .into_iter(),
+                AttrAnnotatedTokenTree::Attributes(data) => {
+                    let mut outer_attrs = Vec::new();
+                    let mut inner_attrs = Vec::new();
+                    let attrs: Vec<_> = data.attrs.clone().into();
+                    for attr in attrs {
+                        match attr.style {
+                            crate::AttrStyle::Outer => {
+                                assert!(
+                                    inner_attrs.len() == 0,
+                                    "Found outer attribute {:?} after inner attrs {:?}",
+                                    attr,
+                                    inner_attrs
+                                );
+                                outer_attrs.push(attr);
+                            }
+                            crate::AttrStyle::Inner => {
+                                inner_attrs.push(attr);
+                            }
+                        }
+                    }
+
+                    let mut target_tokens: Vec<_> = data
+                        .tokens
+                        .create_token_stream()
+                        .to_tokenstream()
+                        .0
+                        .iter()
+                        .cloned()
+                        .collect();
+                    if !inner_attrs.is_empty() {
+                        let mut found = false;
+                        // Check the last two trees (to account for a trailing semi)
+                        for (tree, _) in target_tokens.iter_mut().rev().take(2) {
+                            if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
+                                // Inner attributes are only supported on extern blocks, functions, impls,
+                                // and modules. All of these have their inner attributes placed at
+                                // the beginning of the rightmost outermost braced group:
+                                // e.g. fn foo() { #![my_attr} }
+                                //
+                                // Therefore, we can insert them back into the right location
+                                // without needing to do any extra position tracking.
+                                //
+                                // Note: Outline modules are an exception - they can
+                                // have attributes like `#![my_attr]` at the start of a file.
+                                // Support for custom attributes in this position is not
+                                // properly implemented - we always synthesize fake tokens,
+                                // so we never reach this code.
+
+                                let mut builder = TokenStreamBuilder::new();
+                                for inner_attr in &inner_attrs {
+                                    builder.push(inner_attr.tokens().to_tokenstream());
+                                }
+                                builder.push(delim_tokens.clone());
+                                *tree = TokenTree::Delimited(*span, *delim, builder.build());
+                                found = true;
+                                break;
+                            }
+                        }
+
+                        assert!(
+                            found,
+                            "Failed to find trailing delimited group in: {:?}",
+                            target_tokens
+                        );
+                    }
+                    let mut flat: SmallVec<[_; 1]> = SmallVec::new();
+                    for attr in outer_attrs {
+                        // FIXME: Make this more efficient
+                        flat.extend(attr.tokens().to_tokenstream().0.clone().iter().cloned());
+                    }
+                    flat.extend(target_tokens);
+                    flat.into_iter()
+                }
+            })
+            .collect();
+        TokenStream::new(trees)
+    }
+}
+
+/// Stores the tokens for an attribute target, along
+/// with its attributes.
+///
+/// This is constructed during parsing when we need to capture
+/// tokens.
+///
+/// For example, `#[cfg(FALSE)] struct Foo {}` would
+/// have an `attrs` field containing the `#[cfg(FALSE)]` attr,
+/// and a `tokens` field storing the (unparesd) tokens `struct Foo {}`
+#[derive(Clone, Debug, Encodable, Decodable)]
+pub struct AttributesData {
+    /// Attributes, both outer and inner.
+    /// These are stored in the original order that they were parsed in.
+    pub attrs: AttrVec,
+    /// The underlying tokens for the attribute target that `attrs`
+    /// are applied to
+    pub tokens: LazyTokenStream,
+}
+
 /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
 ///
 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
 /// instead of a representation of the abstract syntax tree.
 /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
-/// backwards compatability.
+/// backwards compatibility.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>);
 
@@ -239,6 +375,12 @@ impl TokenStream {
     }
 }
 
+impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream {
+    fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream {
+        AttrAnnotatedTokenStream::new(vec![(tree, spacing)])
+    }
+}
+
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
         TokenStream::new(vec![(tree, Spacing::Alone)])
@@ -278,14 +420,6 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn span(&self) -> Option<Span> {
-        match &**self.0 {
-            [] => None,
-            [(tt, _)] => Some(tt.span()),
-            [(tt_start, _), .., (tt_end, _)] => Some(tt_start.span().to(tt_end.span())),
-        }
-    }
-
     pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::default(),
@@ -325,10 +459,6 @@ impl TokenStream {
         }
     }
 
-    pub fn trees_ref(&self) -> CursorRef<'_> {
-        CursorRef::new(self)
-    }
-
     pub fn trees(&self) -> Cursor {
         self.clone().into_trees()
     }
@@ -341,7 +471,7 @@ impl TokenStream {
     pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
         let mut t1 = self.trees();
         let mut t2 = other.trees();
-        for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
+        for (t1, t2) in iter::zip(&mut t1, &mut t2) {
             if !t1.eq_unspanned(&t2) {
                 return false;
             }
@@ -427,10 +557,6 @@ pub struct CursorRef<'t> {
 }
 
 impl<'t> CursorRef<'t> {
-    fn new(stream: &TokenStream) -> CursorRef<'_> {
-        CursorRef { stream, index: 0 }
-    }
-
     fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> {
         self.stream.0.get(self.index).map(|tree| {
             self.index += 1;
@@ -477,6 +603,10 @@ impl Cursor {
         }
     }
 
+    pub fn index(&self) -> usize {
+        self.index
+    }
+
     pub fn append(&mut self, new_stream: TokenStream) {
         if new_stream.is_empty() {
             return;

compiler/rustc_ast/src/visit.rs

@@ -835,7 +835,6 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
                 match op {
                     InlineAsmOperand::In { expr, .. }
                     | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Const { expr, .. }
                     | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr),
                     InlineAsmOperand::Out { expr, .. } => {
                         if let Some(expr) = expr {
@@ -848,6 +847,9 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
                             visitor.visit_expr(out_expr);
                         }
                     }
+                    InlineAsmOperand::Const { anon_const, .. } => {
+                        visitor.visit_anon_const(anon_const)
+                    }
                 }
             }
         }

compiler/rustc_ast_lowering/src/expr.rs

@@ -1411,9 +1411,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             out_expr: out_expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
                         }
                     }
-                    InlineAsmOperand::Const { ref expr } => {
+                    InlineAsmOperand::Const { ref anon_const } => hir::InlineAsmOperand::Const {
-                        hir::InlineAsmOperand::Const { expr: self.lower_expr_mut(expr) }
+                        anon_const: self.lower_anon_const(anon_const),
-                    }
+                    },
                     InlineAsmOperand::Sym { ref expr } => {
                         hir::InlineAsmOperand::Sym { expr: self.lower_expr_mut(expr) }
                     }
@@ -1499,46 +1499,64 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // previous iteration.
                 required_features.clear();
 
-                // Validate register classes against currently enabled target
-                // features. We check that at least one type is available for
-                // the current target.
                 let reg_class = reg.reg_class();
                 if reg_class == asm::InlineAsmRegClass::Err {
                     continue;
                 }
-                for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) {
+
-                    if let Some(feature) = feature {
+                // We ignore target feature requirements for clobbers: if the
-                        if self.sess.target_features.contains(&Symbol::intern(feature)) {
+                // feature is disabled then the compiler doesn't care what we
+                // do with the registers.
+                //
+                // Note that this is only possible for explicit register
+                // operands, which cannot be used in the asm string.
+                let is_clobber = matches!(
+                    op,
+                    hir::InlineAsmOperand::Out {
+                        reg: asm::InlineAsmRegOrRegClass::Reg(_),
+                        late: _,
+                        expr: None
+                    }
+                );
+
+                if !is_clobber {
+                    // Validate register classes against currently enabled target
+                    // features. We check that at least one type is available for
+                    // the current target.
+                    for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) {
+                        if let Some(feature) = feature {
+                            if self.sess.target_features.contains(&Symbol::intern(feature)) {
+                                required_features.clear();
+                                break;
+                            } else {
+                                required_features.push(feature);
+                            }
+                        } else {
                             required_features.clear();
                             break;
-                        } else {
-                            required_features.push(feature);
                         }
-                    } else {
-                        required_features.clear();
-                        break;
                     }
-                }
+                    // We are sorting primitive strs here and can use unstable sort here
-                // We are sorting primitive strs here and can use unstable sort here
+                    required_features.sort_unstable();
-                required_features.sort_unstable();
+                    required_features.dedup();
-                required_features.dedup();
+                    match &required_features[..] {
-                match &required_features[..] {
+                        [] => {}
-                    [] => {}
+                        [feature] => {
-                    [feature] => {
+                            let msg = format!(
-                        let msg = format!(
+                                "register class `{}` requires the `{}` target feature",
-                            "register class `{}` requires the `{}` target feature",
+                                reg_class.name(),
-                            reg_class.name(),
+                                feature
-                            feature
+                            );
-                        );
+                            sess.struct_span_err(op_sp, &msg).emit();
-                        sess.struct_span_err(op_sp, &msg).emit();
+                        }
-                    }
+                        features => {
-                    features => {
+                            let msg = format!(
-                        let msg = format!(
+                                "register class `{}` requires at least one target feature: {}",
-                            "register class `{}` requires at least one target feature: {}",
+                                reg_class.name(),
-                            reg_class.name(),
+                                features.join(", ")
-                            features.join(", ")
+                            );
-                        );
+                            sess.struct_span_err(op_sp, &msg).emit();
-                        sess.struct_span_err(op_sp, &msg).emit();
+                        }
                     }
                 }
 

compiler/rustc_ast_lowering/src/item.rs

@@ -18,6 +18,7 @@ use rustc_target::spec::abi;
 use smallvec::{smallvec, SmallVec};
 use tracing::debug;
 
+use std::iter;
 use std::mem;
 
 pub(super) struct ItemLowerer<'a, 'lowering, 'hir> {
@@ -206,7 +207,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             UseTreeKind::Glob => {}
             UseTreeKind::Simple(_, id1, id2) => {
                 for (_, &id) in
-                    self.expect_full_res_from_use(base_id).skip(1).zip([id1, id2].iter())
+                    iter::zip(self.expect_full_res_from_use(base_id).skip(1), &[id1, id2])
                 {
                     vec.push(id);
                 }
@@ -537,7 +538,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // won't be dealing with macros in the rest of the compiler.
                 // Essentially a single `use` which imports two names is desugared into
                 // two imports.
-                for (res, &new_node_id) in resolutions.zip([id1, id2].iter()) {
+                for (res, &new_node_id) in iter::zip(resolutions, &[id1, id2]) {
                     let ident = *ident;
                     let mut path = path.clone();
                     for seg in &mut path.segments {
@@ -835,9 +836,17 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)))
             }
             AssocItemKind::Fn(box FnKind(_, ref sig, ref generics, Some(ref body))) => {
-                let body_id = self.lower_fn_body_block(i.span, &sig.decl, Some(body));
+                let asyncness = sig.header.asyncness;
-                let (generics, sig) =
+                let body_id =
-                    self.lower_method_sig(generics, sig, trait_item_def_id, false, None, i.id);
+                    self.lower_maybe_async_body(i.span, &sig.decl, asyncness, Some(&body));
+                let (generics, sig) = self.lower_method_sig(
+                    generics,
+                    sig,
+                    trait_item_def_id,
+                    false,
+                    asyncness.opt_return_id(),
+                    i.id,
+                );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)))
             }
             AssocItemKind::TyAlias(box TyAliasKind(_, ref generics, ref bounds, ref default)) => {

compiler/rustc_ast_lowering/src/lib.rs

@@ -31,13 +31,14 @@
 //! in the HIR, especially for multiple identifiers.
 
 #![feature(crate_visibility_modifier)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_patterns)]
+#![feature(iter_zip)]
 #![recursion_limit = "256"]
 
 use rustc_ast::node_id::NodeMap;
-use rustc_ast::token::{self, DelimToken, Nonterminal, Token};
+use rustc_ast::token::{self, Token};
-use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, DelimSpan, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream, TokenTree};
 use rustc_ast::visit::{self, AssocCtxt, Visitor};
 use rustc_ast::walk_list;
 use rustc_ast::{self as ast, *};
@@ -55,7 +56,7 @@ use rustc_hir::{ConstArg, GenericArg, ParamName};
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_session::lint::builtin::{BARE_TRAIT_OBJECTS, MISSING_ABI};
 use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer};
-use rustc_session::parse::ParseSess;
+use rustc_session::utils::{FlattenNonterminals, NtToTokenstream};
 use rustc_session::Session;
 use rustc_span::hygiene::ExpnId;
 use rustc_span::source_map::{respan, DesugaringKind};
@@ -92,7 +93,7 @@ struct LoweringContext<'a, 'hir: 'a> {
 
     /// HACK(Centril): there is a cyclic dependency between the parser and lowering
     /// if we don't have this function pointer. To avoid that dependency so that
-    /// librustc_middle is independent of the parser, we use dynamic dispatch here.
+    /// `rustc_middle` is independent of the parser, we use dynamic dispatch here.
     nt_to_tokenstream: NtToTokenstream,
 
     /// Used to allocate HIR nodes.
@@ -212,8 +213,6 @@ pub trait ResolverAstLowering {
     ) -> LocalDefId;
 }
 
-type NtToTokenstream = fn(&Nonterminal, &ParseSess, CanSynthesizeMissingTokens) -> TokenStream;
-
 /// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
 /// and if so, what meaning it has.
 #[derive(Debug)]
@@ -402,42 +401,6 @@ enum AnonymousLifetimeMode {
     PassThrough,
 }
 
-struct TokenStreamLowering<'a> {
-    parse_sess: &'a ParseSess,
-    synthesize_tokens: CanSynthesizeMissingTokens,
-    nt_to_tokenstream: NtToTokenstream,
-}
-
-impl<'a> TokenStreamLowering<'a> {
-    fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
-        tokens.into_trees().flat_map(|tree| self.lower_token_tree(tree).into_trees()).collect()
-    }
-
-    fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
-        match tree {
-            TokenTree::Token(token) => self.lower_token(token),
-            TokenTree::Delimited(span, delim, tts) => {
-                TokenTree::Delimited(span, delim, self.lower_token_stream(tts)).into()
-            }
-        }
-    }
-
-    fn lower_token(&mut self, token: Token) -> TokenStream {
-        match token.kind {
-            token::Interpolated(nt) => {
-                let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
-                TokenTree::Delimited(
-                    DelimSpan::from_single(token.span),
-                    DelimToken::NoDelim,
-                    self.lower_token_stream(tts),
-                )
-                .into()
-            }
-            _ => TokenTree::Token(token).into(),
-        }
-    }
-}
-
 impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn lower_crate(mut self, c: &Crate) -> hir::Crate<'hir> {
         /// Full-crate AST visitor that inserts into a fresh
@@ -520,10 +483,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         }
                         self.visit_fn_ret_ty(&f.decl.output)
                     }
-                    TyKind::ImplTrait(def_node_id, _) => {
-                        self.lctx.allocate_hir_id_counter(def_node_id);
-                        visit::walk_ty(self, t);
-                    }
                     _ => visit::walk_ty(self, t),
                 }
             }
@@ -572,7 +531,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         }
 
         hir::Crate {
-            item: hir::CrateItem { module, span: c.span },
+            item: module,
             exported_macros: self.arena.alloc_from_iter(self.exported_macros),
             non_exported_macro_attrs: self.arena.alloc_from_iter(self.non_exported_macro_attrs),
             items: self.items,
@@ -1040,12 +999,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     }
                 }
 
-                let tokens = TokenStreamLowering {
+                let tokens = FlattenNonterminals {
                     parse_sess: &self.sess.parse_sess,
                     synthesize_tokens: CanSynthesizeMissingTokens::Yes,
                     nt_to_tokenstream: self.nt_to_tokenstream,
                 }
-                .lower_token(token.clone());
+                .process_token(token.clone());
                 MacArgs::Eq(eq_span, unwrap_single_token(self.sess, tokens, token.span))
             }
         }
@@ -1056,12 +1015,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         tokens: TokenStream,
         synthesize_tokens: CanSynthesizeMissingTokens,
     ) -> TokenStream {
-        TokenStreamLowering {
+        FlattenNonterminals {
             parse_sess: &self.sess.parse_sess,
             synthesize_tokens,
             nt_to_tokenstream: self.nt_to_tokenstream,
         }
-        .lower_token_stream(tokens)
+        .process_token_stream(tokens)
     }
 
     /// Given an associated type constraint like one of these:
@@ -1431,14 +1390,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         // Add a definition for the in-band `Param`.
                         let def_id = self.resolver.local_def_id(def_node_id);
 
-                        self.allocate_hir_id_counter(def_node_id);
+                        let hir_bounds = self.lower_param_bounds(
-
+                            bounds,
-                        let hir_bounds = self.with_hir_id_owner(def_node_id, |this| {
+                            ImplTraitContext::Universal(in_band_ty_params, parent_def_id),
-                            this.lower_param_bounds(
+                        );
-                                bounds,
-                                ImplTraitContext::Universal(in_band_ty_params, parent_def_id),
-                            )
-                        });
                         // Set the name to `impl Bound1 + Bound2`.
                         let ident = Ident::from_str_and_span(&pprust::ty_to_string(t), span);
                         in_band_ty_params.push(hir::GenericParam {
@@ -2266,13 +2221,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                 let kind = hir::GenericParamKind::Type {
                     default: default.as_ref().map(|x| {
-                        self.lower_ty(
+                        self.lower_ty(x, ImplTraitContext::Disallowed(ImplTraitPosition::Other))
-                            x,
-                            ImplTraitContext::OtherOpaqueTy {
-                                capturable_lifetimes: &mut FxHashSet::default(),
-                                origin: hir::OpaqueTyOrigin::Misc,
-                            },
-                        )
                     }),
                     synthetic: param
                         .attrs
@@ -2290,7 +2239,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         this.lower_ty(&ty, ImplTraitContext::disallowed())
                     });
                 let default = default.as_ref().map(|def| self.lower_anon_const(def));
-
                 (hir::ParamName::Plain(param.ident), hir::GenericParamKind::Const { ty, default })
             }
         };

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -532,6 +532,25 @@ impl<'a> AstValidator<'a> {
         }
     }
 
+    /// An item in `extern { ... }` cannot use non-ascii identifier.
+    fn check_foreign_item_ascii_only(&self, ident: Ident) {
+        let symbol_str = ident.as_str();
+        if !symbol_str.is_ascii() {
+            let n = 83942;
+            self.err_handler()
+                .struct_span_err(
+                    ident.span,
+                    "items in `extern` blocks cannot use non-ascii identifiers",
+                )
+                .span_label(self.current_extern_span(), "in this `extern` block")
+                .note(&format!(
+                    "This limitation may be lifted in the future; see issue #{} <https://github.com/rust-lang/rust/issues/{}> for more information",
+                    n, n,
+                ))
+                .emit();
+        }
+    }
+
     /// Reject C-varadic type unless the function is foreign,
     /// or free and `unsafe extern "C"` semantically.
     fn check_c_varadic_type(&self, fk: FnKind<'a>) {
@@ -592,7 +611,7 @@ impl<'a> AstValidator<'a> {
             self.session,
             ident.span,
             E0754,
-            "trying to load file for module `{}` with non ascii identifer name",
+            "trying to load file for module `{}` with non-ascii identifier name",
             ident.name
         )
         .help("consider using `#[path]` attribute to specify filesystem path")
@@ -735,7 +754,7 @@ fn validate_generic_param_order(
             GenericParamKind::Type { default: _ } => (ParamKindOrd::Type, ident),
             GenericParamKind::Const { ref ty, kw_span: _, default: _ } => {
                 let ty = pprust::ty_to_string(ty);
-                let unordered = sess.features_untracked().const_generics;
+                let unordered = sess.features_untracked().unordered_const_ty_params();
                 (ParamKindOrd::Const { unordered }, Some(format!("const {}: {}", param.ident, ty)))
             }
         };
@@ -1103,15 +1122,18 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self.check_defaultness(fi.span, *def);
                 self.check_foreign_fn_bodyless(fi.ident, body.as_deref());
                 self.check_foreign_fn_headerless(fi.ident, fi.span, sig.header);
+                self.check_foreign_item_ascii_only(fi.ident);
             }
             ForeignItemKind::TyAlias(box TyAliasKind(def, generics, bounds, body)) => {
                 self.check_defaultness(fi.span, *def);
                 self.check_foreign_kind_bodyless(fi.ident, "type", body.as_ref().map(|b| b.span));
                 self.check_type_no_bounds(bounds, "`extern` blocks");
                 self.check_foreign_ty_genericless(generics);
+                self.check_foreign_item_ascii_only(fi.ident);
             }
             ForeignItemKind::Static(_, _, body) => {
                 self.check_foreign_kind_bodyless(fi.ident, "static", body.as_ref().map(|b| b.span));
+                self.check_foreign_item_ascii_only(fi.ident);
             }
             ForeignItemKind::MacCall(..) => {}
         }
@@ -1150,20 +1172,23 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
     }
 
     fn visit_generics(&mut self, generics: &'a Generics) {
-        let mut prev_ty_default = None;
+        let cg_defaults = self.session.features_untracked().const_generics_defaults;
+
+        let mut prev_param_default = None;
         for param in &generics.params {
             match param.kind {
                 GenericParamKind::Lifetime => (),
-                GenericParamKind::Type { default: Some(_), .. } => {
+                GenericParamKind::Type { default: Some(_), .. }
-                    prev_ty_default = Some(param.ident.span);
+                | GenericParamKind::Const { default: Some(_), .. } => {
+                    prev_param_default = Some(param.ident.span);
                 }
                 GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
-                    if let Some(span) = prev_ty_default {
+                    if let Some(span) = prev_param_default {
                         let mut err = self.err_handler().struct_span_err(
                             span,
-                            "type parameters with a default must be trailing",
+                            "generic parameters with a default must be trailing",
                         );
-                        if matches!(param.kind, GenericParamKind::Const { .. }) {
+                        if matches!(param.kind, GenericParamKind::Const { .. }) && !cg_defaults {
                             err.note(
                                 "using type defaults and const parameters \
                                  in the same parameter list is currently not permitted",
@@ -1188,8 +1213,41 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 deny_equality_constraints(self, predicate, generics);
             }
         }
+        walk_list!(self, visit_generic_param, &generics.params);
+        for predicate in &generics.where_clause.predicates {
+            match predicate {
+                WherePredicate::BoundPredicate(bound_pred) => {
+                    // A type binding, eg `for<'c> Foo: Send+Clone+'c`
+                    self.check_late_bound_lifetime_defs(&bound_pred.bound_generic_params);
 
-        visit::walk_generics(self, generics)
+                    // This is slightly complicated. Our representation for poly-trait-refs contains a single
+                    // binder and thus we only allow a single level of quantification. However,
+                    // the syntax of Rust permits quantification in two places in where clauses,
+                    // e.g., `T: for <'a> Foo<'a>` and `for <'a, 'b> &'b T: Foo<'a>`. If both are
+                    // defined, then error.
+                    if !bound_pred.bound_generic_params.is_empty() {
+                        for bound in &bound_pred.bounds {
+                            match bound {
+                                GenericBound::Trait(t, _) => {
+                                    if !t.bound_generic_params.is_empty() {
+                                        struct_span_err!(
+                                            self.err_handler(),
+                                            t.span,
+                                            E0316,
+                                            "nested quantification of lifetimes"
+                                        )
+                                        .emit();
+                                    }
+                                }
+                                GenericBound::Outlives(_) => {}
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+            self.visit_where_predicate(predicate);
+        }
     }
 
     fn visit_generic_param(&mut self, param: &'a GenericParam) {
@@ -1238,14 +1296,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         visit::walk_pat(self, pat)
     }
 
-    fn visit_where_predicate(&mut self, p: &'a WherePredicate) {
-        if let &WherePredicate::BoundPredicate(ref bound_predicate) = p {
-            // A type binding, eg `for<'c> Foo: Send+Clone+'c`
-            self.check_late_bound_lifetime_defs(&bound_predicate.bound_generic_params);
-        }
-        visit::walk_where_predicate(self, p);
-    }
-
     fn visit_poly_trait_ref(&mut self, t: &'a PolyTraitRef, m: &'a TraitBoundModifier) {
         self.check_late_bound_lifetime_defs(&t.bound_generic_params);
         visit::walk_poly_trait_ref(self, t, m);

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -8,7 +8,7 @@ use rustc_feature::{Features, GateIssue};
 use rustc_session::parse::{feature_err, feature_err_issue};
 use rustc_session::Session;
 use rustc_span::source_map::Spanned;
-use rustc_span::symbol::{sym, Symbol};
+use rustc_span::symbol::sym;
 use rustc_span::Span;
 
 use tracing::debug;
@@ -196,6 +196,14 @@ impl<'a> PostExpansionVisitor<'a> {
                     "thiscall-unwind ABI is experimental and subject to change"
                 );
             }
+            "wasm" => {
+                gate_feature_post!(
+                    &self,
+                    wasm_abi,
+                    span,
+                    "wasm ABI is experimental and subject to change"
+                );
+            }
             abi => self
                 .sess
                 .parse_sess
@@ -313,24 +321,13 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     include => external_doc
                     cfg => doc_cfg
                     masked => doc_masked
-                    spotlight => doc_spotlight
+                    notable_trait => doc_notable_trait
                     keyword => doc_keyword
                 );
             }
         }
     }
 
-    fn visit_name(&mut self, sp: Span, name: Symbol) {
-        if !name.as_str().is_ascii() {
-            gate_feature_post!(
-                &self,
-                non_ascii_idents,
-                self.sess.parse_sess.source_map().guess_head_span(sp),
-                "non-ascii idents are not fully supported"
-            );
-        }
-    }
-
     fn visit_item(&mut self, i: &'a ast::Item) {
         match i.kind {
             ast::ItemKind::ForeignMod(ref foreign_module) => {
@@ -358,16 +355,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                          over time"
                     );
                 }
-                if self.sess.contains_name(&i.attrs[..], sym::main) {
-                    gate_feature_post!(
-                        &self,
-                        main,
-                        i.span,
-                        "declaration of a non-standard `#[main]` \
-                         function may change over time, for now \
-                         a top-level `fn main()` is required"
-                    );
-                }
             }
 
             ast::ItemKind::Struct(..) => {
@@ -599,12 +586,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
     fn visit_assoc_item(&mut self, i: &'a ast::AssocItem, ctxt: AssocCtxt) {
         let is_fn = match i.kind {
-            ast::AssocItemKind::Fn(box ast::FnKind(_, ref sig, _, _)) => {
+            ast::AssocItemKind::Fn(_) => true,
-                if let (ast::Const::Yes(_), AssocCtxt::Trait) = (sig.header.constness, ctxt) {
-                    gate_feature_post!(&self, const_fn, i.span, "const fn is unstable");
-                }
-                true
-            }
             ast::AssocItemKind::TyAlias(box ast::TyAliasKind(_, ref generics, _, ref ty)) => {
                 if let (Some(_), AssocCtxt::Trait) = (ty, ctxt) {
                     gate_feature_post!(
@@ -686,7 +668,6 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
         "to use an async block, remove the `||`: `async {`"
     );
     gate_all!(generators, "yield syntax is experimental");
-    gate_all!(or_patterns, "or-patterns syntax is experimental");
     gate_all!(raw_ref_op, "raw address of syntax is experimental");
     gate_all!(const_trait_bound_opt_out, "`?const` on trait bounds is experimental");
     gate_all!(const_trait_impl, "const trait impls are experimental");
@@ -705,7 +686,6 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
         // involved, so we only emit errors where there are no other parsing errors.
         gate_all!(destructuring_assignment, "destructuring assignments are unstable");
     }
-    gate_all!(pub_macro_rules, "`pub` on `macro_rules` items is unstable");
 
     // All uses of `gate_all!` below this point were added in #65742,
     // and subsequently disabled (with the non-early gating readded).
@@ -741,16 +721,46 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
 }
 
 fn maybe_stage_features(sess: &Session, krate: &ast::Crate) {
+    use rustc_errors::Applicability;
+
     if !sess.opts.unstable_features.is_nightly_build() {
+        let lang_features = &sess.features_untracked().declared_lang_features;
         for attr in krate.attrs.iter().filter(|attr| sess.check_name(attr, sym::feature)) {
-            struct_span_err!(
+            let mut err = struct_span_err!(
                 sess.parse_sess.span_diagnostic,
                 attr.span,
                 E0554,
                 "`#![feature]` may not be used on the {} release channel",
                 option_env!("CFG_RELEASE_CHANNEL").unwrap_or("(unknown)")
-            )
+            );
-            .emit();
+            let mut all_stable = true;
+            for ident in
+                attr.meta_item_list().into_iter().flatten().map(|nested| nested.ident()).flatten()
+            {
+                let name = ident.name;
+                let stable_since = lang_features
+                    .iter()
+                    .flat_map(|&(feature, _, since)| if feature == name { since } else { None })
+                    .next();
+                if let Some(since) = stable_since {
+                    err.help(&format!(
+                        "the feature `{}` has been stable since {} and no longer requires \
+                                  an attribute to enable",
+                        name, since
+                    ));
+                } else {
+                    all_stable = false;
+                }
+            }
+            if all_stable {
+                err.span_suggestion(
+                    attr.span,
+                    "remove the attribute",
+                    String::new(),
+                    Applicability::MachineApplicable,
+                );
+            }
+            err.emit();
         }
     }
 }

compiler/rustc_ast_pretty/src/lib.rs

@@ -1,6 +1,6 @@
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_patterns)]
 #![recursion_limit = "256"]
 

compiler/rustc_ast_pretty/src/pprust/mod.rs

@@ -22,10 +22,6 @@ pub fn token_to_string(token: &Token) -> String {
     State::new().token_to_string(token)
 }
 
-pub fn token_to_string_ext(token: &Token, convert_dollar_crate: bool) -> String {
-    State::new().token_to_string_ext(token, convert_dollar_crate)
-}
-
 pub fn ty_to_string(ty: &ast::Ty) -> String {
     State::new().ty_to_string(ty)
 }
@@ -50,18 +46,10 @@ pub fn tts_to_string(tokens: &TokenStream) -> String {
     State::new().tts_to_string(tokens)
 }
 
-pub fn stmt_to_string(stmt: &ast::Stmt) -> String {
-    State::new().stmt_to_string(stmt)
-}
-
 pub fn item_to_string(i: &ast::Item) -> String {
     State::new().item_to_string(i)
 }
 
-pub fn generic_params_to_string(generic_params: &[ast::GenericParam]) -> String {
-    State::new().generic_params_to_string(generic_params)
-}
-
 pub fn path_to_string(p: &ast::Path) -> String {
     State::new().path_to_string(p)
 }
@@ -74,26 +62,14 @@ pub fn vis_to_string(v: &ast::Visibility) -> String {
     State::new().vis_to_string(v)
 }
 
-pub fn block_to_string(blk: &ast::Block) -> String {
-    State::new().block_to_string(blk)
-}
-
 pub fn meta_list_item_to_string(li: &ast::NestedMetaItem) -> String {
     State::new().meta_list_item_to_string(li)
 }
 
-pub fn attr_item_to_string(ai: &ast::AttrItem) -> String {
-    State::new().attr_item_to_string(ai)
-}
-
 pub fn attribute_to_string(attr: &ast::Attribute) -> String {
     State::new().attribute_to_string(attr)
 }
 
-pub fn param_to_string(arg: &ast::Param) -> String {
-    State::new().param_to_string(arg)
-}
-
 pub fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
     State::new().to_string(f)
 }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -2149,10 +2149,10 @@ impl<'a> State<'a> {
                                     None => s.word("_"),
                                 }
                             }
-                            InlineAsmOperand::Const { expr } => {
+                            InlineAsmOperand::Const { anon_const } => {
                                 s.word("const");
                                 s.space();
-                                s.print_expr(expr);
+                                s.print_expr(&anon_const.value);
                             }
                             InlineAsmOperand::Sym { expr } => {
                                 s.word("sym");
@@ -2292,10 +2292,6 @@ impl<'a> State<'a> {
         }
     }
 
-    pub fn print_usize(&mut self, i: usize) {
-        self.s.word(i.to_string())
-    }
-
     crate fn print_name(&mut self, name: Symbol) {
         self.s.word(name.to_string());
         self.ann.post(self, AnnNode::Name(&name))
@@ -2659,8 +2655,10 @@ impl<'a> State<'a> {
                     s.word_space(":");
                     s.print_type(ty);
                     s.print_type_bounds(":", &param.bounds);
-                    if let Some(ref _default) = default {
+                    if let Some(ref default) = default {
-                        // FIXME(const_generics_defaults): print the `default` value here
+                        s.s.space();
+                        s.word_space("=");
+                        s.print_expr(&default.value);
                     }
                 }
             }

compiler/rustc_attr/src/lib.rs

@@ -4,7 +4,7 @@
 //! The goal is to move the definition of `MetaItem` and things that don't need to be in `syntax`
 //! to this crate.
 
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 
 #[macro_use]
 extern crate rustc_macros;

compiler/rustc_builtin_macros/src/asm.rs

@@ -7,11 +7,10 @@ use rustc_errors::{Applicability, DiagnosticBuilder};
 use rustc_expand::base::{self, *};
 use rustc_parse::parser::Parser;
 use rustc_parse_format as parse;
-use rustc_span::{
+use rustc_session::lint;
-    symbol::{kw, sym, Symbol},
+use rustc_span::symbol::{kw, sym, Symbol};
-    BytePos,
-};
 use rustc_span::{InnerSpan, Span};
+use rustc_target::asm::InlineAsmArch;
 
 struct AsmArgs {
     templates: Vec<P<ast::Expr>>,
@@ -137,8 +136,8 @@ fn parse_args<'a>(
                 ast::InlineAsmOperand::InOut { reg, expr, late: true }
             }
         } else if p.eat_keyword(kw::Const) {
-            let expr = p.parse_expr()?;
+            let anon_const = p.parse_anon_const_expr()?;
-            ast::InlineAsmOperand::Const { expr }
+            ast::InlineAsmOperand::Const { anon_const }
         } else if p.eat_keyword(sym::sym) {
             let expr = p.parse_expr()?;
             match expr.kind {
@@ -402,8 +401,6 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
     let mut line_spans = Vec::with_capacity(args.templates.len());
     let mut curarg = 0;
 
-    let default_dialect = ecx.sess.inline_asm_dialect();
-
     for template_expr in args.templates.into_iter() {
         if !template.is_empty() {
             template.push(ast::InlineAsmTemplatePiece::String("\n".to_string()));
@@ -430,56 +427,36 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
         let template_str = &template_str.as_str();
         let template_snippet = ecx.source_map().span_to_snippet(template_sp).ok();
 
-        if let Some(snippet) = &template_snippet {
+        if let Some(InlineAsmArch::X86 | InlineAsmArch::X86_64) = ecx.sess.asm_arch {
-            let snippet = snippet.trim_matches('"');
+            let find_span = |needle: &str| -> Span {
-            match default_dialect {
+                if let Some(snippet) = &template_snippet {
-                ast::LlvmAsmDialect::Intel => {
+                    if let Some(pos) = snippet.find(needle) {
-                    if let Some(span) = check_syntax_directive(snippet, ".intel_syntax") {
+                        let end = pos
-                        let span = template_span.from_inner(span);
+                            + &snippet[pos..]
-                        let mut err = ecx.struct_span_err(span, "intel syntax is the default syntax on this target, and trying to use this directive may cause issues");
+                                .find(|c| matches!(c, '\n' | ';' | '\\' | '"'))
-                        err.span_suggestion(
+                                .unwrap_or(snippet[pos..].len() - 1);
-                            span,
+                        let inner = InnerSpan::new(pos, end);
-                            "remove this assembler directive",
+                        return template_sp.from_inner(inner);
-                            "".to_string(),
-                            Applicability::MachineApplicable,
-                        );
-                        err.emit();
-                    }
-
-                    if let Some(span) = check_syntax_directive(snippet, ".att_syntax") {
-                        let span = template_span.from_inner(span);
-                        let mut err = ecx.struct_span_err(span, "using the .att_syntax directive may cause issues, use the att_syntax option instead");
-                        let asm_end = sp.hi() - BytePos(2);
-                        let suggestions = vec![
-                            (span, "".to_string()),
-                            (
-                                Span::new(asm_end, asm_end, sp.ctxt()),
-                                ", options(att_syntax)".to_string(),
-                            ),
-                        ];
-                        err.multipart_suggestion(
-                        "remove the assembler directive and replace it with options(att_syntax)",
-                        suggestions,
-                        Applicability::MachineApplicable,
-                    );
-                        err.emit();
                     }
                 }
-                ast::LlvmAsmDialect::Att => {
+                template_sp
-                    if let Some(span) = check_syntax_directive(snippet, ".att_syntax") {
+            };
-                        let span = template_span.from_inner(span);
-                        let mut err = ecx.struct_span_err(span, "att syntax is the default syntax on this target, and trying to use this directive may cause issues");
-                        err.span_suggestion(
-                            span,
-                            "remove this assembler directive",
-                            "".to_string(),
-                            Applicability::MachineApplicable,
-                        );
-                        err.emit();
-                    }
 
-                    // Use of .intel_syntax is ignored
+            if template_str.contains(".intel_syntax") {
-                }
+                ecx.parse_sess().buffer_lint(
+                    lint::builtin::BAD_ASM_STYLE,
+                    find_span(".intel_syntax"),
+                    ecx.resolver.lint_node_id(ecx.current_expansion.id),
+                    "avoid using `.intel_syntax`, Intel syntax is the default",
+                );
+            }
+            if template_str.contains(".att_syntax") {
+                ecx.parse_sess().buffer_lint(
+                    lint::builtin::BAD_ASM_STYLE,
+                    find_span(".att_syntax"),
+                    ecx.resolver.lint_node_id(ecx.current_expansion.id),
+                    "avoid using `.att_syntax`, prefer using `options(att_syntax)` instead",
+                );
             }
         }
 
@@ -690,15 +667,3 @@ pub fn expand_asm<'cx>(
         }
     }
 }
-
-fn check_syntax_directive<S: AsRef<str>>(piece: S, syntax: &str) -> Option<InnerSpan> {
-    let piece = piece.as_ref();
-    if let Some(idx) = piece.find(syntax) {
-        let end =
-            idx + &piece[idx..].find(|c| matches!(c, '\n' | ';')).unwrap_or(piece[idx..].len());
-        // Offset by one because these represent the span with the " removed
-        Some(InnerSpan::new(idx + 1, end + 1))
-    } else {
-        None
-    }
-}

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -1,11 +1,18 @@
 use crate::util::check_builtin_macro_attribute;
 
-use rustc_ast::mut_visit::{self, MutVisitor};
+use rustc_ast as ast;
-use rustc_ast::ptr::P;
+use rustc_ast::mut_visit::MutVisitor;
-use rustc_ast::{self as ast, AstLike};
+use rustc_ast::tokenstream::CanSynthesizeMissingTokens;
+use rustc_ast::visit::Visitor;
+use rustc_ast::{mut_visit, visit};
+use rustc_ast::{AstLike, Attribute};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_expand::config::StripUnconfigured;
 use rustc_expand::configure;
+use rustc_parse::parser::ForceCollect;
+use rustc_session::utils::FlattenNonterminals;
+
+use rustc_ast::ptr::P;
 use rustc_span::symbol::sym;
 use rustc_span::Span;
 use smallvec::SmallVec;
@@ -22,74 +29,179 @@ crate fn expand(
 
 crate fn cfg_eval(ecx: &ExtCtxt<'_>, annotatable: Annotatable) -> Vec<Annotatable> {
     let mut visitor = CfgEval {
-        cfg: StripUnconfigured { sess: ecx.sess, features: ecx.ecfg.features, modified: false },
+        cfg: &mut StripUnconfigured {
+            sess: ecx.sess,
+            features: ecx.ecfg.features,
+            config_tokens: true,
+        },
     };
-    let mut annotatable = visitor.configure_annotatable(annotatable);
+    let annotatable = visitor.configure_annotatable(annotatable);
-    if visitor.cfg.modified {
-        // Erase the tokens if cfg-stripping modified the item
-        // This will cause us to synthesize fake tokens
-        // when `nt_to_tokenstream` is called on this item.
-        if let Some(tokens) = annotatable.tokens_mut() {
-            *tokens = None;
-        }
-    }
     vec![annotatable]
 }
 
-struct CfgEval<'a> {
+struct CfgEval<'a, 'b> {
-    cfg: StripUnconfigured<'a>,
+    cfg: &'a mut StripUnconfigured<'b>,
 }
 
-impl CfgEval<'_> {
+fn flat_map_annotatable(vis: &mut impl MutVisitor, annotatable: Annotatable) -> Annotatable {
+    // Since the item itself has already been configured by the InvocationCollector,
+    // we know that fold result vector will contain exactly one element
+    match annotatable {
+        Annotatable::Item(item) => Annotatable::Item(vis.flat_map_item(item).pop().unwrap()),
+        Annotatable::TraitItem(item) => {
+            Annotatable::TraitItem(vis.flat_map_trait_item(item).pop().unwrap())
+        }
+        Annotatable::ImplItem(item) => {
+            Annotatable::ImplItem(vis.flat_map_impl_item(item).pop().unwrap())
+        }
+        Annotatable::ForeignItem(item) => {
+            Annotatable::ForeignItem(vis.flat_map_foreign_item(item).pop().unwrap())
+        }
+        Annotatable::Stmt(stmt) => {
+            Annotatable::Stmt(stmt.map(|stmt| vis.flat_map_stmt(stmt).pop().unwrap()))
+        }
+        Annotatable::Expr(mut expr) => Annotatable::Expr({
+            vis.visit_expr(&mut expr);
+            expr
+        }),
+        Annotatable::Arm(arm) => Annotatable::Arm(vis.flat_map_arm(arm).pop().unwrap()),
+        Annotatable::ExprField(field) => {
+            Annotatable::ExprField(vis.flat_map_expr_field(field).pop().unwrap())
+        }
+        Annotatable::PatField(fp) => {
+            Annotatable::PatField(vis.flat_map_pat_field(fp).pop().unwrap())
+        }
+        Annotatable::GenericParam(param) => {
+            Annotatable::GenericParam(vis.flat_map_generic_param(param).pop().unwrap())
+        }
+        Annotatable::Param(param) => Annotatable::Param(vis.flat_map_param(param).pop().unwrap()),
+        Annotatable::FieldDef(sf) => {
+            Annotatable::FieldDef(vis.flat_map_field_def(sf).pop().unwrap())
+        }
+        Annotatable::Variant(v) => Annotatable::Variant(vis.flat_map_variant(v).pop().unwrap()),
+    }
+}
+
+struct CfgFinder {
+    has_cfg_or_cfg_attr: bool,
+}
+
+impl CfgFinder {
+    fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
+        let mut finder = CfgFinder { has_cfg_or_cfg_attr: false };
+        match annotatable {
+            Annotatable::Item(item) => finder.visit_item(&item),
+            Annotatable::TraitItem(item) => finder.visit_assoc_item(&item, visit::AssocCtxt::Trait),
+            Annotatable::ImplItem(item) => finder.visit_assoc_item(&item, visit::AssocCtxt::Impl),
+            Annotatable::ForeignItem(item) => finder.visit_foreign_item(&item),
+            Annotatable::Stmt(stmt) => finder.visit_stmt(&stmt),
+            Annotatable::Expr(expr) => finder.visit_expr(&expr),
+            Annotatable::Arm(arm) => finder.visit_arm(&arm),
+            Annotatable::ExprField(field) => finder.visit_expr_field(&field),
+            Annotatable::PatField(field) => finder.visit_pat_field(&field),
+            Annotatable::GenericParam(param) => finder.visit_generic_param(&param),
+            Annotatable::Param(param) => finder.visit_param(&param),
+            Annotatable::FieldDef(field) => finder.visit_field_def(&field),
+            Annotatable::Variant(variant) => finder.visit_variant(&variant),
+        };
+        finder.has_cfg_or_cfg_attr
+    }
+}
+
+impl<'ast> visit::Visitor<'ast> for CfgFinder {
+    fn visit_attribute(&mut self, attr: &'ast Attribute) {
+        // We want short-circuiting behavior, so don't use the '|=' operator.
+        self.has_cfg_or_cfg_attr = self.has_cfg_or_cfg_attr
+            || attr
+                .ident()
+                .map_or(false, |ident| ident.name == sym::cfg || ident.name == sym::cfg_attr);
+    }
+}
+
+impl CfgEval<'_, '_> {
     fn configure<T: AstLike>(&mut self, node: T) -> Option<T> {
         self.cfg.configure(node)
     }
 
-    fn configure_annotatable(&mut self, annotatable: Annotatable) -> Annotatable {
+    pub fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Annotatable {
-        // Since the item itself has already been configured by the InvocationCollector,
+        // Tokenizing and re-parsing the `Annotatable` can have a significant
-        // we know that fold result vector will contain exactly one element
+        // performance impact, so try to avoid it if possible
-        match annotatable {
+        if !CfgFinder::has_cfg_or_cfg_attr(&annotatable) {
-            Annotatable::Item(item) => Annotatable::Item(self.flat_map_item(item).pop().unwrap()),
+            return annotatable;
-            Annotatable::TraitItem(item) => {
-                Annotatable::TraitItem(self.flat_map_trait_item(item).pop().unwrap())
-            }
-            Annotatable::ImplItem(item) => {
-                Annotatable::ImplItem(self.flat_map_impl_item(item).pop().unwrap())
-            }
-            Annotatable::ForeignItem(item) => {
-                Annotatable::ForeignItem(self.flat_map_foreign_item(item).pop().unwrap())
-            }
-            Annotatable::Stmt(stmt) => {
-                Annotatable::Stmt(stmt.map(|stmt| self.flat_map_stmt(stmt).pop().unwrap()))
-            }
-            Annotatable::Expr(mut expr) => Annotatable::Expr({
-                self.visit_expr(&mut expr);
-                expr
-            }),
-            Annotatable::Arm(arm) => Annotatable::Arm(self.flat_map_arm(arm).pop().unwrap()),
-            Annotatable::ExprField(field) => {
-                Annotatable::ExprField(self.flat_map_expr_field(field).pop().unwrap())
-            }
-            Annotatable::PatField(fp) => {
-                Annotatable::PatField(self.flat_map_pat_field(fp).pop().unwrap())
-            }
-            Annotatable::GenericParam(param) => {
-                Annotatable::GenericParam(self.flat_map_generic_param(param).pop().unwrap())
-            }
-            Annotatable::Param(param) => {
-                Annotatable::Param(self.flat_map_param(param).pop().unwrap())
-            }
-            Annotatable::FieldDef(sf) => {
-                Annotatable::FieldDef(self.flat_map_field_def(sf).pop().unwrap())
-            }
-            Annotatable::Variant(v) => {
-                Annotatable::Variant(self.flat_map_variant(v).pop().unwrap())
-            }
         }
+
+        // The majority of parsed attribute targets will never need to have early cfg-expansion
+        // run (e.g. they are not part of a `#[derive]` or `#[cfg_eval]` macro inoput).
+        // Therefore, we normally do not capture the necessary information about `#[cfg]`
+        // and `#[cfg_attr]` attributes during parsing.
+        //
+        // Therefore, when we actually *do* run early cfg-expansion, we need to tokenize
+        // and re-parse the attribute target, this time capturing information about
+        // the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization
+        // process is lossless, so this process is invisible to proc-macros.
+
+        // FIXME - get rid of this clone
+        let nt = annotatable.clone().into_nonterminal();
+
+        let mut orig_tokens = rustc_parse::nt_to_tokenstream(
+            &nt,
+            &self.cfg.sess.parse_sess,
+            CanSynthesizeMissingTokens::No,
+        );
+
+        // 'Flatten' all nonterminals (i.e. `TokenKind::Interpolated`)
+        // to `None`-delimited groups containing the corresponding tokens. This
+        // is normally delayed until the proc-macro server actually needs to
+        // provide a `TokenKind::Interpolated` to a proc-macro. We do this earlier,
+        // so that we can handle cases like:
+        //
+        // ```rust
+        // #[cfg_eval] #[cfg] $item
+        //```
+        //
+        // where `$item` is `#[cfg_attr] struct Foo {}`. We want to make
+        // sure to evaluate *all* `#[cfg]` and `#[cfg_attr]` attributes - the simplest
+        // way to do this is to do a single parse of a stream without any nonterminals.
+        let mut flatten = FlattenNonterminals {
+            nt_to_tokenstream: rustc_parse::nt_to_tokenstream,
+            parse_sess: &self.cfg.sess.parse_sess,
+            synthesize_tokens: CanSynthesizeMissingTokens::No,
+        };
+        orig_tokens = flatten.process_token_stream(orig_tokens);
+
+        // Re-parse the tokens, setting the `capture_cfg` flag to save extra information
+        // to the captured `AttrAnnotatedTokenStream` (specifically, we capture
+        // `AttrAnnotatedTokenTree::AttributesData` for all occurences of `#[cfg]` and `#[cfg_attr]`)
+        let mut parser =
+            rustc_parse::stream_to_parser(&self.cfg.sess.parse_sess, orig_tokens, None);
+        parser.capture_cfg = true;
+        annotatable = match annotatable {
+            Annotatable::Item(_) => {
+                Annotatable::Item(parser.parse_item(ForceCollect::Yes).unwrap().unwrap())
+            }
+            Annotatable::TraitItem(_) => Annotatable::TraitItem(
+                parser.parse_trait_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
+            ),
+            Annotatable::ImplItem(_) => Annotatable::ImplItem(
+                parser.parse_impl_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
+            ),
+            Annotatable::ForeignItem(_) => Annotatable::ForeignItem(
+                parser.parse_foreign_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
+            ),
+            Annotatable::Stmt(_) => {
+                Annotatable::Stmt(P(parser.parse_stmt(ForceCollect::Yes).unwrap().unwrap()))
+            }
+            Annotatable::Expr(_) => Annotatable::Expr(parser.parse_expr_force_collect().unwrap()),
+            _ => unreachable!(),
+        };
+
+        // Now that we have our re-parsed `AttrAnnotatedTokenStream`, recursively configuring
+        // our attribute target will correctly the tokens as well.
+        flat_map_annotatable(self, annotatable)
     }
 }
 
-impl MutVisitor for CfgEval<'_> {
+impl MutVisitor for CfgEval<'_, '_> {
     fn visit_expr(&mut self, expr: &mut P<ast::Expr>) {
         self.cfg.configure_expr(expr);
         mut_visit::noop_visit_expr(expr, self);

compiler/rustc_builtin_macros/src/derive.rs

@@ -1,6 +1,6 @@
 use crate::cfg_eval::cfg_eval;
 
-use rustc_ast::{self as ast, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
+use rustc_ast::{self as ast, attr, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
 use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
 use rustc_feature::AttributeTemplate;
@@ -26,32 +26,39 @@ impl MultiItemModifier for Expander {
             return ExpandResult::Ready(vec![item]);
         }
 
-        let template =
+        let result =
-            AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() };
+            ecx.resolver.resolve_derives(ecx.current_expansion.id, ecx.force_mode, &|| {
-        let attr = ecx.attribute(meta_item.clone());
+                let template =
-        validate_attr::check_builtin_attribute(&sess.parse_sess, &attr, sym::derive, template);
+                    AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() };
+                let attr = attr::mk_attr_outer(meta_item.clone());
+                validate_attr::check_builtin_attribute(
+                    &sess.parse_sess,
+                    &attr,
+                    sym::derive,
+                    template,
+                );
 
-        let derives: Vec<_> = attr
+                attr.meta_item_list()
-            .meta_item_list()
+                    .unwrap_or_default()
-            .unwrap_or_default()
+                    .into_iter()
-            .into_iter()
+                    .filter_map(|nested_meta| match nested_meta {
-            .filter_map(|nested_meta| match nested_meta {
+                        NestedMetaItem::MetaItem(meta) => Some(meta),
-                NestedMetaItem::MetaItem(meta) => Some(meta),
+                        NestedMetaItem::Literal(lit) => {
-                NestedMetaItem::Literal(lit) => {
+                            // Reject `#[derive("Debug")]`.
-                    // Reject `#[derive("Debug")]`.
+                            report_unexpected_literal(sess, &lit);
-                    report_unexpected_literal(sess, &lit);
+                            None
-                    None
+                        }
-                }
+                    })
-            })
+                    .map(|meta| {
-            .map(|meta| {
+                        // Reject `#[derive(Debug = "value", Debug(abc))]`, but recover the paths.
-                // Reject `#[derive(Debug = "value", Debug(abc))]`, but recover the paths.
+                        report_path_args(sess, &meta);
-                report_path_args(sess, &meta);
+                        meta.path
-                meta.path
+                    })
-            })
+                    .map(|path| (path, None))
-            .collect();
+                    .collect()
+            });
 
-        // FIXME: Try to cache intermediate results to avoid collecting same paths multiple times.
+        match result {
-        match ecx.resolver.resolve_derives(ecx.current_expansion.id, derives, ecx.force_mode) {
             Ok(()) => ExpandResult::Ready(cfg_eval(ecx, item)),
             Err(Indeterminate) => ExpandResult::Retry(item),
         }

compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs

@@ -15,10 +15,12 @@ pub fn expand_deriving_eq(
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
 ) {
+    let span = cx.with_def_site_ctxt(span);
     let inline = cx.meta_word(span, sym::inline);
     let hidden = rustc_ast::attr::mk_nested_word_item(Ident::new(sym::hidden, span));
     let doc = rustc_ast::attr::mk_list_item(Ident::new(sym::doc, span), vec![hidden]);
-    let attrs = vec![cx.attribute(inline), cx.attribute(doc)];
+    let no_coverage = cx.meta_word(span, sym::no_coverage);
+    let attrs = vec![cx.attribute(inline), cx.attribute(doc), cx.attribute(no_coverage)];
     let trait_def = TraitDef {
         span,
         attributes: Vec::new(),

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -541,7 +541,7 @@ impl<'a> TraitDef<'a> {
             self.generics.to_generics(cx, self.span, type_ident, generics);
 
         // Create the generic parameters
-        params.extend(generics.params.iter().map(|param| match param.kind {
+        params.extend(generics.params.iter().map(|param| match &param.kind {
             GenericParamKind::Lifetime { .. } => param.clone(),
             GenericParamKind::Type { .. } => {
                 // I don't think this can be moved out of the loop, since
@@ -561,7 +561,18 @@ impl<'a> TraitDef<'a> {
 
                 cx.typaram(self.span, param.ident, vec![], bounds, None)
             }
-            GenericParamKind::Const { .. } => param.clone(),
+            GenericParamKind::Const { ty, kw_span, .. } => {
+                let const_nodefault_kind = GenericParamKind::Const {
+                    ty: ty.clone(),
+                    kw_span: kw_span.clone(),
+
+                    // We can't have default values inside impl block
+                    default: None,
+                };
+                let mut param_clone = param.clone();
+                param_clone.kind = const_nodefault_kind;
+                param_clone
+            }
         }));
 
         // and similarly for where clauses
@@ -1034,7 +1045,7 @@ impl<'a> MethodDef<'a> {
         // make a series of nested matches, to destructure the
         // structs. This is actually right-to-left, but it shouldn't
         // matter.
-        for (arg_expr, pat) in self_args.iter().zip(patterns) {
+        for (arg_expr, pat) in iter::zip(self_args, patterns) {
             body = cx.expr_match(
                 trait_.span,
                 arg_expr.clone(),
@@ -1351,7 +1362,7 @@ impl<'a> MethodDef<'a> {
             let mut discriminant_test = cx.expr_bool(sp, true);
 
             let mut first_ident = None;
-            for (&ident, self_arg) in vi_idents.iter().zip(&self_args) {
+            for (&ident, self_arg) in iter::zip(&vi_idents, &self_args) {
                 let self_addr = cx.expr_addr_of(sp, self_arg.clone());
                 let variant_value =
                     deriving::call_intrinsic(cx, sp, sym::discriminant_value, vec![self_addr]);
@@ -1571,9 +1582,7 @@ impl<'a> TraitDef<'a> {
         let subpats = self.create_subpatterns(cx, paths, mutbl, use_temporaries);
         let pattern = match *struct_def {
             VariantData::Struct(..) => {
-                let field_pats = subpats
+                let field_pats = iter::zip(subpats, &ident_exprs)
-                    .into_iter()
-                    .zip(&ident_exprs)
                     .map(|(pat, &(sp, ident, ..))| {
                         if ident.is_none() {
                             cx.span_bug(sp, "a braced struct with unnamed fields in `derive`");

compiler/rustc_builtin_macros/src/global_allocator.rs

@@ -14,31 +14,31 @@ pub fn expand(
     ecx: &mut ExtCtxt<'_>,
     _span: Span,
     meta_item: &ast::MetaItem,
-    mut item: Annotatable,
+    item: Annotatable,
 ) -> Vec<Annotatable> {
     check_builtin_macro_attribute(ecx, meta_item, sym::global_allocator);
 
-    let not_static = |item: Annotatable| {
-        ecx.sess.parse_sess.span_diagnostic.span_err(item.span(), "allocators must be statics");
-        vec![item]
-    };
     let orig_item = item.clone();
-    let mut is_stmt = false;
+    let not_static = || {
+        ecx.sess.parse_sess.span_diagnostic.span_err(item.span(), "allocators must be statics");
+        vec![orig_item.clone()]
+    };
 
     // Allow using `#[global_allocator]` on an item statement
-    if let Annotatable::Stmt(stmt) = &item {
+    // FIXME - if we get deref patterns, use them to reduce duplication here
-        if let StmtKind::Item(item_) = &stmt.kind {
+    let (item, is_stmt) = match &item {
-            item = Annotatable::Item(item_.clone());
-            is_stmt = true;
-        }
-    }
-
-    let item = match item {
         Annotatable::Item(item) => match item.kind {
-            ItemKind::Static(..) => item,
+            ItemKind::Static(..) => (item, false),
-            _ => return not_static(Annotatable::Item(item)),
+            _ => return not_static(),
         },
-        _ => return not_static(item),
+        Annotatable::Stmt(stmt) => match &stmt.kind {
+            StmtKind::Item(item_) => match item_.kind {
+                ItemKind::Static(..) => (item_, true),
+                _ => return not_static(),
+            },
+            _ => return not_static(),
+        },
+        _ => return not_static(),
     };
 
     // Generate a bunch of new items using the AllocFnFactory

compiler/rustc_builtin_macros/src/lib.rs

@@ -7,8 +7,9 @@
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
+#![feature(iter_zip)]
 #![feature(nll)]
-#![feature(or_patterns)]
+#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_quote)]
 #![recursion_limit = "256"]

compiler/rustc_builtin_macros/src/test_harness.rs

@@ -142,7 +142,7 @@ fn entry_point_type(sess: &Session, item: &ast::Item, depth: usize) -> EntryPoin
         ast::ItemKind::Fn(..) => {
             if sess.contains_name(&item.attrs, sym::start) {
                 EntryPointType::Start
-            } else if sess.contains_name(&item.attrs, sym::main) {
+            } else if sess.contains_name(&item.attrs, sym::rustc_main) {
                 EntryPointType::MainAttr
             } else if item.ident.name == sym::main {
                 if depth == 1 {
@@ -187,7 +187,7 @@ impl<'a> MutVisitor for EntryPointCleaner<'a> {
                     let attrs = attrs
                         .into_iter()
                         .filter(|attr| {
-                            !self.sess.check_name(attr, sym::main)
+                            !self.sess.check_name(attr, sym::rustc_main)
                                 && !self.sess.check_name(attr, sym::start)
                         })
                         .chain(iter::once(allow_dead_code))
@@ -220,7 +220,7 @@ fn generate_test_harness(
     let expn_id = ext_cx.resolver.expansion_for_ast_pass(
         DUMMY_SP,
         AstPass::TestHarness,
-        &[sym::main, sym::test, sym::rustc_attrs],
+        &[sym::test, sym::rustc_attrs],
         None,
     );
     let def_site = DUMMY_SP.with_def_site_ctxt(expn_id);
@@ -247,7 +247,7 @@ fn generate_test_harness(
 /// By default this expands to
 ///
 /// ```
-/// #[main]
+/// #[rustc_main]
 /// pub fn main() {
 ///     extern crate test;
 ///     test::test_main_static(&[
@@ -297,8 +297,8 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
     let test_extern_stmt =
         ecx.stmt_item(sp, ecx.item(sp, test_id, vec![], ast::ItemKind::ExternCrate(None)));
 
-    // #[main]
+    // #[rustc_main]
-    let main_meta = ecx.meta_word(sp, sym::main);
+    let main_meta = ecx.meta_word(sp, sym::rustc_main);
     let main_attr = ecx.attribute(main_meta);
 
     // pub fn main() { ... }

compiler/rustc_codegen_cranelift/.github/workflows/bootstrap_rustc.yml

@@ -1,44 +0,0 @@
-name: Bootstrap rustc using cg_clif
-
-on:
-  - push
-
-jobs:
-  bootstrap_rustc:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Cache cargo installed crates
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v2
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
-    - name: Cache cargo target dir
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
-
-    - name: Prepare dependencies
-      run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./prepare.sh
-
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        ./scripts/test_bootstrap.sh

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -7,11 +7,18 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
 
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          # cross-compile from Linux to Windows using mingw
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
 
     steps:
     - uses: actions/checkout@v2
@@ -36,6 +43,12 @@ jobs:
         path: target
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
+    - name: Install MinGW toolchain and wine
+      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: |
+        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+        rustup target add x86_64-pc-windows-gnu
+
     - name: Prepare dependencies
       run: |
         git config --global user.email "user@example.com"
@@ -43,6 +56,8 @@ jobs:
         ./prepare.sh
 
     - name: Test
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
       run: |
         # Enable backtraces for easier debugging
         export RUST_BACKTRACE=1
@@ -51,12 +66,16 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
+        # Enable extra checks
+        export CG_CLIF_ENABLE_VERIFIER=1
+
         ./test.sh
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz build
 
     - name: Upload prebuilt cg_clif
+      if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
       uses: actions/upload-artifact@v2
       with:
         name: cg_clif-${{ runner.os }}

compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml

@@ -0,0 +1,82 @@
+name: Various rustc tests
+
+on:
+  - push
+
+jobs:
+  bootstrap_rustc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_bootstrap.sh
+  rustc_test_suite:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-cargo-installed-crates
+
+    - name: Cache cargo registry and index
+      uses: actions/cache@v2
+      with:
+        path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./prepare.sh
+
+    - name: Test
+      run: |
+        # Enable backtraces for easier debugging
+        export RUST_BACKTRACE=1
+
+        ./scripts/test_rustc_tests.sh

compiler/rustc_codegen_cranelift/.vscode/settings.json

@@ -2,7 +2,7 @@
     // source for rustc_* is not included in the rust-src component; disable the errors about this
     "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
     "rust-analyzer.assist.importMergeBehavior": "last",
-    "rust-analyzer.cargo.loadOutDirsFromCheck": true,
+    "rust-analyzer.cargo.runBuildScripts": true,
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",
         //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml",

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -39,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -65,8 +65,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -74,18 +74,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -95,8 +95,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -113,8 +113,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -125,8 +125,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "cranelift-codegen",
  "target-lexicon",
@@ -134,8 +134,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.70.0"
+version = "0.72.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -16,7 +16,7 @@ cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
-object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
+object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
@@ -75,3 +75,6 @@ debug = false
 [profile.release.package.syn]
 opt-level = 0
 debug = false
+
+[package.metadata.rust-analyzer]
+rustc_private = true

compiler/rustc_codegen_cranelift/Readme.md

@@ -34,70 +34,19 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo bui
 
 Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
 
-### Cargo
-
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/build/cargo.sh run
+$ $cg_clif_dir/build/cargo.sh build
 ```
 
-This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
 
-### Rustc
+For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md).
-
-> You should prefer using the Cargo method.
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
-```
-
-### Jit mode
-
-In jit mode cg_clif will immediately execute your code without creating an executable file.
-
-> This requires all dependencies to be available as dynamic library.
-> The jit mode will probably need cargo integration to make this possible.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh jit
-```
-
-or
-
-```bash
-$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
-```
-
-There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
-first called. It currently does not work with multi-threaded programs. When a not yet compiled
-function is called from another thread than the main thread, you will get an ICE.
-
-```bash
-$ $cg_clif_dir/build/cargo.sh lazy-jit
-```
-
-### Shell
-
-These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
-
-```bash
-function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
-}
-
-function jit() {
-    jit_naked "fn main() { $@ }"
-}
-
-function jit_calc() {
-    jit 'println!("0x{:x}", ' $@ ');';
-}
-```
 
 ## Env vars
 
-[see env_vars.md](docs/env_vars.md)
+See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift.
 
 ## Not yet supported
 
@@ -106,3 +55,20 @@ function jit_calc() {
       `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You
       have to specify specific registers instead.
 * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work)
+
+## License
+
+Licensed under either of
+
+  * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or
+    http://www.apache.org/licenses/LICENSE-2.0)
+  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+    http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally submitted
+for inclusion in the work by you shall be dual licensed as above, without any
+additional terms or conditions.

compiler/rustc_codegen_cranelift/build.sh

@@ -55,6 +55,7 @@ ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib
 ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir"
 
 mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
+mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/"
 if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
     cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
 fi
@@ -64,12 +65,18 @@ case "$build_sysroot" in
         ;;
     "llvm")
         cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/"
+        fi
         ;;
     "clif")
         echo "[BUILD] sysroot"
         dir=$(pwd)
         cd "$target_dir"
         time "$dir/build_sysroot/build_sysroot.sh"
+        if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+            time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh"
+        fi
         cp lib/rustlib/*/lib/libstd-* lib/
         ;;
     *)

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -16,9 +16,9 @@ dependencies = [
 
 [[package]]
 name = "adler"
-version = "0.2.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -110,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.9.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -132,18 +132,18 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.86"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
+checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
 dependencies = [
  "rustc-std-workspace-core",
 ]
 
 [[package]]
 name = "miniz_oxide"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
 dependencies = [
  "adler",
  "autocfg",

compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh

@@ -28,7 +28,7 @@ export __CARGO_DEFAULT_LIB_METADATA="cg_clif"
 if [[ "$1" != "--debug" ]]; then
     sysroot_channel='release'
     # FIXME Enable incremental again once rust-lang/rust#74946 is fixed
-    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release
+    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release
 else
     sysroot_channel='debug'
     cargo build --target "$TARGET_TRIPLE"

compiler/rustc_codegen_cranelift/docs/env_vars.md

@@ -8,5 +8,8 @@
     to make it possible to use incremental mode for all analyses performed by rustc without caching
     object files when their content should have been changed by a change to cg_clif.</dd>
     <dt>CG_CLIF_DISPLAY_CG_TIME</dt>
-    <dd>If "1", display the time it took to perform codegen for a crate</dd>
+    <dd>If "1", display the time it took to perform codegen for a crate.</dd>
+    <dt>CG_CLIF_ENABLE_VERIFIER</dt>
+    <dd>Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once
+    before passing the clif ir to Cranelift for compilation.</dt>
 </dl>

compiler/rustc_codegen_cranelift/docs/usage.md

@@ -0,0 +1,66 @@
+# Usage
+
+rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
+
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
+
+## Cargo
+
+In the directory with your project (where you can do the usual `cargo build`), run:
+
+```bash
+$ $cg_clif_dir/build/cargo.sh build
+```
+
+This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
+
+## Rustc
+
+> You should prefer using the Cargo method.
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif my_crate.rs
+```
+
+## Jit mode
+
+In jit mode cg_clif will immediately execute your code without creating an executable file.
+
+> This requires all dependencies to be available as dynamic library.
+> The jit mode will probably need cargo integration to make this possible.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh jit
+```
+
+or
+
+```bash
+$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+```
+
+There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
+first called. It currently does not work with multi-threaded programs. When a not yet compiled
+function is called from another thread than the main thread, you will get an ICE.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh lazy-jit
+```
+
+## Shell
+
+These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit.
+
+```bash
+function jit_naked() {
+    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
+}
+
+function jit() {
+    jit_naked "fn main() { $@ }"
+}
+
+function jit_calc() {
+    jit 'println!("0x{:x}", ' $@ ');';
+}
+```

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -621,6 +621,7 @@ struct PanicLocation {
 }
 
 #[no_mangle]
+#[cfg(not(windows))]
 pub fn get_tls() -> u8 {
     #[thread_local]
     static A: u8 = 42;

compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs

@@ -1,7 +1,4 @@
-#![feature(
+#![feature(no_core, lang_items, box_syntax, never_type, linkage, extern_types, thread_local)]
-    no_core, start, lang_items, box_syntax, never_type, linkage,
-    extern_types, thread_local
-)]
 #![no_core]
 #![allow(dead_code, non_camel_case_types)]
 
@@ -239,7 +236,7 @@ fn main() {
 
     assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42);
 
-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
     {
         extern {
             #[linkage = "extern_weak"]
@@ -292,7 +289,7 @@ fn main() {
 
     from_decimal_string();
 
-    #[cfg(not(jit))]
+    #[cfg(not(any(jit, windows)))]
     test_tls();
 
     #[cfg(all(not(jit), target_os = "linux"))]

compiler/rustc_codegen_cranelift/prepare.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-rustup component add rust-src rustc-dev llvm-tools-preview
 ./build_sysroot/prepare_sysroot_src.sh
 cargo install hyperfine || echo "Skipping hyperfine install"
 

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1 +1,3 @@
-nightly-2021-03-05
+[toolchain]
+channel = "nightly-2021-03-29"
+components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

compiler/rustc_codegen_cranelift/scripts/cargo.sh

@@ -4,7 +4,7 @@ dir=$(dirname "$0")
 source "$dir/config.sh"
 
 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat "$dir/rust-toolchain")
+TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/")
 
 cmd=$1
 shift || true

compiler/rustc_codegen_cranelift/scripts/config.sh

@@ -2,15 +2,7 @@
 
 set -e
 
-unamestr=$(uname)
+dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -)
-if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then
-   dylib_ext='so'
-elif [[ "$unamestr" == 'Darwin' ]]; then
-   dylib_ext='dylib'
-else
-   echo "Unsupported os"
-   exit 1
-fi
 
 if echo "$RUSTC_WRAPPER" | grep sccache; then
 echo
@@ -24,10 +16,10 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 export RUSTC=$dir"/bin/cg_clif"
 
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
+'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir
 
 # FIXME fix `#[linkage = "extern_weak"]` without this
-if [[ "$unamestr" == 'Darwin' ]]; then
+if [[ "$(uname)" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 

compiler/rustc_codegen_cranelift/scripts/rustup.sh

@@ -8,7 +8,7 @@ case $1 in
 
         echo "=> Installing new nightly"
         rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
-        echo "nightly-${TOOLCHAIN}" > rust-toolchain
+        sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
         rustup component add rustfmt || true
 
         echo "=> Uninstalling all old nighlies"

compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e
+
+./build.sh
+source build/config.sh
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
+git apply - <<EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 5bd1147cad5..10d68a2ff14 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+ rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
+ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+
++compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
++
+ [patch."https://github.com/rust-lang/rust-clippy"]
+ clippy_lints = { path = "src/tools/clippy/clippy_lints" }
+diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
+index 23e689fcae7..5f077b765b6 100644
+--- a/compiler/rustc_data_structures/Cargo.toml
++++ b/compiler/rustc_data_structures/Cargo.toml
+@@ -32,7 +32,6 @@ tempfile = "3.0.5"
+
+ [dependencies.parking_lot]
+ version = "0.11"
+-features = ["nightly"]
+
+ [target.'cfg(windows)'.dependencies]
+ winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
+ rand = "0.7"
+EOF
+
+cat > config.toml <<EOF
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../build/bin/cg_clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+EOF
+popd

compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh

@@ -3,70 +3,10 @@ set -e
 
 cd "$(dirname "$0")/../"
 
-./build.sh
+source ./scripts/setup_rust_fork.sh
-source build/config.sh
 
 echo "[TEST] Bootstrap of rustc"
-git clone https://github.com/rust-lang/rust.git || true
 pushd rust
-git fetch
-git checkout -- .
-git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
-
-git apply - <<EOF
-diff --git a/Cargo.toml b/Cargo.toml
-index 5bd1147cad5..10d68a2ff14 100644
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
- rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- 
-+compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
-+
- [patch."https://github.com/rust-lang/rust-clippy"]
- clippy_lints = { path = "src/tools/clippy/clippy_lints" }
-diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
-index 23e689fcae7..5f077b765b6 100644
---- a/compiler/rustc_data_structures/Cargo.toml
-+++ b/compiler/rustc_data_structures/Cargo.toml
-@@ -32,7 +32,6 @@ tempfile = "3.0.5"
-
- [dependencies.parking_lot]
- version = "0.11"
--features = ["nightly"]
-
- [target.'cfg(windows)'.dependencies]
- winapi = { version = "0.3", features = ["fileapi", "psapi"] }
-diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
-index d95b5b7f17f..00b6f0e3635 100644
---- a/library/alloc/Cargo.toml
-+++ b/library/alloc/Cargo.toml
-@@ -8,7 +8,7 @@ edition = "2018"
- 
- [dependencies]
- core = { path = "../core" }
--compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
- 
- [dev-dependencies]
- rand = "0.7"
-EOF
-
-cat > config.toml <<EOF
-[llvm]
-ninja = false
-
-[build]
-rustc = "$(pwd)/../build/bin/cg_clif"
-cargo = "$(rustup which cargo)"
-full-bootstrap = true
-local-rebuild = true
-
-[rust]
-codegen-backends = ["cranelift"]
-EOF
-
 rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+cargo install ripgrep
+
+rm -r src/test/ui/{extern/,panics/,unsized-locals/,thinlto/,simd*,*lto*.rs,linkage*,unwind-*.rs} || true
+for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto" src/test/ui); do
+  rm $test
+done
+
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
+  rm $test
+done
+
+git checkout -- src/test/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+
+# these all depend on unwinding support
+rm src/test/ui/backtrace.rs
+rm src/test/ui/array-slice-vec/box-of-array-of-drop-*.rs
+rm src/test/ui/array-slice-vec/slice-panic-*.rs
+rm src/test/ui/array-slice-vec/nested-vec-3.rs
+rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
+rm src/test/ui/issues/issue-26655.rs
+rm src/test/ui/issues/issue-29485.rs
+rm src/test/ui/issues/issue-30018-panic.rs
+rm src/test/ui/multi-panic.rs
+rm src/test/ui/sepcomp/sepcomp-unwind.rs
+rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
+rm src/test/ui/terminate-in-initializer.rs
+rm src/test/ui/threads-sendsync/task-stderr.rs
+rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
+rm src/test/ui/drop/drop-trait-enum.rs
+rm src/test/ui/numbers-arithmetic/issue-8460.rs
+
+rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
+rm src/test/ui/init-large-type.rs # same
+rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
+rm src/test/ui/issues/issue-33992.rs # unsupported linkages
+rm src/test/ui/issues/issue-51947.rs # same
+rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
+rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
+rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
+rm src/test/ui/generator/size-moved-locals.rs # same
+rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
+rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
+rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
+
+rm src/test/incremental/hashes/inline_asm.rs # inline asm
+rm src/test/incremental/issue-72386.rs # same
+rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
+rm src/test/incremental/issue-49482.rs # same
+rm src/test/incremental/issue-54059.rs # same
+rm src/test/incremental/lto.rs # requires lto
+
+rm src/test/pretty/asm.rs # inline asm
+rm src/test/pretty/raw-str-nonexpr.rs # same
+
+rm -r src/test/run-pass-valgrind/unsized-locals
+
+rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
+rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
+
+rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
+rm src/test/ui/cfg/cfg-panic.rs
+rm src/test/ui/default-alloc-error-hook.rs
+rm -r src/test/ui/hygiene/
+
+rm -r src/test/ui/polymorphization/ # polymorphization not yet supported
+rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same
+
+rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
+rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
+rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
+popd

compiler/rustc_codegen_cranelift/scripts/tests.sh

@@ -71,14 +71,20 @@ function base_sysroot_tests() {
     echo "[AOT] mod_bench"
     $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/mod_bench
-
-    pushd rand
-    rm -r ./target || true
-    ../build/cargo.sh test --workspace
-    popd
 }
 
 function extended_sysroot_tests() {
+    pushd rand
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-random/rand"
+        ../build/cargo.sh test --workspace
+    else
+        echo "[AOT] rust-random/rand"
+        ../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests
+    fi
+    popd
+
     pushd simple-raytracer
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
@@ -92,27 +98,40 @@ function extended_sysroot_tests() {
     else
         echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
         echo "[COMPILE] ebobby/simple-raytracer"
-        ../cargo.sh build
+        ../build/cargo.sh build --target $TARGET_TRIPLE
         echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
     fi
     popd
 
     pushd build_sysroot/sysroot_src/library/core/tests
     echo "[TEST] libcore"
-    rm -r ./target || true
+    cargo clean
-    ../../../../../build/cargo.sh test
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        ../../../../../build/cargo.sh test
+    else
+        ../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests
+    fi
     popd
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo.sh clean
+    cargo clean
     # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo.sh build --example shootout-regex-dna
+    ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
-    cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-    diff -u res.txt examples/regexdna-output.txt
+        cat examples/regexdna-input.txt \
+            | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \
+            | grep -v "Spawned thread" > res.txt
+        diff -u res.txt examples/regexdna-output.txt
+    fi
 
-    echo "[TEST] rust-lang/regex tests"
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-    ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+        echo "[TEST] rust-lang/regex tests"
+        ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    else
+        echo "[AOT] rust-lang/regex tests"
+        ../build/cargo.sh build --tests --target $TARGET_TRIPLE
+    fi
     popd
 }
 

compiler/rustc_codegen_cranelift/src/abi/comments.rs

@@ -11,9 +11,11 @@ use cranelift_codegen::entity::EntityRef;
 use crate::prelude::*;
 
 pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(
+    if fx.clif_comments.enabled() {
-        "kind  loc.idx   param    pass mode                            ty".to_string(),
+        fx.add_global_comment(
-    );
+            "kind  loc.idx   param    pass mode                            ty".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_arg_comment<'tcx>(
@@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>(
     arg_abi_mode: PassMode,
     arg_layout: TyAndLayout<'tcx>,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
+
     let local = if let Some(local) = local {
         Cow::Owned(format!("{:?}", local))
     } else {
@@ -59,10 +65,12 @@ pub(super) fn add_arg_comment<'tcx>(
 }
 
 pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(String::new());
+    if fx.clif_comments.enabled() {
-    fx.add_global_comment(
+        fx.add_global_comment(String::new());
-        "kind  local ty                              size align (abi,pref)".to_string(),
+        fx.add_global_comment(
-    );
+            "kind  local ty                              size align (abi,pref)".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_local_place_comments<'tcx>(
@@ -70,6 +78,9 @@ pub(super) fn add_local_place_comments<'tcx>(
     place: CPlace<'tcx>,
     local: Local,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
     let TyAndLayout { ty, layout } = place.layout();
     let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
         layout;
@@ -90,7 +101,7 @@ pub(super) fn add_local_place_comments<'tcx>(
             } else {
                 Cow::Borrowed("")
             };
-            match ptr.base_and_offset() {
+            match ptr.debug_base_and_offset() {
                 (crate::pointer::PointerBase::Addr(addr), offset) => {
                     ("reuse", format!("storage={}{}{}", addr, offset, meta).into())
                 }

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -1,6 +1,5 @@
 //! Handling of everything related to the calling convention. Also fills `fx.local_map`.
 
-#[cfg(debug_assertions)]
 mod comments;
 mod pass_mode;
 mod returning;
@@ -75,8 +74,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let func_id = import_function(self.tcx, self.cx.module, inst);
         let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
 
-        #[cfg(debug_assertions)]
+        if self.clif_comments.enabled() {
-        self.add_comment(func_ref, format!("{:?}", inst));
+            self.add_comment(func_ref, format!("{:?}", inst));
+        }
 
         func_ref
     }
@@ -92,8 +92,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
         let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
-        #[cfg(debug_assertions)]
+        if self.clif_comments.enabled() {
-        {
             self.add_comment(call_inst, format!("easy_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
@@ -149,7 +148,6 @@ fn make_local_place<'tcx>(
         CPlace::new_stack_slot(fx, layout)
     };
 
-    #[cfg(debug_assertions)]
     self::comments::add_local_place_comments(fx, place, local);
 
     place
@@ -163,7 +161,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
 
     let ssa_analyzed = crate::analyze::analyze(fx);
 
-    #[cfg(debug_assertions)]
     self::comments::add_args_header_comment(fx);
 
     let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
@@ -228,7 +225,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     fx.fn_abi = Some(fn_abi);
     assert!(block_params_iter.next().is_none(), "arg_value left behind");
 
-    #[cfg(debug_assertions)]
     self::comments::add_locals_header_comment(fx);
 
     for (local, arg_kind, ty) in func_params {
@@ -256,7 +252,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
                         CPlace::for_ptr(addr, val.layout())
                     };
 
-                    #[cfg(debug_assertions)]
                     self::comments::add_local_place_comments(fx, place, local);
 
                     assert_eq!(fx.local_map.push(place), local);
@@ -392,8 +387,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let (func_ref, first_arg) = match instance {
         // Trait object call
         Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
-            #[cfg(debug_assertions)]
+            if fx.clif_comments.enabled() {
-            {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(
                     nop_inst,
@@ -414,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
         // Indirect call
         None => {
-            #[cfg(debug_assertions)]
+            if fx.clif_comments.enabled() {
-            {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(nop_inst, "indirect call");
             }
@@ -477,10 +470,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
         if !matches!(fn_sig.abi, Abi::C { .. }) {
-            fx.tcx.sess.span_fatal(
+            fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
-                span,
-                &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
-            );
         }
         let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
         let abi_params = call_args

compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

@@ -208,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
     });
     let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
     let mut offset = 0;
-    let mut block_params_iter = block_params.into_iter().copied();
+    let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {
         let val = ptr.offset_i64(fx, offset).store(
             fx,
@@ -248,8 +248,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
+    local: Option<mir::Local>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
+    local_field: Option<usize>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> Option<CValue<'tcx>> {
@@ -263,7 +263,6 @@ pub(super) fn cvalue_for_param<'tcx>(
         })
         .collect::<SmallVec<[_; 2]>>();
 
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "arg",

compiler/rustc_codegen_cranelift/src/abi/returning.rs

@@ -84,10 +84,6 @@ pub(super) fn codegen_return_param<'tcx>(
         }
     };
 
-    #[cfg(not(debug_assertions))]
-    let _ = ret_param;
-
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "ret",
@@ -146,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, T>(
                 let results = fx
                     .bcx
                     .inst_results(call_inst)
-                    .into_iter()
+                    .iter()
                     .copied()
                     .collect::<SmallVec<[Value; 2]>>();
                 let result =

compiler/rustc_codegen_cranelift/src/allocator.rs

@@ -3,6 +3,7 @@
 
 use crate::prelude::*;
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 use rustc_span::symbol::sym;
 
@@ -92,7 +93,7 @@ fn codegen_inner(
             bcx.finalize();
         }
         module
-            .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap();
         unwind_context.add_function(func_id, &ctx, module.isa());
     }
@@ -132,7 +133,7 @@ fn codegen_inner(
         bcx.finalize();
     }
     module
-        .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+        .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
         .unwrap();
     unwind_context.add_function(func_id, &ctx, module.isa());
 }

compiler/rustc_codegen_cranelift/src/base.rs

@@ -1,5 +1,6 @@
 //! Codegen of a single function
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiExt;
@@ -7,11 +8,7 @@ use rustc_target::abi::call::FnAbi;
 
 use crate::prelude::*;
 
-pub(crate) fn codegen_fn<'tcx>(
+pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
-    cx: &mut crate::CodegenCx<'_, 'tcx>,
-    instance: Instance<'tcx>,
-    linkage: Linkage,
-) {
     let tcx = cx.tcx;
 
     let _inst_guard =
@@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>(
     // Declare function
     let name = tcx.symbol_name(instance).name.to_string();
     let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
-    let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
+    let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
 
     cx.cached_context.clear();
 
@@ -131,7 +128,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let module = &mut cx.module;
     tcx.sess.time("define function", || {
         module
-            .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap()
     });
 
@@ -219,8 +216,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             codegen_stmt(fx, block, stmt);
         }
 
-        #[cfg(debug_assertions)]
+        if fx.clif_comments.enabled() {
-        {
             let mut terminator_head = "\n".to_string();
             bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
@@ -433,12 +429,14 @@ fn codegen_stmt<'tcx>(
 
     fx.set_debug_loc(stmt.source_info);
 
-    #[cfg(false_debug_assertions)]
+    #[cfg(disabled)]
     match &stmt.kind {
         StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
-            let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+            if fx.clif_comments.enabled() {
-            fx.add_comment(inst, format!("{:?}", stmt));
+                let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+                fx.add_comment(inst, format!("{:?}", stmt));
+            }
         }
     }
 
@@ -464,16 +462,16 @@ fn codegen_stmt<'tcx>(
                     let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
                     lval.write_cvalue(fx, val);
                 }
-                Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => {
+                Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
-                    let lhs = codegen_operand(fx, lhs);
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
-                    let rhs = codegen_operand(fx, rhs);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
                     lval.write_cvalue(fx, res);
                 }
-                Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => {
+                Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
-                    let lhs = codegen_operand(fx, lhs);
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
-                    let rhs = codegen_operand(fx, rhs);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = if !fx.tcx.sess.overflow_checks() {
                         let val =
@@ -659,7 +657,9 @@ fn codegen_stmt<'tcx>(
                         .val
                         .try_to_bits(fx.tcx.data_layout.pointer_size)
                         .unwrap();
-                    if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+                    if operand.layout().size.bytes() == 0 {
+                        // Do nothing for ZST's
+                    } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
                         let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
@@ -832,25 +832,18 @@ fn codegen_stmt<'tcx>(
             }
         }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
-        StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+        StatementKind::CopyNonOverlapping(inner) => {
-          src,
+            let dst = codegen_operand(fx, &inner.dst);
-          dst,
-          count,
-        }) => {
-            let dst = codegen_operand(fx, dst);
             let pointee = dst
-              .layout()
+                .layout()
-              .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+                .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
-              .expect("Expected pointer");
+                .expect("Expected pointer");
             let dst = dst.load_scalar(fx);
-            let src = codegen_operand(fx, src).load_scalar(fx);
+            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
-            let count = codegen_operand(fx, count).load_scalar(fx);
+            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
             let elem_size: u64 = pointee.size.bytes();
-            let bytes = if elem_size != 1 {
+            let bytes =
-               fx.bcx.ins().imul_imm(count, elem_size as i64)
+                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
-            } else {
-               count
-            };
             fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
         }
     }

compiler/rustc_codegen_cranelift/src/codegen_i128.rs

@@ -32,18 +32,56 @@ pub(crate) fn maybe_codegen<'tcx>(
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked => {
             let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-            Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            if fx.tcx.sess.target.is_like_windows {
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args =
+                    [ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                fx.lib_call(
+                    "__multi3",
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    vec![],
+                    &args,
+                );
+                Some(ret_place.to_cvalue(fx))
+            } else {
+                Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            }
         }
         BinOp::Add | BinOp::Sub | BinOp::Mul => {
             assert!(checked);
             let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let param_types = vec![
+            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
-                AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
-                AbiParam::new(types::I128),
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
-                AbiParam::new(types::I128),
+                assert!(lhs_extra.is_none());
-            ];
+                assert!(rhs_extra.is_none());
-            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
+                )
+            } else {
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(types::I128),
+                        AbiParam::new(types::I128),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
+                )
+            };
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -57,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>(
             Some(out_place.to_cvalue(fx))
         }
         BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
-        BinOp::Div => {
+        BinOp::Div | BinOp::Rem => {
             assert!(!checked);
-            if is_signed {
+            let name = match (bin_op, is_signed) {
-                Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128))
+                (BinOp::Div, false) => "__udivti3",
+                (BinOp::Div, true) => "__divti3",
+                (BinOp::Rem, false) => "__umodti3",
+                (BinOp::Rem, true) => "__modti3",
+                _ => unreachable!(),
+            };
+            if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let ret = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))],
+                    vec![AbiParam::new(types::I64X2)],
+                    &args,
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
+                Some(ret_place.to_cvalue(fx))
             } else {
-                Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128))
+                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
-            }
-        }
-        BinOp::Rem => {
-            assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128))
-            } else {
-                Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128))
             }
         }
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {

compiler/rustc_codegen_cranelift/src/common.rs

@@ -361,8 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let _ = self.cx.module.define_data(msg_id, &data_ctx);
 
         let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
-        #[cfg(debug_assertions)]
+        if self.clif_comments.enabled() {
-        {
             self.add_comment(local_msg_id, msg);
         }
         self.bcx.ins().global_value(self.pointer_type, local_msg_id)

compiler/rustc_codegen_cranelift/src/compiler_builtins.rs

@@ -0,0 +1,41 @@
+macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
+    #[cfg(feature = "jit")]
+    #[allow(improper_ctypes)]
+    extern "C" {
+        $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
+    }
+
+    #[cfg(feature = "jit")]
+    pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
+        for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] {
+            builder.symbol(name, val);
+        }
+    }
+}
+
+builtin_functions! {
+    register_functions_for_jit;
+
+    // integers
+    fn __multi3(a: i128, b: i128) -> i128;
+    fn __udivti3(n: u128, d: u128) -> u128;
+    fn __divti3(n: i128, d: i128) -> i128;
+    fn __umodti3(n: u128, d: u128) -> u128;
+    fn __modti3(n: i128, d: i128) -> i128;
+    fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
+
+    // floats
+    fn __floattisf(i: i128) -> f32;
+    fn __floattidf(i: i128) -> f64;
+    fn __floatuntisf(i: u128) -> f32;
+    fn __floatuntidf(i: u128) -> f64;
+    fn __fixsfti(f: f32) -> i128;
+    fn __fixdfti(f: f64) -> i128;
+    fn __fixunssfti(f: f32) -> u128;
+    fn __fixunsdfti(f: f64) -> u128;
+}

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -45,9 +45,9 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
         };
         match const_.val {
             ConstKind::Value(_) => {}
-            ConstKind::Unevaluated(def, ref substs, promoted) => {
+            ConstKind::Unevaluated(unevaluated) => {
                 if let Err(err) =
-                    fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
+                    fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None)
                 {
                     all_constants_ok = false;
                     match err {
@@ -85,8 +85,9 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 ) -> CValue<'tcx> {
     let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
+    if fx.clif_comments.enabled() {
-    fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+        fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    }
     let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
     CValue::by_val(tls_ptr, layout)
 }
@@ -98,8 +99,9 @@ fn codegen_static_ref<'tcx>(
 ) -> CPlace<'tcx> {
     let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
+    if fx.clif_comments.enabled() {
-    fx.add_comment(local_data_id, format!("{:?}", def_id));
+        fx.add_comment(local_data_id, format!("{:?}", def_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     assert!(!layout.is_unsized(), "unsized statics aren't supported");
     assert!(
@@ -122,14 +124,16 @@ pub(crate) fn codegen_constant<'tcx>(
     };
     let const_val = match const_.val {
         ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(def, ref substs, promoted) if fx.tcx.is_static(def.did) => {
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
             return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
         }
-        ConstKind::Unevaluated(def, ref substs, promoted) => {
+        ConstKind::Unevaluated(unevaluated) => {
-            match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None) {
+            match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
                 Ok(const_val) => const_val,
                 Err(_) => {
                     span_bug!(constant.span, "erroneous constant not captured by required_consts");
@@ -183,8 +187,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                                 data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
+                            if fx.clif_comments.enabled() {
-                            fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                                fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         Some(GlobalAlloc::Function(instance)) => {
@@ -199,8 +204,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                             let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
+                            if fx.clif_comments.enabled() {
-                            fx.add_comment(local_data_id, format!("{:?}", def_id));
+                                fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         None => bug!("missing allocation {:?}", ptr.alloc_id),
@@ -241,8 +247,9 @@ fn pointer_for_allocation<'tcx>(
     let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
 
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
+    if fx.clif_comments.enabled() {
-    fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+        fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     crate::pointer::Pointer::new(global_ptr)
 }

compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs

@@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
     #[cfg(unix)]
     {
         use std::os::unix::ffi::OsStrExt;
-        return path.as_bytes();
+        path.as_bytes()
     }
     #[cfg(not(unix))]
     {
-        return path.to_str().unwrap().as_bytes();
+        path.to_str().unwrap().as_bytes()
     }
 }
 

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -119,11 +119,10 @@ fn module_codegen(
         tcx.sess.opts.debuginfo != DebugInfo::None,
     );
     super::predefine_mono_items(&mut cx, &mono_items);
-    for (mono_item, (linkage, visibility)) in mono_items {
+    for (mono_item, _) in mono_items {
-        let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
         match mono_item {
             MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
             }
             MonoItem::Static(def_id) => {
                 crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@@ -163,6 +162,21 @@ pub(super) fn run_aot(
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
+    use rustc_span::symbol::sym;
+
+    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
+    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
+    let windows_subsystem = subsystem.map(|subsystem| {
+        if subsystem != sym::windows && subsystem != sym::console {
+            tcx.sess.fatal(&format!(
+                "invalid windows subsystem `{}`, only \
+                                    `windows` and `console` are allowed",
+                subsystem
+            ));
+        }
+        subsystem.to_string()
+    });
+
     let mut work_products = FxHashMap::default();
 
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
@@ -280,7 +294,7 @@ pub(super) fn run_aot(
             allocator_module,
             metadata_module,
             metadata,
-            windows_subsystem: None, // Windows is not yet supported
+            windows_subsystem,
             linker_info: LinkerInfo::new(tcx),
             crate_info: CrateInfo::new(tcx),
         },

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -5,8 +5,10 @@ use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
+use rustc_session::config::EntryFnType;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
@@ -28,20 +30,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     let mut jit_builder =
         JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
     jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
+    crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
-    let sig = Signature {
-        params: vec![
-            AbiParam::new(jit_module.target_config().pointer_type()),
-            AbiParam::new(jit_module.target_config().pointer_type()),
-        ],
-        returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
-        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-    };
-    let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
-
     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
     let mono_items = cgus
         .iter()
@@ -55,15 +48,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
 
     super::time(tcx, "codegen mono items", || {
         super::predefine_mono_items(&mut cx, &mono_items);
-        for (mono_item, (linkage, visibility)) in mono_items {
+        for (mono_item, _) in mono_items {
-            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
             match mono_item {
                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx
+                        cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
-                            .sess
-                            .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
                     }
                     CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
                 },
@@ -86,24 +76,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
     tcx.sess.abort_if_errors();
 
     jit_module.finalize_definitions();
-
     let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
 
-    let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
-
     println!(
         "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
     );
 
-    let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-        unsafe { ::std::mem::transmute(finalized_main) };
-
     let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
     let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
         .chain(args.split(' '))
@@ -118,12 +101,58 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     BACKEND_CONFIG.with(|tls_backend_config| {
         assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
     });
-    CURRENT_MODULE
-        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
 
-    let ret = f(args.len() as c_int, argv.as_ptr());
+    let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
+    let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
 
-    std::process::exit(ret);
+    match entry_ty {
+        EntryFnType::Main => {
+            // FIXME set program arguments somehow
+
+            let main_sig = Signature {
+                params: vec![],
+                returns: vec![],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let main_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
+                .unwrap();
+            let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
+            f();
+            std::process::exit(0);
+        }
+        EntryFnType::Start => {
+            let start_sig = Signature {
+                params: vec![
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                ],
+                returns: vec![AbiParam::new(
+                    jit_module.target_config().pointer_type(), /*isize*/
+                )],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let start_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
+                .unwrap();
+            let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
+                unsafe { ::std::mem::transmute(finalized_start) };
+            let ret = f(args.len() as c_int, argv.as_ptr());
+            std::process::exit(ret);
+        }
+    }
 }
 
 #[no_mangle]
@@ -144,8 +173,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
             jit_module.prepare_for_function_redefine(func_id).unwrap();
 
             let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
-            tcx.sess
+            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
-                .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
 
             let (global_asm, _debug_context, unwind_context) = cx.finalize();
             assert!(global_asm.is_empty());
@@ -220,7 +248,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     imported_symbols
 }
 
-pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let pointer_type = cx.module.target_config().pointer_type();
@@ -267,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'t
         .define_function(
             func_id,
             &mut Context::for_function(trampoline),
-            &mut cranelift_codegen::binemit::NullTrapSink {},
+            &mut NullTrapSink {},
+            &mut NullStackMapSink {},
         )
         .unwrap();
 }

compiler/rustc_codegen_cranelift/src/driver/mod.rs

@@ -44,13 +44,19 @@ fn predefine_mono_items<'tcx>(
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
     cx.tcx.sess.time("predefine functions", || {
+        let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
                 MonoItem::Fn(instance) => {
                     let name = cx.tcx.symbol_name(instance).name.to_string();
                     let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
                     let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
-                    let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+                    let linkage = crate::linkage::get_clif_linkage(
+                        mono_item,
+                        linkage,
+                        visibility,
+                        is_compiler_builtins,
+                    );
                     cx.module.declare_function(&name, linkage, &sig).unwrap();
                 }
                 MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     if template.is_empty() {
         // Black box
         return;
+    } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+        let true_ = fx.bcx.ins().iconst(types::I32, 1);
+        fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+        return;
     }
 
     let mut slot_size = Size::from_bytes(0);
@@ -193,8 +197,9 @@ fn call_inline_asm<'tcx>(
         offset: None,
         size: u32::try_from(slot_size.bytes()).unwrap(),
     });
-    #[cfg(debug_assertions)]
+    if fx.clif_comments.enabled() {
-    fx.add_comment(stack_slot, "inline asm scratch slot");
+        fx.add_comment(stack_slot, "inline asm scratch slot");
+    }
 
     let inline_asm_func = fx
         .cx
@@ -210,8 +215,9 @@ fn call_inline_asm<'tcx>(
         )
         .unwrap();
     let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
+    if fx.clif_comments.enabled() {
-    fx.add_comment(inline_asm_func, asm_name);
+        fx.add_comment(inline_asm_func, asm_name);
+    }
 
     for (_reg, offset, value) in inputs {
         fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -88,7 +88,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let idx_bytes = match idx_const {
                     ConstValue::ByRef { alloc, offset } => {
                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
-                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
+                        let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
                         alloc.get_bytes(fx, ptr, size).unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
@@ -277,5 +277,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         // simd_select
         // simd_rem
         // simd_neg
+        // simd_trunc
+        // simd_floor
     }
 }

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -1,13 +1,4 @@
-#![feature(
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
-    rustc_private,
-    decl_macro,
-    type_alias_impl_trait,
-    associated_type_bounds,
-    never_type,
-    try_blocks,
-    box_patterns,
-    hash_drain_filter
-)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
@@ -57,6 +48,7 @@ mod base;
 mod cast;
 mod codegen_i128;
 mod common;
+mod compiler_builtins;
 mod constant;
 mod debuginfo;
 mod discriminant;
@@ -224,8 +216,10 @@ pub struct CraneliftCodegenBackend {
 
 impl CodegenBackend for CraneliftCodegenBackend {
     fn init(&self, sess: &Session) {
-        if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode {
+        use rustc_session::config::Lto;
-            sess.warn("LTO is not supported. You may get a linker error.");
+        match sess.lto() {
+            Lto::No | Lto::ThinLocal => {}
+            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
         }
     }
 
@@ -240,9 +234,9 @@ impl CodegenBackend for CraneliftCodegenBackend {
         vec![]
     }
 
-    fn codegen_crate<'tcx>(
+    fn codegen_crate(
         &self,
-        tcx: TyCtxt<'tcx>,
+        tcx: TyCtxt<'_>,
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
@@ -252,9 +246,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                 .unwrap_or_else(|err| tcx.sess.fatal(&err))
         };
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
+        driver::codegen_crate(tcx, metadata, need_metadata_module, config)
-
-        res
     }
 
     fn join_codegen(
@@ -300,9 +292,9 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     let mut flags_builder = settings::builder();
     flags_builder.enable("is_pic").unwrap();
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
-    flags_builder
+    let enable_verifier =
-        .set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
+        cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
-        .unwrap();
+    flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
 
     let tls_model = match target_triple.binary_format {
         BinaryFormat::Elf => "elf_gd",
@@ -314,18 +306,17 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
 
     flags_builder.set("enable_simd", "true").unwrap();
 
+    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
             flags_builder.set("opt_level", "none").unwrap();
         }
         OptLevel::Less | OptLevel::Default => {}
-        OptLevel::Aggressive => {
+        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
             flags_builder.set("opt_level", "speed_and_size").unwrap();
         }
-        OptLevel::Size | OptLevel::SizeMin => {
-            sess.warn("Optimizing for size is not supported. Just ignoring the request");
-        }
     }
 
     let flags = settings::Flags::new(flags_builder);

compiler/rustc_codegen_cranelift/src/linkage.rs

@@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage(
     mono_item: MonoItem<'_>,
     linkage: RLinkage,
     visibility: Visibility,
+    is_compiler_builtins: bool,
 ) -> Linkage {
     match (linkage, visibility) {
+        (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden,
         (RLinkage::External, Visibility::Default) => Linkage::Export,
         (RLinkage::Internal, Visibility::Default) => Linkage::Local,
         (RLinkage::External, Visibility::Hidden) => Linkage::Hidden,

compiler/rustc_codegen_cranelift/src/main_shim.rs

@@ -1,3 +1,4 @@
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
 use rustc_session::config::EntryFnType;
 
@@ -12,7 +13,7 @@ pub(crate) fn maybe_create_entry_wrapper(
 ) {
     let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) {
         Some((def_id, entry_ty)) => (
-            def_id.to_def_id(),
+            def_id,
             match entry_ty {
                 EntryFnType::Main => true,
                 EntryFnType::Start => false,
@@ -100,12 +101,8 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        m.define_function(
+        m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
-            cmain_func_id,
+            .unwrap();
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
         unwind_context.add_function(cmain_func_id, &ctx, m.isa());
     }
 }

compiler/rustc_codegen_cranelift/src/metadata.rs

@@ -1,10 +1,10 @@
 //! Reading and writing of the rustc metadata for rlibs and dylibs
 
-use std::convert::TryFrom;
 use std::fs::File;
 use std::path::Path;
 
 use rustc_codegen_ssa::METADATA_FILENAME;
+use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::owning_ref::OwningRef;
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
@@ -17,38 +17,43 @@ use crate::backend::WriteMetadata;
 
 pub(crate) struct CraneliftMetadataLoader;
 
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
+    let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?;
+    let metadata = OwningRef::new(data).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
 impl MetadataLoader for CraneliftMetadataLoader {
     fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
+        load_metadata_with(path, |data| {
-        // Iterate over all entries in the archive:
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
-        while let Some(entry_result) = archive.next_entry() {
+                .map_err(|e| format!("{:?}", e))?;
-            let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
-                let mut buf = Vec::with_capacity(
-                    usize::try_from(entry.header().size())
-                        .expect("Rlib metadata file too big to load into memory."),
-                );
-                ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-                return Ok(rustc_erase_owner!(buf.map_owner_box()));
-            }
-        }
 
-        Err("couldn't find metadata entry".to_string())
+            for entry_result in archive.members() {
+                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
+            }
+
+            Err("couldn't find metadata entry".to_string())
+        })
     }
 
     fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
         use object::{Object, ObjectSection};
-        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
+
-        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
+        load_metadata_with(path, |data| {
-        let buf = file
+            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
-            .section_by_name(".rustc")
+            file.section_by_name(".rustc")
-            .ok_or("no .rustc section")?
+                .ok_or("no .rustc section")?
-            .data()
+                .data()
-            .map_err(|e| format!("failed to read .rustc section: {:?}", e))?
+                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
-            .to_owned();
+        })
-        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-        Ok(rustc_erase_owner!(buf.map_owner_box()))
     }
 }
 

compiler/rustc_codegen_cranelift/src/num.rs

@@ -166,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
         BinOp::Shl => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             fx.bcx.ins().ishl(lhs, actual_shift)
         }
         BinOp::Shr => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             if signed {
                 fx.bcx.ins().sshr(lhs, actual_shift)
             } else {
@@ -387,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let lhs = in_lhs.load_scalar(fx);
                 let rhs = in_rhs.load_scalar(fx);
 
-                return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
+                codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
             }
             BinOp::Offset => {
                 let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
@@ -396,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
                 let base_val = base.load_scalar(fx);
                 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
-                return CValue::by_val(res, base.layout());
+                CValue::by_val(res, base.layout())
             }
             _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
-        };
+        }
     } else {
         let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
         let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);

compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs

@@ -181,7 +181,6 @@ impl<'a> OptimizeContext<'a> {
 
 pub(super) fn optimize_function(
     ctx: &mut Context,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))]
     clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
     combine_stack_addr_with_load_store(&mut ctx.func);
@@ -192,8 +191,7 @@ pub(super) fn optimize_function(
 
     remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
 
-    #[cfg(debug_assertions)]
+    if clif_comments.enabled() {
-    {
         for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
             clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
         }
@@ -209,25 +207,27 @@ pub(super) fn optimize_function(
         for load in users.stack_load.clone().into_iter() {
             let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
 
-            #[cfg(debug_assertions)]
+            if clif_comments.enabled() {
-            for &store in &potential_stores {
+                for &store in &potential_stores {
-                clif_comments.add_comment(
+                    clif_comments.add_comment(
-                    load,
+                        load,
-                    format!(
+                        format!(
-                        "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
+                            "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
-                    ),
+                        ),
-                );
+                    );
+                }
             }
 
             match *potential_stores {
                 [] => {
-                    #[cfg(debug_assertions)]
+                    if clif_comments.enabled() {
-                    clif_comments
+                        clif_comments
-                        .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                            .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    }
                 }
                 [store]
                     if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
@@ -237,9 +237,12 @@ pub(super) fn optimize_function(
                     // Only one store could have been the origin of the value.
                     let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
 
-                    #[cfg(debug_assertions)]
+                    if clif_comments.enabled() {
-                    clif_comments
+                        clif_comments.add_comment(
-                        .add_comment(load, format!("Store to load forward {} -> {}", store, load));
+                            load,
+                            format!("Store to load forward {} -> {}", store, load),
+                        );
+                    }
 
                     users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
                 }
@@ -250,33 +253,35 @@ pub(super) fn optimize_function(
         for store in users.stack_store.clone().into_iter() {
             let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
 
-            #[cfg(debug_assertions)]
+            if clif_comments.enabled() {
-            for &load in &potential_loads {
+                for &load in &potential_loads {
-                clif_comments.add_comment(
+                    clif_comments.add_comment(
-                    store,
+                        store,
-                    format!(
+                        format!(
-                        "Potential load from store {} <- {} ({:?}, {:?})",
+                            "Potential load from store {} <- {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
-                    ),
+                        ),
-                );
+                    );
+                }
             }
 
             if potential_loads.is_empty() {
                 // Never loaded; can safely remove all stores and the stack slot.
                 // FIXME also remove stores when there is always a next store before a load.
 
-                #[cfg(debug_assertions)]
+                if clif_comments.enabled() {
-                clif_comments.add_comment(
+                    clif_comments.add_comment(
-                    store,
+                        store,
-                    format!(
+                        format!(
-                        "Remove dead stack store {} of {}",
+                            "Remove dead stack store {} of {}",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        stack_slot.0
+                            stack_slot.0
-                    ),
+                        ),
-                );
+                    );
+                }
 
                 users.remove_dead_store(&mut opt_ctx.ctx.func, store);
             }

compiler/rustc_codegen_cranelift/src/pointer.rs

@@ -39,8 +39,7 @@ impl Pointer {
         Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
     }
 
-    #[cfg(debug_assertions)]
+    pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) {
-    pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) {
         (self.base, self.offset)
     }
 

compiler/rustc_codegen_cranelift/src/pretty_clif.rs

@@ -69,13 +69,15 @@ use crate::prelude::*;
 
 #[derive(Debug)]
 pub(crate) struct CommentWriter {
+    enabled: bool,
     global_comments: Vec<String>,
     entity_comments: FxHashMap<AnyEntity, String>,
 }
 
 impl CommentWriter {
     pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
-        let global_comments = if cfg!(debug_assertions) {
+        let enabled = should_write_ir(tcx);
+        let global_comments = if enabled {
             vec![
                 format!("symbol {}", tcx.symbol_name(instance).name),
                 format!("instance {:?}", instance),
@@ -86,13 +88,17 @@ impl CommentWriter {
             vec![]
         };
 
-        CommentWriter { global_comments, entity_comments: FxHashMap::default() }
+        CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
     }
 }
 
-#[cfg(debug_assertions)]
 impl CommentWriter {
+    pub(crate) fn enabled(&self) -> bool {
+        self.enabled
+    }
+
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
+        debug_assert!(self.enabled);
         self.global_comments.push(comment.into());
     }
 
@@ -101,6 +107,8 @@ impl CommentWriter {
         entity: E,
         comment: S,
     ) {
+        debug_assert!(self.enabled);
+
         use std::collections::hash_map::Entry;
         match self.entity_comments.entry(entity.into()) {
             Entry::Occupied(mut occ) => {
@@ -179,7 +187,6 @@ impl FuncWriter for &'_ CommentWriter {
     }
 }
 
-#[cfg(debug_assertions)]
 impl FunctionCx<'_, '_, '_> {
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
         self.clif_comments.add_global_comment(comment);
@@ -198,8 +205,8 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
     tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
 }
 
-pub(crate) fn write_ir_file<'tcx>(
+pub(crate) fn write_ir_file(
-    tcx: TyCtxt<'tcx>,
+    tcx: TyCtxt<'_>,
     name: &str,
     write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
@@ -217,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>(
 
     let clif_file_name = clif_output_dir.join(name);
 
-    let res: std::io::Result<()> = try {
+    let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
-        let mut file = std::fs::File::create(clif_file_name)?;
-        write(&mut file)?;
-    };
     if let Err(err) = res {
         tcx.sess.warn(&format!("error writing ir file: {}", err));
     }

compiler/rustc_codegen_cranelift/src/trap.rs

@@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
         )
         .unwrap();
     let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
+    if fx.clif_comments.enabled() {
-    {
         fx.add_comment(puts, "puts");
     }
 

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -2,7 +2,6 @@
 
 use crate::prelude::*;
 
-use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
 fn codegen_field<'tcx>(
@@ -414,7 +413,7 @@ impl<'tcx> CPlace<'tcx> {
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
-        #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
+        method: &'static str,
     ) {
         fn transmute_value<'tcx>(
             fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -462,8 +461,7 @@ impl<'tcx> CPlace<'tcx> {
 
         assert_eq!(self.layout().size, from.layout().size);
 
-        #[cfg(debug_assertions)]
+        if fx.clif_comments.enabled() {
-        {
             use cranelift_codegen::cursor::{Cursor, CursorPosition};
             let cur_block = match fx.bcx.cursor().position() {
                 CursorPosition::After(block) => block,
@@ -707,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>(
             }
             // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
         }
+        (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
+            if adt_def_a.did == adt_def_b.did =>
+        {
+            let mut types_a = substs_a.types();
+            let mut types_b = substs_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
         _ => {
             assert_eq!(
                 from_ty, to_ty,

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -1,6 +1,6 @@
 //! Codegen vtables and vtable accesses.
 //!
-//! See librustc_codegen_llvm/meth.rs for reference
+//! See `rustc_codegen_ssa/src/meth.rs` for reference.
 // FIXME dedup this logic between miri, cg_llvm and cg_clif
 
 use crate::prelude::*;

compiler/rustc_codegen_llvm/src/abi.rs

@@ -41,12 +41,23 @@ impl ArgAttributeExt for ArgAttribute {
 }
 
 pub trait ArgAttributesExt {
-    fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value);
+    fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value);
-    fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value);
+    fn apply_attrs_to_callsite(
+        &self,
+        idx: AttributePlace,
+        cx: &CodegenCx<'_, '_>,
+        callsite: &Value,
+    );
+}
+
+fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool {
+    // While #84958 has been fixed, mutable noalias is not enabled by default
+    // in Rust 1.53 out of an abundance of caution.
+    cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(false)
 }
 
 impl ArgAttributesExt for ArgAttributes {
-    fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value) {
+    fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
         let mut regular = self.regular;
         unsafe {
             let deref = self.pointee_size.bytes();
@@ -62,6 +73,9 @@ impl ArgAttributesExt for ArgAttributes {
                 llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32);
             }
             regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn));
+            if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
+                llvm::Attribute::NoAlias.apply_llfn(idx, llfn);
+            }
             match self.arg_ext {
                 ArgExtension::None => {}
                 ArgExtension::Zext => {
@@ -74,7 +88,12 @@ impl ArgAttributesExt for ArgAttributes {
         }
     }
 
-    fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value) {
+    fn apply_attrs_to_callsite(
+        &self,
+        idx: AttributePlace,
+        cx: &CodegenCx<'_, '_>,
+        callsite: &Value,
+    ) {
         let mut regular = self.regular;
         unsafe {
             let deref = self.pointee_size.bytes();
@@ -98,6 +117,9 @@ impl ArgAttributesExt for ArgAttributes {
                 );
             }
             regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite));
+            if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
+                llvm::Attribute::NoAlias.apply_callsite(idx, callsite);
+            }
             match self.arg_ext {
                 ArgExtension::None => {}
                 ArgExtension::Zext => {
@@ -419,13 +441,13 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 
         let mut i = 0;
         let mut apply = |attrs: &ArgAttributes| {
-            attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), llfn);
+            attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn);
             i += 1;
             i - 1
         };
         match self.ret.mode {
             PassMode::Direct(ref attrs) => {
-                attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, llfn);
+                attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
             }
             PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
                 assert!(!on_stack);
@@ -480,18 +502,18 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite.
 
         let mut i = 0;
-        let mut apply = |attrs: &ArgAttributes| {
+        let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| {
-            attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), callsite);
+            attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite);
             i += 1;
             i - 1
         };
         match self.ret.mode {
             PassMode::Direct(ref attrs) => {
-                attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, callsite);
+                attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, &bx.cx, callsite);
             }
             PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
                 assert!(!on_stack);
-                let i = apply(attrs);
+                let i = apply(bx.cx, attrs);
                 unsafe {
                     llvm::LLVMRustAddStructRetCallSiteAttr(
                         callsite,
@@ -517,12 +539,12 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         }
         for arg in &self.args {
             if arg.pad.is_some() {
-                apply(&ArgAttributes::new());
+                apply(bx.cx, &ArgAttributes::new());
             }
             match arg.mode {
                 PassMode::Ignore => {}
                 PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
-                    let i = apply(attrs);
+                    let i = apply(bx.cx, attrs);
                     unsafe {
                         llvm::LLVMRustAddByValCallSiteAttr(
                             callsite,
@@ -533,22 +555,22 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 }
                 PassMode::Direct(ref attrs)
                 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
-                    apply(attrs);
+                    apply(bx.cx, attrs);
                 }
                 PassMode::Indirect {
                     ref attrs,
                     extra_attrs: Some(ref extra_attrs),
                     on_stack: _,
                 } => {
-                    apply(attrs);
+                    apply(bx.cx, attrs);
-                    apply(extra_attrs);
+                    apply(bx.cx, extra_attrs);
                 }
                 PassMode::Pair(ref a, ref b) => {
-                    apply(a);
+                    apply(bx.cx, a);
-                    apply(b);
+                    apply(bx.cx, b);
                 }
                 PassMode::Cast(_) => {
-                    apply(&ArgAttributes::new());
+                    apply(bx.cx, &ArgAttributes::new());
                 }
             }
         }

compiler/rustc_codegen_llvm/src/asm.rs

@@ -14,7 +14,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir as hir;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::{bug, span_bug};
-use rustc_span::{Pos, Span};
+use rustc_span::{Pos, Span, Symbol};
 use rustc_target::abi::*;
 use rustc_target::asm::*;
 
@@ -125,15 +125,39 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 
         // Collect the types of output operands
         let mut constraints = vec![];
+        let mut clobbers = vec![];
         let mut output_types = vec![];
         let mut op_idx = FxHashMap::default();
         for (idx, op) in operands.iter().enumerate() {
             match *op {
                 InlineAsmOperandRef::Out { reg, late, place } => {
+                    let is_target_supported = |reg_class: InlineAsmRegClass| {
+                        for &(_, feature) in reg_class.supported_types(asm_arch) {
+                            if let Some(feature) = feature {
+                                if self.tcx.sess.target_features.contains(&Symbol::intern(feature))
+                                {
+                                    return true;
+                                }
+                            } else {
+                                // Register class is unconditionally supported
+                                return true;
+                            }
+                        }
+                        false
+                    };
+
                     let mut layout = None;
                     let ty = if let Some(ref place) = place {
                         layout = Some(&place.layout);
                         llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
+                    } else if !is_target_supported(reg.reg_class()) {
+                        // We turn discarded outputs into clobber constraints
+                        // if the target feature needed by the register class is
+                        // disabled. This is necessary otherwise LLVM will try
+                        // to actually allocate a register for the dummy output.
+                        assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_)));
+                        clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
+                        continue;
                     } else {
                         // If the output is discarded, we don't really care what
                         // type is used. We're just using this to tell LLVM to
@@ -244,6 +268,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             }
         }
 
+        constraints.append(&mut clobbers);
         if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
             match asm_arch {
                 InlineAsmArch::AArch64 | InlineAsmArch::Arm => {

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -11,9 +11,10 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_session::config::{OptLevel, SanitizerSet};
+use rustc_session::config::OptLevel;
 use rustc_session::Session;
-use rustc_target::spec::StackProbeType;
+use rustc_target::spec::abi::Abi;
+use rustc_target::spec::{SanitizerSet, StackProbeType};
 
 use crate::attributes;
 use crate::llvm::AttributePlace::Function;
@@ -254,6 +255,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
         attributes::emit_uwtable(llfn, true);
     }
 
+    // FIXME: none of these three functions interact with source level attributes.
     set_frame_pointer_elimination(cx, llfn);
     set_instrument_function(cx, llfn);
     set_probestack(cx, llfn);
@@ -289,7 +291,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
     // The target doesn't care; the subtarget reads our attribute.
     apply_tune_cpu_attr(cx, llfn);
 
-    let function_features = codegen_fn_attrs
+    let mut function_features = codegen_fn_attrs
         .target_features
         .iter()
         .map(|f| {
@@ -301,23 +303,10 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
             InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(),
         }))
         .collect::<Vec<String>>();
-    if !function_features.is_empty() {
-        let mut global_features = llvm_util::llvm_global_features(cx.tcx.sess);
-        global_features.extend(function_features.into_iter());
-        let features = global_features.join(",");
-        let val = CString::new(features).unwrap();
-        llvm::AddFunctionAttrStringValue(
-            llfn,
-            llvm::AttributePlace::Function,
-            cstr!("target-features"),
-            &val,
-        );
-    }
 
-    // Note that currently the `wasm-import-module` doesn't do anything, but
+    if cx.tcx.sess.target.is_like_wasm {
-    // eventually LLVM 7 should read this and ferry the appropriate import
+        // If this function is an import from the environment but the wasm
-    // module to the output file.
+        // import has a specific module/name, apply them here.
-    if cx.tcx.sess.target.arch == "wasm32" {
         if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) {
             llvm::AddFunctionAttrStringValue(
                 llfn,
@@ -336,6 +325,30 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
                 &name,
             );
         }
+
+        // The `"wasm"` abi on wasm targets automatically enables the
+        // `+multivalue` feature because the purpose of the wasm abi is to match
+        // the WebAssembly specification, which has this feature. This won't be
+        // needed when LLVM enables this `multivalue` feature by default.
+        if !cx.tcx.is_closure(instance.def_id()) {
+            let abi = cx.tcx.fn_sig(instance.def_id()).abi();
+            if abi == Abi::Wasm {
+                function_features.push("+multivalue".to_string());
+            }
+        }
+    }
+
+    if !function_features.is_empty() {
+        let mut global_features = llvm_util::llvm_global_features(cx.tcx.sess);
+        global_features.extend(function_features.into_iter());
+        let features = global_features.join(",");
+        let val = CString::new(features).unwrap();
+        llvm::AddFunctionAttrStringValue(
+            llfn,
+            llvm::AttributePlace::Function,
+            cstr!("target-features"),
+            &val,
+        );
     }
 }
 

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -24,6 +24,7 @@ use tracing::{debug, info};
 use std::ffi::{CStr, CString};
 use std::fs::File;
 use std::io;
+use std::iter;
 use std::path::Path;
 use std::ptr;
 use std::slice;
@@ -916,9 +917,7 @@ impl ThinLTOKeysMap {
         modules: &[llvm::ThinLTOModule],
         names: &[CString],
     ) -> Self {
-        let keys = modules
+        let keys = iter::zip(modules, names)
-            .iter()
-            .zip(names.iter())
             .map(|(module, name)| {
                 let key = build_string(|rust_str| unsafe {
                     llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0);

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -23,11 +23,11 @@ use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{self, Lto, OutputType, Passes, SanitizerSet, SwitchWithOptPath};
+use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
-use rustc_target::spec::{CodeModel, RelocModel, SplitDebuginfo};
+use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo};
 use tracing::debug;
 
 use libc::{c_char, c_int, c_uint, c_void, size_t};
@@ -170,10 +170,7 @@ pub fn target_machine_factory(
     // On the wasm target once the `atomics` feature is enabled that means that
     // we're no longer single-threaded, or otherwise we don't want LLVM to
     // lower atomic operations to single-threaded operations.
-    if singlethread
+    if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) {
-        && sess.target.llvm_target.contains("wasm32")
-        && sess.target_features.contains(&sym::atomics)
-    {
         singlethread = false;
     }
 
@@ -548,6 +545,15 @@ pub(crate) unsafe fn optimize(
                     llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap());
                     continue;
                 }
+                if pass_name == "insert-gcov-profiling" || pass_name == "instrprof" {
+                    // Instrumentation must be inserted before optimization,
+                    // otherwise LLVM may optimize some functions away which
+                    // breaks llvm-cov.
+                    //
+                    // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp.
+                    llvm::LLVMRustAddPass(mpm, find_pass(pass_name).unwrap());
+                    continue;
+                }
 
                 if let Some(pass) = find_pass(pass_name) {
                     extra_passes.push(pass);
@@ -1041,7 +1047,7 @@ pub unsafe fn with_llvm_pmb(
     // thresholds copied from clang.
     match (opt_level, opt_size, inline_threshold) {
         (.., Some(t)) => {
-            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t as u32);
+            llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t);
         }
         (llvm::CodeGenOptLevel::Aggressive, ..) => {
             llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275);

compiler/rustc_codegen_llvm/src/base.rs

@@ -32,8 +32,9 @@ use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::middle::exported_symbols;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{DebugInfo, SanitizerSet};
+use rustc_session::config::DebugInfo;
 use rustc_span::symbol::Symbol;
+use rustc_target::spec::SanitizerSet;
 
 use std::ffi::CString;
 use std::time::Instant;
@@ -143,7 +144,7 @@ pub fn compile_codegen_unit(
 
             // Finalize code coverage by injecting the coverage map. Note, the coverage map will
             // also be added to the `llvm.used` variable, created next.
-            if cx.sess().opts.debugging_opts.instrument_coverage {
+            if cx.sess().instrument_coverage() {
                 cx.coverageinfo_finalize();
             }
 

compiler/rustc_codegen_llvm/src/builder.rs

@@ -2,6 +2,7 @@ use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm::{self, BasicBlock, False};
 use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope};
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -16,11 +17,12 @@ use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::{sym, Span};
+use rustc_span::Span;
 use rustc_target::abi::{self, Align, Size};
 use rustc_target::spec::{HasTargetSpec, Target};
 use std::borrow::Cow;
 use std::ffi::CStr;
+use std::iter;
 use std::ops::{Deref, Range};
 use std::ptr;
 use tracing::debug;
@@ -260,7 +262,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -268,7 +270,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -276,7 +278,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -284,7 +286,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -292,7 +294,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -668,81 +670,47 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
-        // WebAssembly has saturating floating point to integer casts if the
+        if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() {
-        // `nontrapping-fptoint` target feature is activated. We'll use those if
-        // they are available.
-        if self.sess().target.arch == "wasm32"
-            && self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
-        {
             let src_ty = self.cx.val_ty(val);
             let float_width = self.cx.float_width(src_ty);
             let int_width = self.cx.int_width(dest_ty);
-            let name = match (int_width, float_width) {
+            let name = format!("llvm.fptoui.sat.i{}.f{}", int_width, float_width);
-                (32, 32) => Some("llvm.wasm.trunc.saturate.unsigned.i32.f32"),
+            let intrinsic = self.get_intrinsic(&name);
-                (32, 64) => Some("llvm.wasm.trunc.saturate.unsigned.i32.f64"),
+            return Some(self.call(intrinsic, &[val], None));
-                (64, 32) => Some("llvm.wasm.trunc.saturate.unsigned.i64.f32"),
-                (64, 64) => Some("llvm.wasm.trunc.saturate.unsigned.i64.f64"),
-                _ => None,
-            };
-            if let Some(name) = name {
-                let intrinsic = self.get_intrinsic(name);
-                return Some(self.call(intrinsic, &[val], None));
-            }
         }
+
         None
     }
 
     fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
-        // WebAssembly has saturating floating point to integer casts if the
+        if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() {
-        // `nontrapping-fptoint` target feature is activated. We'll use those if
-        // they are available.
-        if self.sess().target.arch == "wasm32"
-            && self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
-        {
             let src_ty = self.cx.val_ty(val);
             let float_width = self.cx.float_width(src_ty);
             let int_width = self.cx.int_width(dest_ty);
-            let name = match (int_width, float_width) {
+            let name = format!("llvm.fptosi.sat.i{}.f{}", int_width, float_width);
-                (32, 32) => Some("llvm.wasm.trunc.saturate.signed.i32.f32"),
+            let intrinsic = self.get_intrinsic(&name);
-                (32, 64) => Some("llvm.wasm.trunc.saturate.signed.i32.f64"),
+            return Some(self.call(intrinsic, &[val], None));
-                (64, 32) => Some("llvm.wasm.trunc.saturate.signed.i64.f32"),
-                (64, 64) => Some("llvm.wasm.trunc.saturate.signed.i64.f64"),
-                _ => None,
-            };
-            if let Some(name) = name {
-                let intrinsic = self.get_intrinsic(name);
-                return Some(self.call(intrinsic, &[val], None));
-            }
         }
+
         None
     }
 
-    fn fptosui_may_trap(&self, val: &'ll Value, dest_ty: &'ll Type) -> bool {
-        // Most of the time we'll be generating the `fptosi` or `fptoui`
-        // instruction for floating-point-to-integer conversions. These
-        // instructions by definition in LLVM do not trap. For the WebAssembly
-        // target, however, we'll lower in some cases to intrinsic calls instead
-        // which may trap. If we detect that this is a situation where we'll be
-        // using the intrinsics then we report that the call map trap, which
-        // callers might need to handle.
-        if !self.wasm_and_missing_nontrapping_fptoint() {
-            return false;
-        }
-        let src_ty = self.cx.val_ty(val);
-        let float_width = self.cx.float_width(src_ty);
-        let int_width = self.cx.int_width(dest_ty);
-        matches!((int_width, float_width), (32, 32) | (32, 64) | (64, 32) | (64, 64))
-    }
-
     fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
-        // When we can, use the native wasm intrinsics which have tighter
+        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
-        // codegen. Note that this has a semantic difference in that the
+        // poor codegen. The reason for this is that the corresponding wasm
-        // intrinsic can trap whereas `fptoui` never traps. That difference,
+        // instructions, `i32.trunc_f32_s` for example, will trap when the float
-        // however, is handled by `fptosui_may_trap` above.
+        // is out-of-bounds, infinity, or nan. This means that LLVM
+        // automatically inserts control flow around `fptoui` and `fptosi`
+        // because the LLVM instruction `fptoui` is defined as producing a
+        // poison value, not having UB on out-of-bounds values.
         //
-        // Note that we skip the wasm intrinsics for vector types where `fptoui`
+        // This method, however, is only used with non-saturating casts that
-        // must be used instead.
+        // have UB on out-of-bounds values. This means that it's ok if we use
-        if self.wasm_and_missing_nontrapping_fptoint() {
+        // the raw wasm instruction since out-of-bounds values can do whatever
+        // we like. To ensure that LLVM picks the right instruction we choose
+        // the raw wasm intrinsic functions which avoid LLVM inserting all the
+        // other control flow automatically.
+        if self.sess().target.arch == "wasm32" {
             let src_ty = self.cx.val_ty(val);
             if self.cx.type_kind(src_ty) != TypeKind::Vector {
                 let float_width = self.cx.float_width(src_ty);
@@ -764,7 +732,8 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
-        if self.wasm_and_missing_nontrapping_fptoint() {
+        // see `fptoui` above for why wasm is different here
+        if self.sess().target.arch == "wasm32" {
             let src_ty = self.cx.val_ty(val);
             if self.cx.type_kind(src_ty) != TypeKind::Vector {
                 let float_width = self.cx.float_width(src_ty);
@@ -1241,14 +1210,14 @@ impl Builder<'a, 'll, 'tcx> {
     pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
     pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
         unsafe {
             let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -1281,7 +1250,7 @@ impl Builder<'a, 'll, 'tcx> {
         unsafe {
             let instr =
                 llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -1289,7 +1258,7 @@ impl Builder<'a, 'll, 'tcx> {
         unsafe {
             let instr =
                 llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
-            llvm::LLVMRustSetHasUnsafeAlgebra(instr);
+            llvm::LLVMRustSetFastMath(instr);
             instr
         }
     }
@@ -1352,18 +1321,14 @@ impl Builder<'a, 'll, 'tcx> {
 
         let param_tys = self.cx.func_params_types(fn_ty);
 
-        let all_args_match = param_tys
+        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.val_ty(v)))
-            .iter()
-            .zip(args.iter().map(|&v| self.val_ty(v)))
             .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
 
         if all_args_match {
             return Cow::Borrowed(args);
         }
 
-        let casted_args: Vec<_> = param_tys
+        let casted_args: Vec<_> = iter::zip(param_tys, args)
-            .into_iter()
-            .zip(args.iter())
             .enumerate()
             .map(|(i, (expected_ty, &actual_val))| {
                 let actual_ty = self.val_ty(actual_val);
@@ -1423,8 +1388,11 @@ impl Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn wasm_and_missing_nontrapping_fptoint(&self) -> bool {
+    fn fptoint_sat_broken_in_llvm(&self) -> bool {
-        self.sess().target.arch == "wasm32"
+        match self.tcx.sess.target.arch.as_str() {
-            && !self.sess().target_features.contains(&sym::nontrapping_dash_fptoint)
+            // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083
+            "riscv64" => llvm_util::get_version() < (13, 0, 0),
+            _ => false,
+        }
     }
 }

compiler/rustc_codegen_llvm/src/callee.rs

@@ -14,6 +14,7 @@ use tracing::debug;
 
 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
 use rustc_middle::ty::{self, Instance, TypeFoldable};
+use rustc_target::spec::RelocModel;
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
@@ -170,17 +171,19 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value
                     }
                 }
             }
-        }
 
-        // MinGW: For backward compatibility we rely on the linker to decide whether it
+            // MinGW: For backward compatibility we rely on the linker to decide whether it
-        // should use dllimport for functions.
+            // should use dllimport for functions.
-        if cx.use_dll_storage_attrs
+            if cx.use_dll_storage_attrs
-            && tcx.is_dllimport_foreign_item(instance_def_id)
+                && tcx.is_dllimport_foreign_item(instance_def_id)
-            && tcx.sess.target.env != "gnu"
+                && tcx.sess.target.env != "gnu"
-        {
+            {
-            unsafe {
                 llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport);
             }
+
+            if cx.tcx.sess.relocation_model() == RelocModel::Static {
+                llvm::LLVMRustSetDSOLocal(llfn, true);
+            }
         }
 
         llfn

compiler/rustc_codegen_llvm/src/context.rs

@@ -79,7 +79,7 @@ pub struct CodegenCx<'ll, 'tcx> {
     pub pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>,
     pub isize_ty: &'ll Type,
 
-    pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'tcx>>,
+    pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>,
     pub dbg_cx: Option<debuginfo::CrateDebugContext<'ll, 'tcx>>,
 
     eh_personality: Cell<Option<&'ll Value>>,
@@ -101,10 +101,6 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode {
     }
 }
 
-fn strip_x86_address_spaces(data_layout: String) -> String {
-    data_layout.replace("-p270:32:32-p271:32:32-p272:64:64-", "-")
-}
-
 fn strip_powerpc64_vectors(data_layout: String) -> String {
     data_layout.replace("-v256:256:256-v512:512:512", "")
 }
@@ -119,11 +115,6 @@ pub unsafe fn create_module(
     let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx);
 
     let mut target_data_layout = sess.target.data_layout.clone();
-    if llvm_util::get_version() < (10, 0, 0)
-        && (sess.target.arch == "x86" || sess.target.arch == "x86_64")
-    {
-        target_data_layout = strip_x86_address_spaces(target_data_layout);
-    }
     if llvm_util::get_version() < (12, 0, 0) && sess.target.arch == "powerpc64" {
         target_data_layout = strip_powerpc64_vectors(target_data_layout);
     }
@@ -280,7 +271,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
 
         let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod());
 
-        let coverage_cx = if tcx.sess.opts.debugging_opts.instrument_coverage {
+        let coverage_cx = if tcx.sess.instrument_coverage() {
             let covctx = coverageinfo::CrateCoverageContext::new();
             Some(covctx)
         } else {
@@ -331,7 +322,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 
     #[inline]
-    pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'tcx>> {
+    pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'ll, 'tcx>> {
         self.coverage_cx.as_ref()
     }
 }
@@ -512,14 +503,6 @@ impl CodegenCx<'b, 'tcx> {
         let t_f32 = self.type_f32();
         let t_f64 = self.type_f64();
 
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.wasm.trunc.saturate.unsigned.i64.f64", fn(t_f64) -> t_i64);
-        ifn!("llvm.wasm.trunc.saturate.signed.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.signed.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.wasm.trunc.saturate.signed.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.wasm.trunc.saturate.signed.i64.f64", fn(t_f64) -> t_i64);
         ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
         ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
         ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64);
@@ -529,6 +512,28 @@ impl CodegenCx<'b, 'tcx> {
         ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64);
         ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64);
 
+        ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8);
+        ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16);
+        ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32);
+        ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64);
+        ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128);
+        ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8);
+        ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16);
+        ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32);
+        ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64);
+        ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128);
+
+        ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8);
+        ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16);
+        ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32);
+        ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64);
+        ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128);
+        ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8);
+        ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16);
+        ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32);
+        ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64);
+        ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128);
+
         ifn!("llvm.trap", fn() -> void);
         ifn!("llvm.debugtrap", fn() -> void);
         ifn!("llvm.frameaddress", fn(t_i32) -> i8p);
@@ -712,7 +717,7 @@ impl CodegenCx<'b, 'tcx> {
         ifn!("llvm.va_end", fn(i8p) -> void);
         ifn!("llvm.va_copy", fn(i8p, i8p) -> void);
 
-        if self.sess().opts.debugging_opts.instrument_coverage {
+        if self.sess().instrument_coverage() {
             ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void);
         }
 

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -3,13 +3,13 @@ use crate::coverageinfo;
 use crate::llvm;
 
 use llvm::coverageinfo::CounterMappingRegion;
-use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression, FunctionCoverage};
+use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression};
-use rustc_codegen_ssa::traits::ConstMethods;
+use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE};
 use rustc_llvm::RustString;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::coverage::CodeRegion;
-use rustc_middle::ty::{Instance, TyCtxt};
 use rustc_span::Symbol;
 
 use std::ffi::CString;
@@ -20,16 +20,17 @@ use tracing::debug;
 ///
 /// This Coverage Map complies with Coverage Mapping Format version 4 (zero-based encoded as 3),
 /// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format)
-/// and published in Rust's current (November 2020) fork of LLVM. This version is supported by the
+/// and published in Rust's November 2020 fork of LLVM. This version is supported by the LLVM
-/// LLVM coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM.
+/// coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM.
 ///
 /// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with
-/// version 3. Clang's implementation of Coverage Map generation was referenced when implementing
+/// the same version. Clang's implementation of Coverage Map generation was referenced when
-/// this Rust version, and though the format documentation is very explicit and detailed, some
+/// implementing this Rust version, and though the format documentation is very explicit and
-/// undocumented details in Clang's implementation (that may or may not be important) were also
+/// detailed, some undocumented details in Clang's implementation (that may or may not be important)
-/// replicated for Rust's Coverage Map.
+/// were also replicated for Rust's Coverage Map.
 pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     let tcx = cx.tcx;
+
     // Ensure LLVM supports Coverage Map Version 4 (encoded as a zero-based value: 3).
     // If not, the LLVM Version must be less than 11.
     let version = coverageinfo::mapping_version();
@@ -39,17 +40,24 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
 
     debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name());
 
-    let mut function_coverage_map = match cx.coverage_context() {
+    // In order to show that unused functions have coverage counts of zero (0), LLVM requires the
+    // functions exist. Generate synthetic functions with a (required) single counter, and add the
+    // MIR `Coverage` code regions to the `function_coverage_map`, before calling
+    // `ctx.take_function_coverage_map()`.
+    if !tcx.sess.instrument_coverage_except_unused_functions() {
+        add_unused_functions(cx);
+    }
+
+    let function_coverage_map = match cx.coverage_context() {
         Some(ctx) => ctx.take_function_coverage_map(),
         None => return,
     };
+
     if function_coverage_map.is_empty() {
         // This module has no functions with coverage instrumentation
         return;
     }
 
-    add_unreachable_coverage(tcx, &mut function_coverage_map);
-
     let mut mapgen = CoverageMapGenerator::new();
 
     // Encode coverage mappings and generate function records
@@ -57,7 +65,8 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     for (instance, function_coverage) in function_coverage_map {
         debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance);
         let mangled_function_name = tcx.symbol_name(instance).to_string();
-        let function_source_hash = function_coverage.source_hash();
+        let source_hash = function_coverage.source_hash();
+        let is_used = function_coverage.is_used();
         let (expressions, counter_regions) =
             function_coverage.get_expressions_and_counter_regions();
 
@@ -69,7 +78,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
             "Every `FunctionCoverage` should have at least one counter"
         );
 
-        function_data.push((mangled_function_name, function_source_hash, coverage_mapping_buffer));
+        function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer));
     }
 
     // Encode all filenames referenced by counters/expressions in this module
@@ -84,13 +93,14 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     // Generate the LLVM IR representation of the coverage map and store it in a well-known global
     let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val);
 
-    for (mangled_function_name, function_source_hash, coverage_mapping_buffer) in function_data {
+    for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data {
         save_function_record(
             cx,
             mangled_function_name,
-            function_source_hash,
+            source_hash,
             filenames_ref,
             coverage_mapping_buffer,
+            is_used,
         );
     }
 
@@ -201,9 +211,10 @@ impl CoverageMapGenerator {
 fn save_function_record(
     cx: &CodegenCx<'ll, 'tcx>,
     mangled_function_name: String,
-    function_source_hash: u64,
+    source_hash: u64,
     filenames_ref: u64,
     coverage_mapping_buffer: Vec<u8>,
+    is_used: bool,
 ) {
     // Concatenate the encoded coverage mappings
     let coverage_mapping_size = coverage_mapping_buffer.len();
@@ -212,128 +223,124 @@ fn save_function_record(
     let func_name_hash = coverageinfo::hash_str(&mangled_function_name);
     let func_name_hash_val = cx.const_u64(func_name_hash);
     let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32);
-    let func_hash_val = cx.const_u64(function_source_hash);
+    let source_hash_val = cx.const_u64(source_hash);
     let filenames_ref_val = cx.const_u64(filenames_ref);
     let func_record_val = cx.const_struct(
         &[
             func_name_hash_val,
             coverage_mapping_size_val,
-            func_hash_val,
+            source_hash_val,
             filenames_ref_val,
             coverage_mapping_val,
         ],
         /*packed=*/ true,
     );
 
-    // At the present time, the coverage map for Rust assumes every instrumented function `is_used`.
-    // Note that Clang marks functions as "unused" in `CodeGenPGO::emitEmptyCounterMapping`. (See:
-    // https://github.com/rust-lang/llvm-project/blob/de02a75e398415bad4df27b4547c25b896c8bf3b/clang%2Flib%2FCodeGen%2FCodeGenPGO.cpp#L877-L878
-    // for example.)
-    //
-    // It's not yet clear if or how this may be applied to Rust in the future, but the `is_used`
-    // argument is available and handled similarly.
-    let is_used = true;
     coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used);
 }
 
 /// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for
 /// the functions that went through codegen; such as public functions and "used" functions
 /// (functions referenced by other "used" or public items). Any other functions considered unused,
-/// or "Unreachable" were still parsed and processed through the MIR stage.
+/// or "Unreachable", were still parsed and processed through the MIR stage, but were not
+/// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but
+/// that flag is known to cause other errors, when combined with `-Z instrument-coverage`; and
+/// `-Clink-dead-code` will not generate code for unused generic functions.)
 ///
-/// We can find the unreachable functions by the set difference of all MIR `DefId`s (`tcx` query
+/// We can find the unused functions (including generic functions) by the set difference of all MIR
-/// `mir_keys`) minus the codegenned `DefId`s (`tcx` query `collect_and_partition_mono_items`).
+/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query
+/// `collect_and_partition_mono_items`).
 ///
 /// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and
 /// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`)
-/// allocated to only one of those CGUs. We must NOT inject any "Unreachable" functions's
+/// allocated to only one of those CGUs. We must NOT inject any unused functions's `CodeRegion`s
-/// `CodeRegion`s more than once, so we have to pick which CGU's `function_coverage_map` to add
+/// more than once, so we have to pick a CGUs `function_coverage_map` into which the unused
-/// each "Unreachable" function to.
+/// function will be inserted.
-///
+fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
-/// Some constraints:
+    let tcx = cx.tcx;
-///
+
-/// 1. The file name of an "Unreachable" function must match the file name of the existing
-///    codegenned (covered) function to which the unreachable code regions will be added.
-/// 2. The function to which the unreachable code regions will be added must not be a generic
-///    function (must not have type parameters) because the coverage tools will get confused
-///    if the codegenned function has more than one instantiation and additional `CodeRegion`s
-///    attached to only one of those instantiations.
-fn add_unreachable_coverage<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    function_coverage_map: &mut FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>,
-) {
     // FIXME(#79622): Can this solution be simplified and/or improved? Are there other sources
     // of compiler state data that might help (or better sources that could be exposed, but
     // aren't yet)?
 
-    // Note: If the crate *only* defines generic functions, there are no codegenerated non-generic
+    let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics();
-    // functions to add any unreachable code to. In this case, the unreachable code regions will
-    // have no coverage, instead of having coverage with zero executions.
-    //
-    // This is probably still an improvement over Clang, which does not generate any coverage
-    // for uninstantiated template functions.
 
-    let has_non_generic_def_ids =
+    let all_def_ids: DefIdSet = tcx
-        function_coverage_map.keys().any(|instance| instance.def.attrs(tcx).len() == 0);
+        .mir_keys(LOCAL_CRATE)
-
+        .iter()
-    if !has_non_generic_def_ids {
+        .filter_map(|local_def_id| {
-        // There are no non-generic functions to add unreachable `CodeRegion`s to
+            let def_id = local_def_id.to_def_id();
-        return;
+            if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) {
-    }
+                return None;
-
+            }
-    let all_def_ids: DefIdSet =
+            Some(local_def_id.to_def_id())
-        tcx.mir_keys(LOCAL_CRATE).iter().map(|local_def_id| local_def_id.to_def_id()).collect();
+        })
+        .collect();
 
     let codegenned_def_ids = tcx.codegened_and_inlined_items(LOCAL_CRATE);
 
-    let mut unreachable_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default();
+    let mut unused_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default();
     for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) {
-        // Make sure the non-codegenned (unreachable) function has a file_name
+        let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id);
+        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
+            continue;
+        }
+        // Make sure the non-codegenned (unused) function has a file_name
         if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) {
-            let def_ids = unreachable_def_ids_by_file
+            let def_ids =
-                .entry(*non_codegenned_file_name)
+                unused_def_ids_by_file.entry(*non_codegenned_file_name).or_insert_with(Vec::new);
-                .or_insert_with(Vec::new);
             def_ids.push(non_codegenned_def_id);
         }
     }
 
-    if unreachable_def_ids_by_file.is_empty() {
+    if unused_def_ids_by_file.is_empty() {
-        // There are no unreachable functions with file names to add (in any CGU)
+        // There are no unused functions with file names to add (in any CGU)
         return;
     }
 
-    // Since there may be multiple `CodegenUnit`s, some codegenned_def_ids may be codegenned in a
+    // Each `CodegenUnit` (CGU) has its own function_coverage_map, and generates a specific binary
-    // different CGU, and will be added to the function_coverage_map for each CGU. Determine which
+    // with its own coverage map.
-    // function_coverage_map has the responsibility for publishing unreachable coverage
-    // based on file name:
     //
-    // For each covered file name, sort ONLY the non-generic codegenned_def_ids, and if
+    // Each covered function `Instance` can be included in only one coverage map, produced from a
-    // covered_def_ids.contains(the first def_id) for a given file_name, add the unreachable code
+    // specific function_coverage_map, from a specific CGU.
-    // region in this function_coverage_map. Otherwise, ignore it and assume another CGU's
+    //
-    // function_coverage_map will be adding it (because it will be first for one, and only one,
+    // Since unused functions did not generate code, they are not associated with any CGU yet.
-    // of them).
+    //
+    // To avoid injecting the unused functions in multiple coverage maps (for multiple CGUs)
+    // determine which function_coverage_map has the responsibility for publishing unreachable
+    // coverage, based on file name: For each unused function, find the CGU that generates the
+    // first function (based on sorted `DefId`) from the same file.
+    //
+    // Add a new `FunctionCoverage` to the `function_coverage_map`, with unreachable code regions
+    // for each region in it's MIR.
+
+    // Convert the `HashSet` of `codegenned_def_ids` to a sortable vector, and sort them.
     let mut sorted_codegenned_def_ids: Vec<DefId> =
         codegenned_def_ids.iter().map(|def_id| *def_id).collect();
     sorted_codegenned_def_ids.sort_unstable();
 
     let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default();
     for &def_id in sorted_codegenned_def_ids.iter() {
-        // Only consider non-generic functions, to potentially add unreachable code regions
+        if let Some(covered_file_name) = tcx.covered_file_name(def_id) {
-        if tcx.generics_of(def_id).count() == 0 {
+            // Only add files known to have unused functions
-            if let Some(covered_file_name) = tcx.covered_file_name(def_id) {
+            if unused_def_ids_by_file.contains_key(covered_file_name) {
-                // Only add files known to have unreachable functions
+                first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id);
-                if unreachable_def_ids_by_file.contains_key(covered_file_name) {
-                    first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id);
-                }
             }
         }
     }
 
     // Get the set of def_ids with coverage regions, known by *this* CoverageContext.
-    let cgu_covered_def_ids: DefIdSet =
+    let cgu_covered_def_ids: DefIdSet = match cx.coverage_context() {
-        function_coverage_map.keys().map(|instance| instance.def.def_id()).collect();
+        Some(ctx) => ctx
+            .function_coverage_map
+            .borrow()
+            .keys()
+            .map(|&instance| instance.def.def_id())
+            .collect(),
+        None => return,
+    };
 
-    let mut cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file
+    let cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file
         .iter()
         .filter_map(
             |(&file_name, def_id)| {
@@ -342,49 +349,13 @@ fn add_unreachable_coverage<'tcx>(
         )
         .collect();
 
-    // Find the first covered, non-generic function (instance) for each cgu_covered_file. Take the
+    // For each file for which this CGU is responsible for adding unused function coverage,
-    // unreachable code regions for that file, and add them to the function.
+    // get the `def_id`s for each unused function (if any), define a synthetic function with a
-    //
+    // single LLVM coverage counter, and add the function's coverage `CodeRegion`s. to the
-    // There are three `for` loops here, but (a) the lists have already been reduced to the minimum
+    // function_coverage_map.
-    // required values, the lists are further reduced (by `remove()` calls) when elements are no
+    for covered_file_name in cgu_covered_files {
-    // longer needed, and there are several opportunities to branch out of loops early.
+        for def_id in unused_def_ids_by_file.remove(&covered_file_name).into_iter().flatten() {
-    for (instance, function_coverage) in function_coverage_map.iter_mut() {
+            cx.define_unused_fn(def_id);
-        if instance.def.attrs(tcx).len() > 0 {
-            continue;
-        }
-        // The covered function is not generic...
-        let covered_def_id = instance.def.def_id();
-        if let Some(covered_file_name) = tcx.covered_file_name(covered_def_id) {
-            if !cgu_covered_files.remove(&covered_file_name) {
-                continue;
-            }
-            // The covered function's file is one of the files with unreachable code regions, so
-            // all of the unreachable code regions for this file will be added to this function.
-            for def_id in
-                unreachable_def_ids_by_file.remove(&covered_file_name).into_iter().flatten()
-            {
-                // Note, this loop adds an unreachable code regions for each MIR-derived region.
-                // Alternatively, we could add a single code region for the maximum span of all
-                // code regions here.
-                //
-                // Observed downsides of this approach are:
-                //
-                // 1. The coverage results will appear inconsistent compared with the same (or
-                //    similar) code in a function that is reached.
-                // 2. If the function is unreachable from one crate but reachable when compiling
-                //    another referencing crate (such as a cross-crate reference to a
-                //    generic function or inlined function), actual coverage regions overlaid
-                //    on a single larger code span of `Zero` coverage can appear confusing or
-                //    wrong. Chaning the unreachable coverage from a `code_region` to a
-                //    `gap_region` can help, but still can look odd with `0` line counts for
-                //    lines between executed (> 0) lines (such as for blank lines or comments).
-                for &region in tcx.covered_code_regions(def_id) {
-                    function_coverage.add_unreachable_region(region.clone());
-                }
-            }
-            if cgu_covered_files.is_empty() {
-                break;
-            }
         }
     }
 }

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -1,5 +1,6 @@
 use crate::llvm;
 
+use crate::abi::{Abi, FnAbi};
 use crate::builder::Builder;
 use crate::common::CodegenCx;
 
@@ -7,33 +8,47 @@ use libc::c_uint;
 use llvm::coverageinfo::CounterMappingRegion;
 use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage};
 use rustc_codegen_ssa::traits::{
-    BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, MiscMethods, StaticMethods,
+    BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods,
+    MiscMethods, StaticMethods,
 };
 use rustc_data_structures::fx::FxHashMap;
+use rustc_hir as hir;
+use rustc_hir::def_id::DefId;
 use rustc_llvm::RustString;
+use rustc_middle::bug;
 use rustc_middle::mir::coverage::{
     CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op,
 };
+use rustc_middle::ty;
+use rustc_middle::ty::layout::FnAbiExt;
+use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::Instance;
 
 use std::cell::RefCell;
 use std::ffi::CString;
 
+use std::iter;
 use tracing::debug;
 
 pub mod mapgen;
 
+const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START;
+
 const VAR_ALIGN_BYTES: usize = 8;
 
 /// A context object for maintaining all state needed by the coverageinfo module.
-pub struct CrateCoverageContext<'tcx> {
+pub struct CrateCoverageContext<'ll, 'tcx> {
     // Coverage data for each instrumented function identified by DefId.
     pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>,
+    pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
 }
 
-impl<'tcx> CrateCoverageContext<'tcx> {
+impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
     pub fn new() -> Self {
-        Self { function_coverage_map: Default::default() }
+        Self {
+            function_coverage_map: Default::default(),
+            pgo_func_name_var_map: Default::default(),
+        }
     }
 
     pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> {
@@ -41,23 +56,47 @@ impl<'tcx> CrateCoverageContext<'tcx> {
     }
 }
 
-impl CoverageInfoMethods for CodegenCx<'ll, 'tcx> {
+impl CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn coverageinfo_finalize(&self) {
         mapgen::finalize(self)
     }
+
+    fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
+        if let Some(coverage_context) = self.coverage_context() {
+            debug!("getting pgo_func_name_var for instance={:?}", instance);
+            let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut();
+            pgo_func_name_var_map
+                .entry(instance)
+                .or_insert_with(|| create_pgo_func_name_var(self, instance))
+        } else {
+            bug!("Could not get the `coverage_context`");
+        }
+    }
+
+    /// Functions with MIR-based coverage are normally codegenned _only_ if
+    /// called. LLVM coverage tools typically expect every function to be
+    /// defined (even if unused), with at least one call to LLVM intrinsic
+    /// `instrprof.increment`.
+    ///
+    /// Codegen a small function that will never be called, with one counter
+    /// that will never be incremented.
+    ///
+    /// For used/called functions, the coverageinfo was already added to the
+    /// `function_coverage_map` (keyed by function `Instance`) during codegen.
+    /// But in this case, since the unused function was _not_ previously
+    /// codegenned, collect the coverage `CodeRegion`s from the MIR and add
+    /// them. The first `CodeRegion` is used to add a single counter, with the
+    /// same counter ID used in the injected `instrprof.increment` intrinsic
+    /// call. Since the function is never called, all other `CodeRegion`s can be
+    /// added as `unreachable_region`s.
+    fn define_unused_fn(&self, def_id: DefId) {
+        let instance = declare_unused_fn(self, &def_id);
+        codegen_unused_fn_and_counter(self, instance);
+        add_unused_function_coverage(self, instance, def_id);
+    }
 }
 
 impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
-    /// Calls llvm::createPGOFuncNameVar() with the given function instance's mangled function name.
-    /// The LLVM API returns an llvm::GlobalVariable containing the function name, with the specific
-    /// variable name and linkage required by LLVM InstrProf source-based coverage instrumentation.
-    fn create_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value {
-        let llfn = self.cx.get_fn(instance);
-        let mangled_fn_name = CString::new(self.tcx.symbol_name(instance).name)
-            .expect("error converting function name to C string");
-        unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) }
-    }
-
     fn set_function_source_hash(
         &mut self,
         instance: Instance<'tcx>,
@@ -145,6 +184,100 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     }
 }
 
+fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx> {
+    let tcx = cx.tcx;
+
+    let instance = Instance::new(
+        *def_id,
+        InternalSubsts::for_item(tcx, *def_id, |param, _| {
+            if let ty::GenericParamDefKind::Lifetime = param.kind {
+                tcx.lifetimes.re_erased.into()
+            } else {
+                tcx.mk_param_from_def(param)
+            }
+        }),
+    );
+
+    let llfn = cx.declare_fn(
+        &tcx.symbol_name(instance).name,
+        &FnAbi::of_fn_ptr(
+            cx,
+            ty::Binder::dummy(tcx.mk_fn_sig(
+                iter::once(tcx.mk_unit()),
+                tcx.mk_unit(),
+                false,
+                hir::Unsafety::Unsafe,
+                Abi::Rust,
+            )),
+            &[],
+        ),
+    );
+
+    llvm::set_linkage(llfn, llvm::Linkage::WeakAnyLinkage);
+    llvm::set_visibility(llfn, llvm::Visibility::Hidden);
+
+    assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none());
+
+    instance
+}
+
+fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) {
+    let llfn = cx.get_fn(instance);
+    let mut bx = Builder::new_block(cx, llfn, "unused_function");
+    let fn_name = bx.get_pgo_func_name_var(instance);
+    let hash = bx.const_u64(0);
+    let num_counters = bx.const_u32(1);
+    let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID));
+    debug!(
+        "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?},
+            index={:?}) for unused function: {:?}",
+        fn_name, hash, num_counters, index, instance
+    );
+    bx.instrprof_increment(fn_name, hash, num_counters, index);
+    bx.ret_void();
+}
+
+fn add_unused_function_coverage(
+    cx: &CodegenCx<'ll, 'tcx>,
+    instance: Instance<'tcx>,
+    def_id: DefId,
+) {
+    let tcx = cx.tcx;
+
+    let mut function_coverage = FunctionCoverage::unused(tcx, instance);
+    for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() {
+        if index == 0 {
+            // Insert at least one real counter so the LLVM CoverageMappingReader will find expected
+            // definitions.
+            function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone());
+        } else {
+            function_coverage.add_unreachable_region(code_region.clone());
+        }
+    }
+
+    if let Some(coverage_context) = cx.coverage_context() {
+        coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage);
+    } else {
+        bug!("Could not get the `coverage_context`");
+    }
+}
+
+/// Calls llvm::createPGOFuncNameVar() with the given function instance's
+/// mangled function name. The LLVM API returns an llvm::GlobalVariable
+/// containing the function name, with the specific variable name and linkage
+/// required by LLVM InstrProf source-based coverage instrumentation. Use
+/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per
+/// `Instance`.
+fn create_pgo_func_name_var(
+    cx: &CodegenCx<'ll, 'tcx>,
+    instance: Instance<'tcx>,
+) -> &'ll llvm::Value {
+    let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name)
+        .expect("error converting function name to C string");
+    let llfn = cx.get_fn(instance);
+    unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) }
+}
+
 pub(crate) fn write_filenames_section_to_buffer<'a>(
     filenames: impl IntoIterator<Item = &'a CString>,
     buffer: &RustString,
@@ -177,6 +310,7 @@ pub(crate) fn write_mapping_to_buffer(
         );
     }
 }
+
 pub(crate) fn hash_str(strval: &str) -> u64 {
     let strval = CString::new(strval).expect("null error converting hashable str to C string");
     unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) }

compiler/rustc_codegen_llvm/src/debuginfo/doc.md

@@ -0,0 +1,180 @@
+# Debug Info Module
+
+This module serves the purpose of generating debug symbols. We use LLVM's
+[source level debugging](https://llvm.org/docs/SourceLevelDebugging.html)
+features for generating the debug information. The general principle is
+this:
+
+Given the right metadata in the LLVM IR, the LLVM code generator is able to
+create DWARF debug symbols for the given code. The
+[metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured
+much like DWARF *debugging information entries* (DIE), representing type
+information such as datatype layout, function signatures, block layout,
+variable location and scope information, etc. It is the purpose of this
+module to generate correct metadata and insert it into the LLVM IR.
+
+As the exact format of metadata trees may change between different LLVM
+versions, we now use LLVM
+[DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html)
+to create metadata where possible. This will hopefully ease the adaption of
+this module to future LLVM versions.
+
+The public API of the module is a set of functions that will insert the
+correct metadata into the LLVM IR when called with the right parameters.
+The module is thus driven from an outside client with functions like
+`debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`.
+
+Internally the module will try to reuse already created metadata by
+utilizing a cache. The way to get a shared metadata node when needed is
+thus to just call the corresponding function in this module:
+
+    let file_metadata = file_metadata(cx, file);
+
+The function will take care of probing the cache for an existing node for
+that exact file path.
+
+All private state used by the module is stored within either the
+CrateDebugContext struct (owned by the CodegenCx) or the
+FunctionDebugContext (owned by the FunctionCx).
+
+This file consists of three conceptual sections:
+1. The public interface of the module
+2. Module-internal metadata creation functions
+3. Minor utility functions
+
+
+## Recursive Types
+
+Some kinds of types, such as structs and enums can be recursive. That means
+that the type definition of some type X refers to some other type which in
+turn (transitively) refers to X. This introduces cycles into the type
+referral graph. A naive algorithm doing an on-demand, depth-first traversal
+of this graph when describing types, can get trapped in an endless loop
+when it reaches such a cycle.
+
+For example, the following simple type for a singly-linked list...
+
+```
+struct List {
+    value: i32,
+    tail: Option<Box<List>>,
+}
+```
+
+will generate the following callstack with a naive DFS algorithm:
+
+```
+describe(t = List)
+  describe(t = i32)
+  describe(t = Option<Box<List>>)
+    describe(t = Box<List>)
+      describe(t = List) // at the beginning again...
+      ...
+```
+
+To break cycles like these, we use "forward declarations". That is, when
+the algorithm encounters a possibly recursive type (any struct or enum), it
+immediately creates a type description node and inserts it into the cache
+*before* describing the members of the type. This type description is just
+a stub (as type members are not described and added to it yet) but it
+allows the algorithm to already refer to the type. After the stub is
+inserted into the cache, the algorithm continues as before. If it now
+encounters a recursive reference, it will hit the cache and does not try to
+describe the type anew.
+
+This behavior is encapsulated in the 'RecursiveTypeDescription' enum,
+which represents a kind of continuation, storing all state needed to
+continue traversal at the type members after the type has been registered
+with the cache. (This implementation approach might be a tad over-
+engineered and may change in the future)
+
+
+## Source Locations and Line Information
+
+In addition to data type descriptions the debugging information must also
+allow to map machine code locations back to source code locations in order
+to be useful. This functionality is also handled in this module. The
+following functions allow to control source mappings:
+
++ `set_source_location()`
++ `clear_source_location()`
++ `start_emitting_source_locations()`
+
+`set_source_location()` allows to set the current source location. All IR
+instructions created after a call to this function will be linked to the
+given source location, until another location is specified with
+`set_source_location()` or the source location is cleared with
+`clear_source_location()`. In the later case, subsequent IR instruction
+will not be linked to any source location. As you can see, this is a
+stateful API (mimicking the one in LLVM), so be careful with source
+locations set by previous calls. It's probably best to not rely on any
+specific state being present at a given point in code.
+
+One topic that deserves some extra attention is *function prologues*. At
+the beginning of a function's machine code there are typically a few
+instructions for loading argument values into allocas and checking if
+there's enough stack space for the function to execute. This *prologue* is
+not visible in the source code and LLVM puts a special PROLOGUE END marker
+into the line table at the first non-prologue instruction of the function.
+In order to find out where the prologue ends, LLVM looks for the first
+instruction in the function body that is linked to a source location. So,
+when generating prologue instructions we have to make sure that we don't
+emit source location information until the 'real' function body begins. For
+this reason, source location emission is disabled by default for any new
+function being codegened and is only activated after a call to the third
+function from the list above, `start_emitting_source_locations()`. This
+function should be called right before regularly starting to codegen the
+top-level block of the given function.
+
+There is one exception to the above rule: `llvm.dbg.declare` instruction
+must be linked to the source location of the variable being declared. For
+function parameters these `llvm.dbg.declare` instructions typically occur
+in the middle of the prologue, however, they are ignored by LLVM's prologue
+detection. The `create_argument_metadata()` and related functions take care
+of linking the `llvm.dbg.declare` instructions to the correct source
+locations even while source location emission is still disabled, so there
+is no need to do anything special with source location handling here.
+
+## Unique Type Identification
+
+In order for link-time optimization to work properly, LLVM needs a unique
+type identifier that tells it across compilation units which types are the
+same as others. This type identifier is created by
+`TypeMap::get_unique_type_id_of_type()` using the following algorithm:
+
+1. Primitive types have their name as ID
+
+2. Structs, enums and traits have a multipart identifier
+
+  1. The first part is the SVH (strict version hash) of the crate they
+     were originally defined in
+
+  2. The second part is the ast::NodeId of the definition in their
+     original crate
+
+  3. The final part is a concatenation of the type IDs of their concrete
+     type arguments if they are generic types.
+
+3. Tuple-, pointer-, and function types are structurally identified, which
+   means that they are equivalent if their component types are equivalent
+   (i.e., `(i32, i32)` is the same regardless in which crate it is used).
+
+This algorithm also provides a stable ID for types that are defined in one
+crate but instantiated from metadata within another crate. We just have to
+take care to always map crate and `NodeId`s back to the original crate
+context.
+
+As a side-effect these unique type IDs also help to solve a problem arising
+from lifetime parameters. Since lifetime parameters are completely omitted
+in debuginfo, more than one `Ty` instance may map to the same debuginfo
+type metadata, that is, some struct `Struct<'a>` may have N instantiations
+with different concrete substitutions for `'a`, and thus there will be N
+`Ty` instances for the type `Struct<'a>` even though it is not generic
+otherwise. Unfortunately this means that we cannot use `ty::type_id()` as
+cheap identifier for type metadata -- we have done this in the past, but it
+led to unnecessary metadata duplication in the best case and LLVM
+assertions in the worst. However, the unique type ID as described above
+*can* be used as identifier. Since it is comparatively expensive to
+construct, though, `ty::type_id()` is still used additionally as an
+optimization for cases where the exact same type has been seen before
+(which is most of the time).

compiler/rustc_codegen_llvm/src/debuginfo/doc.rs

@@ -1,179 +0,0 @@
-//! # Debug Info Module
-//!
-//! This module serves the purpose of generating debug symbols. We use LLVM's
-//! [source level debugging](https://llvm.org/docs/SourceLevelDebugging.html)
-//! features for generating the debug information. The general principle is
-//! this:
-//!
-//! Given the right metadata in the LLVM IR, the LLVM code generator is able to
-//! create DWARF debug symbols for the given code. The
-//! [metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured
-//! much like DWARF *debugging information entries* (DIE), representing type
-//! information such as datatype layout, function signatures, block layout,
-//! variable location and scope information, etc. It is the purpose of this
-//! module to generate correct metadata and insert it into the LLVM IR.
-//!
-//! As the exact format of metadata trees may change between different LLVM
-//! versions, we now use LLVM
-//! [DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html)
-//! to create metadata where possible. This will hopefully ease the adaption of
-//! this module to future LLVM versions.
-//!
-//! The public API of the module is a set of functions that will insert the
-//! correct metadata into the LLVM IR when called with the right parameters.
-//! The module is thus driven from an outside client with functions like
-//! `debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`.
-//!
-//! Internally the module will try to reuse already created metadata by
-//! utilizing a cache. The way to get a shared metadata node when needed is
-//! thus to just call the corresponding function in this module:
-//!
-//!     let file_metadata = file_metadata(cx, file);
-//!
-//! The function will take care of probing the cache for an existing node for
-//! that exact file path.
-//!
-//! All private state used by the module is stored within either the
-//! CrateDebugContext struct (owned by the CodegenCx) or the
-//! FunctionDebugContext (owned by the FunctionCx).
-//!
-//! This file consists of three conceptual sections:
-//! 1. The public interface of the module
-//! 2. Module-internal metadata creation functions
-//! 3. Minor utility functions
-//!
-//!
-//! ## Recursive Types
-//!
-//! Some kinds of types, such as structs and enums can be recursive. That means
-//! that the type definition of some type X refers to some other type which in
-//! turn (transitively) refers to X. This introduces cycles into the type
-//! referral graph. A naive algorithm doing an on-demand, depth-first traversal
-//! of this graph when describing types, can get trapped in an endless loop
-//! when it reaches such a cycle.
-//!
-//! For example, the following simple type for a singly-linked list...
-//!
-//! ```
-//! struct List {
-//!     value: i32,
-//!     tail: Option<Box<List>>,
-//! }
-//! ```
-//!
-//! will generate the following callstack with a naive DFS algorithm:
-//!
-//! ```
-//! describe(t = List)
-//!   describe(t = i32)
-//!   describe(t = Option<Box<List>>)
-//!     describe(t = Box<List>)
-//!       describe(t = List) // at the beginning again...
-//!       ...
-//! ```
-//!
-//! To break cycles like these, we use "forward declarations". That is, when
-//! the algorithm encounters a possibly recursive type (any struct or enum), it
-//! immediately creates a type description node and inserts it into the cache
-//! *before* describing the members of the type. This type description is just
-//! a stub (as type members are not described and added to it yet) but it
-//! allows the algorithm to already refer to the type. After the stub is
-//! inserted into the cache, the algorithm continues as before. If it now
-//! encounters a recursive reference, it will hit the cache and does not try to
-//! describe the type anew.
-//!
-//! This behavior is encapsulated in the 'RecursiveTypeDescription' enum,
-//! which represents a kind of continuation, storing all state needed to
-//! continue traversal at the type members after the type has been registered
-//! with the cache. (This implementation approach might be a tad over-
-//! engineered and may change in the future)
-//!
-//!
-//! ## Source Locations and Line Information
-//!
-//! In addition to data type descriptions the debugging information must also
-//! allow to map machine code locations back to source code locations in order
-//! to be useful. This functionality is also handled in this module. The
-//! following functions allow to control source mappings:
-//!
-//! + set_source_location()
-//! + clear_source_location()
-//! + start_emitting_source_locations()
-//!
-//! `set_source_location()` allows to set the current source location. All IR
-//! instructions created after a call to this function will be linked to the
-//! given source location, until another location is specified with
-//! `set_source_location()` or the source location is cleared with
-//! `clear_source_location()`. In the later case, subsequent IR instruction
-//! will not be linked to any source location. As you can see, this is a
-//! stateful API (mimicking the one in LLVM), so be careful with source
-//! locations set by previous calls. It's probably best to not rely on any
-//! specific state being present at a given point in code.
-//!
-//! One topic that deserves some extra attention is *function prologues*. At
-//! the beginning of a function's machine code there are typically a few
-//! instructions for loading argument values into allocas and checking if
-//! there's enough stack space for the function to execute. This *prologue* is
-//! not visible in the source code and LLVM puts a special PROLOGUE END marker
-//! into the line table at the first non-prologue instruction of the function.
-//! In order to find out where the prologue ends, LLVM looks for the first
-//! instruction in the function body that is linked to a source location. So,
-//! when generating prologue instructions we have to make sure that we don't
-//! emit source location information until the 'real' function body begins. For
-//! this reason, source location emission is disabled by default for any new
-//! function being codegened and is only activated after a call to the third
-//! function from the list above, `start_emitting_source_locations()`. This
-//! function should be called right before regularly starting to codegen the
-//! top-level block of the given function.
-//!
-//! There is one exception to the above rule: `llvm.dbg.declare` instruction
-//! must be linked to the source location of the variable being declared. For
-//! function parameters these `llvm.dbg.declare` instructions typically occur
-//! in the middle of the prologue, however, they are ignored by LLVM's prologue
-//! detection. The `create_argument_metadata()` and related functions take care
-//! of linking the `llvm.dbg.declare` instructions to the correct source
-//! locations even while source location emission is still disabled, so there
-//! is no need to do anything special with source location handling here.
-//!
-//! ## Unique Type Identification
-//!
-//! In order for link-time optimization to work properly, LLVM needs a unique
-//! type identifier that tells it across compilation units which types are the
-//! same as others. This type identifier is created by
-//! `TypeMap::get_unique_type_id_of_type()` using the following algorithm:
-//!
-//! (1) Primitive types have their name as ID
-//! (2) Structs, enums and traits have a multipart identifier
-//!
-//!     (1) The first part is the SVH (strict version hash) of the crate they
-//!          were originally defined in
-//!
-//!     (2) The second part is the ast::NodeId of the definition in their
-//!          original crate
-//!
-//!     (3) The final part is a concatenation of the type IDs of their concrete
-//!          type arguments if they are generic types.
-//!
-//! (3) Tuple-, pointer and function types are structurally identified, which
-//!     means that they are equivalent if their component types are equivalent
-//!     (i.e., (i32, i32) is the same regardless in which crate it is used).
-//!
-//! This algorithm also provides a stable ID for types that are defined in one
-//! crate but instantiated from metadata within another crate. We just have to
-//! take care to always map crate and `NodeId`s back to the original crate
-//! context.
-//!
-//! As a side-effect these unique type IDs also help to solve a problem arising
-//! from lifetime parameters. Since lifetime parameters are completely omitted
-//! in debuginfo, more than one `Ty` instance may map to the same debuginfo
-//! type metadata, that is, some struct `Struct<'a>` may have N instantiations
-//! with different concrete substitutions for `'a`, and thus there will be N
-//! `Ty` instances for the type `Struct<'a>` even though it is not generic
-//! otherwise. Unfortunately this means that we cannot use `ty::type_id()` as
-//! cheap identifier for type metadata -- we have done this in the past, but it
-//! led to unnecessary metadata duplication in the best case and LLVM
-//! assertions in the worst. However, the unique type ID as described above
-//! *can* be used as identifier. Since it is comparatively expensive to
-//! construct, though, `ty::type_id()` is still used additionally as an
-//! optimization for cases where the exact same type has been seen before
-//! (which is most of the time).

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -1083,9 +1083,9 @@ pub fn compile_unit_metadata(
             );
         }
 
-        // Insert `llvm.ident` metadata on the wasm32 targets since that will
+        // Insert `llvm.ident` metadata on the wasm targets since that will
         // get hooked up to the "producer" sections `processed-by` information.
-        if tcx.sess.opts.target_triple.triple().starts_with("wasm32") {
+        if tcx.sess.target.is_like_wasm {
             let name_metadata = llvm::LLVMMDStringInContext(
                 debug_context.llcontext,
                 rustc_producer.as_ptr().cast(),
@@ -1962,9 +1962,7 @@ fn prepare_enum_metadata(
 
     let discriminant_type_metadata = |discr: Primitive| {
         let enumerators_metadata: Vec<_> = match enum_type.kind() {
-            ty::Adt(def, _) => def
+            ty::Adt(def, _) => iter::zip(def.discriminants(tcx), &def.variants)
-                .discriminants(tcx)
-                .zip(&def.variants)
                 .map(|((_, discr), v)| {
                     let name = v.ident.as_str();
                     let is_unsigned = match discr.ty.kind() {
@@ -2336,9 +2334,7 @@ fn compute_type_parameters(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) -> &'ll DIAr
         if substs.types().next().is_some() {
             let generics = cx.tcx.generics_of(def.did);
             let names = get_parameter_names(cx, generics);
-            let template_params: Vec<_> = substs
+            let template_params: Vec<_> = iter::zip(substs, names)
-                .iter()
-                .zip(names)
                 .filter_map(|(kind, name)| {
                     if let GenericArgKind::Type(ty) = kind.unpack() {
                         let actual_type =

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -1,5 +1,4 @@
-// See doc.rs for documentation.
+#![doc = include_str!("doc.md")]
-mod doc;
 
 use rustc_codegen_ssa::mir::debuginfo::VariableKind::*;
 
@@ -38,6 +37,7 @@ use rustc_target::abi::{LayoutOf, Primitive, Size};
 use libc::c_uint;
 use smallvec::SmallVec;
 use std::cell::RefCell;
+use std::iter;
 use tracing::debug;
 
 mod create_scope_map;
@@ -344,7 +344,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             spflags |= DISPFlags::SPFlagOptimized;
         }
         if let Some((id, _)) = self.tcx.entry_fn(LOCAL_CRATE) {
-            if id.to_def_id() == def_id {
+            if id == def_id {
                 spflags |= DISPFlags::SPFlagMainSubprogram;
             }
         }
@@ -449,9 +449,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             // Again, only create type information if full debuginfo is enabled
             let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full {
                 let names = get_parameter_names(cx, generics);
-                substs
+                iter::zip(substs, names)
-                    .iter()
-                    .zip(names)
                     .filter_map(|(kind, name)| {
                         if let GenericArgKind::Type(ty) = kind.unpack() {
                             let actual_type =

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1053,46 +1053,48 @@ fn generic_simd_intrinsic(
         let vec_ty = bx.type_vector(elem_ty, in_len);
 
         let (intr_name, fn_ty) = match name {
-            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
-            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
             sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
             sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
             sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
-            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
+            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
             _ => return_error!("unrecognized intrinsic `{}`", name),
         };
-
         let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
         let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty);
         let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
-        unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
         Ok(c)
     }
 
     if std::matches!(
         name,
-        sym::simd_fsqrt
+        sym::simd_ceil
-            | sym::simd_fsin
-            | sym::simd_fcos
             | sym::simd_fabs
-            | sym::simd_floor
+            | sym::simd_fcos
-            | sym::simd_ceil
-            | sym::simd_fexp
             | sym::simd_fexp2
+            | sym::simd_fexp
             | sym::simd_flog10
             | sym::simd_flog2
             | sym::simd_flog
-            | sym::simd_fpowi
+            | sym::simd_floor
-            | sym::simd_fpow
             | sym::simd_fma
+            | sym::simd_fpow
+            | sym::simd_fpowi
+            | sym::simd_fsin
+            | sym::simd_fsqrt
+            | sym::simd_round
+            | sym::simd_trunc
     ) {
         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
     }