New upstream version 1.51.0+dfsg1

Cargo.toml

@@ -4,6 +4,7 @@ members = [
   "compiler/rustc",
   "library/std",
   "library/test",
+  "src/rustdoc-json-types",
   "src/tools/cargotest",
   "src/tools/clippy",
   "src/tools/compiletest",
@@ -31,6 +32,7 @@ members = [
   "src/tools/rustdoc-themes",
   "src/tools/unicode-table-generator",
   "src/tools/expand-yaml-anchors",
+  "src/tools/jsondocck",
 ]
 
 exclude = [
@@ -104,11 +106,5 @@ rustc-std-workspace-core = { path = 'library/rustc-std-workspace-core' }
 rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
 rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
 
-# This crate's integration with libstd is a bit wonky, so we use a submodule
-# instead of a crates.io dependency. Make sure everything else in the repo is
-# also using the submodule, however, so we can avoid duplicate copies of the
-# source code for this crate.
-backtrace = { path = "library/backtrace" }
-
 [patch."https://github.com/rust-lang/rust-clippy"]
 clippy_lints = { path = "src/tools/clippy/clippy_lints" }

README.md

@@ -1,6 +1,4 @@
-<a href = "https://www.rust-lang.org/">
+# The Rust Programming Language
-<img width = "90%" height = "auto" src = "https://img.shields.io/badge/Rust-Programming%20Language-black?style=flat&logo=rust" alt = "The Rust Programming Language">
-</a>
 
 This is the main source code repository for [Rust]. It contains the compiler,
 standard library, and documentation.

RELEASES.md

@@ -1,3 +1,174 @@
+Version 1.51.0 (2021-03-25)
+============================
+
+Language
+--------
+- [You can now parameterize items such as functions, traits, and `struct`s by constant
+  values in addition to by types and lifetimes.][79135] Also known as "const generics"
+  E.g. you can now write the following. Note: Only values of primitive integers, 
+  `bool`, or `char` types are currently permitted.
+  ```rust
+  struct GenericArray<T, const LENGTH: usize> {
+      inner: [T; LENGTH]
+  }
+
+  impl<T, const LENGTH: usize> GenericArray<T, LENGTH> {
+      const fn last(&self) -> Option<&T> {
+          if LENGTH == 0 {
+              None
+          } else {
+              Some(&self.inner[LENGTH - 1])
+          }
+      }
+  }
+  ```
+
+
+Compiler
+--------
+
+- [Added the `-Csplit-debuginfo` codegen option for macOS platforms.][79570]
+  This option controls whether debug information is split across multiple files
+  or packed into a single file. **Note** This option is unstable on other platforms.
+- [Added tier 3\* support for `aarch64_be-unknown-linux-gnu`,
+  `aarch64-unknown-linux-gnu_ilp32`, and `aarch64_be-unknown-linux-gnu_ilp32` targets.][81455]
+- [Added tier 3 support for `i386-unknown-linux-gnu` and `i486-unknown-linux-gnu` targets.][80662]
+- [The `target-cpu=native` option will now detect individual features of CPUs.][80749]
+- [Rust now uses `inline-asm` for stack probes when used with LLVM 11.0.1+][77885]
+
+\* Refer to Rust's [platform support page][forge-platform-support] for more
+information on Rust's tiered platform support.
+
+Libraries
+---------
+
+- [`Box::downcast` is now also implemented for any `dyn Any + Send + Sync` object.][80945]
+- [`str` now implements `AsMut<str>`.][80279]
+- [`u64` and `u128` now implement `From<char>`.][79502]
+- [`Error` is now implemented for `&T` where `T` implements `Error`.][75180]
+- [`Poll::{map_ok, map_err}` are now implemented for `Poll<Option<Result<T, E>>>`.][80968]
+- [`unsigned_abs` is now implemented for all signed integer types.][80959]
+- [`io::Empty` now implements `io::Seek`.][78044]
+- [`rc::Weak<T>` and `sync::Weak<T>`'s methods such as `as_ptr` are now implemented for
+  `T: ?Sized` types.][80764]
+
+Stabilized APIs
+---------------
+
+- [`Arc::decrement_strong_count`]
+- [`Arc::increment_strong_count`]
+- [`Once::call_once_force`]
+- [`Peekable::next_if_eq`]
+- [`Peekable::next_if`]
+- [`Seek::stream_position`]
+- [`array::IntoIter`]
+- [`panic::panic_any`]
+- [`ptr::addr_of!`]
+- [`ptr::addr_of_mut!`]
+- [`slice::fill_with`]
+- [`slice::split_inclusive_mut`]
+- [`slice::split_inclusive`]
+- [`slice::strip_prefix`]
+- [`slice::strip_suffix`]
+- [`str::split_inclusive`]
+- [`sync::OnceState`]
+- [`task::Wake`]
+
+Cargo
+-----
+- [Added the `split-debuginfo` profile option to control the -Csplit-debuginfo
+  codegen option.][cargo/9112]
+- [Added the `resolver` field to `Cargo.toml` to enable the new feature resolver
+  and CLI option behavior.][cargo/8997] Version 2 of the feature resolver will try
+  to avoid unifying features of dependencies where that unification could be unwanted.
+  Such as using the same dependency with a `std` feature in a build scripts and
+  proc-macros, while using the `no-std` feature in the final binary. See the
+  [Cargo book documentation][feature-resolver@2.0] for more information on the feature.
+
+Rustdoc
+-------
+
+- [Rustdoc will now include documentation for methods available from `Deref` traits.][80653]
+- [You can now provide a `--default-theme` flag which sets the default theme to use for
+  documentation.][79642]
+
+Various improvements to intra-doc links:
+
+- [You can link to non-path primitives such as `slice`.][80181]
+- [You can link to associated items.][74489]
+- [You can now include generic parameters when linking to items, like `Vec<T>`.][76934]
+
+Misc
+----
+- [You can now pass `--include-ignored` to tests (e.g. with
+  `cargo test -- --include-ignored`) to include testing tests marked `#[ignore]`.][80053]
+
+Compatibility Notes
+-------------------
+
+- [WASI platforms no longer use the `wasm-bindgen` ABI, and instead use the wasm32 ABI.][79998]
+- [`rustc` no longer promotes division, modulo and indexing operations to `const` that
+  could fail.][80579]
+- [The minimum version of glibc for the following platforms has been bumped to version 2.31
+  for the distributed artifacts.][81521]
+    - `armv5te-unknown-linux-gnueabi`
+    - `sparc64-unknown-linux-gnu`
+    - `thumbv7neon-unknown-linux-gnueabihf`
+    - `armv7-unknown-linux-gnueabi`
+    - `x86_64-unknown-linux-gnux32`
+
+Internal Only
+-------------
+
+- [Consistently avoid constructing optimized MIR when not doing codegen][80718]
+
+[79135]: https://github.com/rust-lang/rust/pull/79135
+[74489]: https://github.com/rust-lang/rust/pull/74489
+[76934]: https://github.com/rust-lang/rust/pull/76934
+[79570]: https://github.com/rust-lang/rust/pull/79570
+[80181]: https://github.com/rust-lang/rust/pull/80181
+[79642]: https://github.com/rust-lang/rust/pull/79642
+[80945]: https://github.com/rust-lang/rust/pull/80945
+[80279]: https://github.com/rust-lang/rust/pull/80279
+[80053]: https://github.com/rust-lang/rust/pull/80053
+[79502]: https://github.com/rust-lang/rust/pull/79502
+[75180]: https://github.com/rust-lang/rust/pull/75180
+[79135]: https://github.com/rust-lang/rust/pull/79135
+[81521]: https://github.com/rust-lang/rust/pull/81521
+[80968]: https://github.com/rust-lang/rust/pull/80968
+[80959]: https://github.com/rust-lang/rust/pull/80959
+[80718]: https://github.com/rust-lang/rust/pull/80718
+[80653]: https://github.com/rust-lang/rust/pull/80653
+[80579]: https://github.com/rust-lang/rust/pull/80579
+[79998]: https://github.com/rust-lang/rust/pull/79998
+[78044]: https://github.com/rust-lang/rust/pull/78044
+[81455]: https://github.com/rust-lang/rust/pull/81455
+[80764]: https://github.com/rust-lang/rust/pull/80764
+[80749]: https://github.com/rust-lang/rust/pull/80749
+[80662]: https://github.com/rust-lang/rust/pull/80662
+[77885]: https://github.com/rust-lang/rust/pull/77885
+[cargo/8997]: https://github.com/rust-lang/cargo/pull/8997
+[cargo/9112]: https://github.com/rust-lang/cargo/pull/9112
+[feature-resolver@2.0]: https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-resolver-version-2
+[`Once::call_once_force`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#method.call_once_force
+[`sync::OnceState`]: https://doc.rust-lang.org/stable/std/sync/struct.OnceState.html
+[`panic::panic_any`]: https://doc.rust-lang.org/stable/std/panic/fn.panic_any.html
+[`slice::strip_prefix`]: https://doc.rust-lang.org/stable/std/primitive.slice.html#method.strip_prefix
+[`slice::strip_suffix`]: https://doc.rust-lang.org/stable/std/primitive.slice.html#method.strip_prefix
+[`Arc::increment_strong_count`]: https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#method.increment_strong_count
+[`Arc::decrement_strong_count`]: https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#method.decrement_strong_count
+[`slice::fill_with`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.fill_with
+[`ptr::addr_of!`]: https://doc.rust-lang.org/nightly/std/ptr/macro.addr_of.html
+[`ptr::addr_of_mut!`]: https://doc.rust-lang.org/nightly/std/ptr/macro.addr_of_mut.html
+[`array::IntoIter`]: https://doc.rust-lang.org/nightly/std/array/struct.IntoIter.html
+[`slice::split_inclusive`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.split_inclusive
+[`slice::split_inclusive_mut`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.split_inclusive_mut
+[`str::split_inclusive`]: https://doc.rust-lang.org/nightly/std/primitive.str.html#method.split_inclusive
+[`task::Wake`]: https://doc.rust-lang.org/nightly/std/task/trait.Wake.html
+[`Seek::stream_position`]: https://doc.rust-lang.org/nightly/std/io/trait.Seek.html#method.stream_position
+[`Peekable::next_if`]: https://doc.rust-lang.org/nightly/std/iter/struct.Peekable.html#method.next_if
+[`Peekable::next_if_eq`]: https://doc.rust-lang.org/nightly/std/iter/struct.Peekable.html#method.next_if_eq
+
 Version 1.50.0 (2021-02-11)
 ============================
 

compiler/rustc_arena/src/lib.rs

@@ -11,12 +11,10 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(no_crate_inject, attr(deny(warnings)))
 )]
-#![feature(array_value_iter_slice)]
 #![feature(dropck_eyepatch)]
 #![feature(new_uninit)]
 #![feature(maybe_uninit_slice)]
-#![feature(array_value_iter)]
+#![cfg_attr(bootstrap, feature(min_const_generics))]
-#![feature(min_const_generics)]
 #![feature(min_specialization)]
 #![cfg_attr(test, feature(test))]
 
@@ -32,7 +30,7 @@ use std::slice;
 
 #[inline(never)]
 #[cold]
-pub fn cold_path<F: FnOnce() -> R, R>(f: F) -> R {
+fn cold_path<F: FnOnce() -> R, R>(f: F) -> R {
     f()
 }
 

compiler/rustc_ast/src/ast.rs

@@ -23,8 +23,8 @@ pub use GenericArgs::*;
 pub use UnsafeSource::*;
 
 use crate::ptr::P;
-use crate::token::{self, CommentKind, DelimToken};
+use crate::token::{self, CommentKind, DelimToken, Token};
-use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream};
+use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -167,10 +167,7 @@ pub enum GenericArgs {
 
 impl GenericArgs {
     pub fn is_angle_bracketed(&self) -> bool {
-        match *self {
+        matches!(self, AngleBracketed(..))
-            AngleBracketed(..) => true,
-            _ => false,
-        }
     }
 
     pub fn span(&self) -> Span {
@@ -245,12 +242,21 @@ impl Into<Option<P<GenericArgs>>> for ParenthesizedArgs {
 /// A path like `Foo(A, B) -> C`.
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct ParenthesizedArgs {
-    /// Overall span
+    /// ```text
+    /// Foo(A, B) -> C
+    /// ^^^^^^^^^^^^^^
+    /// ```
     pub span: Span,
 
     /// `(A, B)`
     pub inputs: Vec<P<Ty>>,
 
+    /// ```text
+    /// Foo(A, B) -> C
+    ///    ^^^^^^
+    /// ```
+    pub inputs_span: Span,
+
     /// `C`
     pub output: FnRetTy,
 }
@@ -371,6 +377,8 @@ pub enum GenericParamKind {
         ty: P<Ty>,
         /// Span of the `const` keyword.
         kw_span: Span,
+        /// Optional default value for the const generic param
+        default: Option<AnonConst>,
     },
 }
 
@@ -434,9 +442,9 @@ pub enum WherePredicate {
 impl WherePredicate {
     pub fn span(&self) -> Span {
         match self {
-            &WherePredicate::BoundPredicate(ref p) => p.span,
+            WherePredicate::BoundPredicate(p) => p.span,
-            &WherePredicate::RegionPredicate(ref p) => p.span,
+            WherePredicate::RegionPredicate(p) => p.span,
-            &WherePredicate::EqPredicate(ref p) => p.span,
+            WherePredicate::EqPredicate(p) => p.span,
         }
     }
 }
@@ -629,23 +637,20 @@ impl Pat {
 
     /// Is this a `..` pattern?
     pub fn is_rest(&self) -> bool {
-        match self.kind {
+        matches!(self.kind, PatKind::Rest)
-            PatKind::Rest => true,
-            _ => false,
-        }
     }
 }
 
-/// A single field in a struct pattern
+/// A single field in a struct pattern.
 ///
-/// Patterns like the fields of Foo `{ x, ref y, ref mut z }`
+/// Patterns like the fields of `Foo { x, ref y, ref mut z }`
-/// are treated the same as` x: x, y: ref y, z: ref mut z`,
+/// are treated the same as `x: x, y: ref y, z: ref mut z`,
-/// except is_shorthand is true
+/// except when `is_shorthand` is true.
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct FieldPat {
-    /// The identifier for the field
+    /// The identifier for the field.
     pub ident: Ident,
-    /// The pattern the field is destructured to
+    /// The pattern the field is destructured to.
     pub pat: P<Pat>,
     pub is_shorthand: bool,
     pub attrs: AttrVec,
@@ -852,10 +857,7 @@ impl BinOpKind {
         }
     }
     pub fn lazy(&self) -> bool {
-        match *self {
+        matches!(self, BinOpKind::And | BinOpKind::Or)
-            BinOpKind::And | BinOpKind::Or => true,
-            _ => false,
-        }
     }
 
     pub fn is_comparison(&self) -> bool {
@@ -923,16 +925,6 @@ impl Stmt {
         }
     }
 
-    pub fn set_tokens(&mut self, tokens: Option<LazyTokenStream>) {
-        match self.kind {
-            StmtKind::Local(ref mut local) => local.tokens = tokens,
-            StmtKind::Item(ref mut item) => item.tokens = tokens,
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens = tokens,
-            StmtKind::Empty => {}
-            StmtKind::MacCall(ref mut mac) => mac.tokens = tokens,
-        }
-    }
-
     pub fn has_trailing_semicolon(&self) -> bool {
         match &self.kind {
             StmtKind::Semi(_) => true,
@@ -963,17 +955,11 @@ impl Stmt {
     }
 
     pub fn is_item(&self) -> bool {
-        match self.kind {
+        matches!(self.kind, StmtKind::Item(_))
-            StmtKind::Item(_) => true,
-            _ => false,
-        }
     }
 
     pub fn is_expr(&self) -> bool {
-        match self.kind {
+        matches!(self.kind, StmtKind::Expr(_))
-            StmtKind::Expr(_) => true,
-            _ => false,
-        }
     }
 }
 
@@ -1107,15 +1093,9 @@ impl Expr {
         if let ExprKind::Block(ref block, _) = self.kind {
             match block.stmts.last().map(|last_stmt| &last_stmt.kind) {
                 // Implicit return
-                Some(&StmtKind::Expr(_)) => true,
+                Some(StmtKind::Expr(_)) => true,
-                Some(&StmtKind::Semi(ref expr)) => {
+                // Last statement is an explicit return?
-                    if let ExprKind::Ret(_) = expr.kind {
+                Some(StmtKind::Semi(expr)) => matches!(expr.kind, ExprKind::Ret(_)),
-                        // Last statement is explicit return.
-                        true
-                    } else {
-                        false
-                    }
-                }
                 // This is a block that doesn't end in either an implicit or explicit return.
                 _ => false,
             }
@@ -1128,7 +1108,7 @@ impl Expr {
     /// Is this expr either `N`, or `{ N }`.
     ///
     /// If this is not the case, name resolution does not resolve `N` when using
-    /// `feature(min_const_generics)` as more complex expressions are not supported.
+    /// `min_const_generics` as more complex expressions are not supported.
     pub fn is_potential_trivial_const_param(&self) -> bool {
         let this = if let ExprKind::Block(ref block, None) = self.kind {
             if block.stmts.len() == 1 {
@@ -1483,8 +1463,8 @@ pub enum MacArgs {
     Eq(
         /// Span of the `=` token.
         Span,
-        /// Token stream of the "value".
+        /// "value" as a nonterminal token.
-        TokenStream,
+        Token,
     ),
 }
 
@@ -1497,10 +1477,10 @@ impl MacArgs {
     }
 
     pub fn span(&self) -> Option<Span> {
-        match *self {
+        match self {
             MacArgs::Empty => None,
             MacArgs::Delimited(dspan, ..) => Some(dspan.entire()),
-            MacArgs::Eq(eq_span, ref tokens) => Some(eq_span.to(tokens.span().unwrap_or(eq_span))),
+            MacArgs::Eq(eq_span, token) => Some(eq_span.to(token.span)),
         }
     }
 
@@ -1509,7 +1489,8 @@ impl MacArgs {
     pub fn inner_tokens(&self) -> TokenStream {
         match self {
             MacArgs::Empty => TokenStream::default(),
-            MacArgs::Delimited(.., tokens) | MacArgs::Eq(.., tokens) => tokens.clone(),
+            MacArgs::Delimited(.., tokens) => tokens.clone(),
+            MacArgs::Eq(.., token) => TokenTree::Token(token.clone()).into(),
         }
     }
 
@@ -1652,26 +1633,17 @@ pub enum LitKind {
 impl LitKind {
     /// Returns `true` if this literal is a string.
     pub fn is_str(&self) -> bool {
-        match *self {
+        matches!(self, LitKind::Str(..))
-            LitKind::Str(..) => true,
-            _ => false,
-        }
     }
 
     /// Returns `true` if this literal is byte literal string.
     pub fn is_bytestr(&self) -> bool {
-        match self {
+        matches!(self, LitKind::ByteStr(_))
-            LitKind::ByteStr(_) => true,
-            _ => false,
-        }
     }
 
     /// Returns `true` if this is a numeric literal.
     pub fn is_numeric(&self) -> bool {
-        match *self {
+        matches!(self, LitKind::Int(..) | LitKind::Float(..))
-            LitKind::Int(..) | LitKind::Float(..) => true,
-            _ => false,
-        }
     }
 
     /// Returns `true` if this literal has no suffix.
@@ -1974,7 +1946,7 @@ impl TyKind {
     }
 
     pub fn is_unit(&self) -> bool {
-        if let TyKind::Tup(ref tys) = *self { tys.is_empty() } else { false }
+        matches!(self, TyKind::Tup(tys) if tys.is_empty())
     }
 }
 
@@ -2237,10 +2209,7 @@ impl FnDecl {
         self.inputs.get(0).map_or(false, Param::is_self)
     }
     pub fn c_variadic(&self) -> bool {
-        self.inputs.last().map_or(false, |arg| match arg.ty.kind {
+        self.inputs.last().map_or(false, |arg| matches!(arg.ty.kind, TyKind::CVarArgs))
-            TyKind::CVarArgs => true,
-            _ => false,
-        })
     }
 }
 
@@ -2686,6 +2655,36 @@ impl Default for FnHeader {
     }
 }
 
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct TraitKind(
+    pub IsAuto,
+    pub Unsafe,
+    pub Generics,
+    pub GenericBounds,
+    pub Vec<P<AssocItem>>,
+);
+
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct TyAliasKind(pub Defaultness, pub Generics, pub GenericBounds, pub Option<P<Ty>>);
+
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct ImplKind {
+    pub unsafety: Unsafe,
+    pub polarity: ImplPolarity,
+    pub defaultness: Defaultness,
+    pub constness: Const,
+    pub generics: Generics,
+
+    /// The trait being implemented, if any.
+    pub of_trait: Option<TraitRef>,
+
+    pub self_ty: P<Ty>,
+    pub items: Vec<P<AssocItem>>,
+}
+
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct FnKind(pub Defaultness, pub FnSig, pub Generics, pub Option<P<Block>>);
+
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub enum ItemKind {
     /// An `extern crate` item, with the optional *original* crate name if the crate was renamed.
@@ -2707,7 +2706,7 @@ pub enum ItemKind {
     /// A function declaration (`fn`).
     ///
     /// E.g., `fn foo(bar: usize) -> usize { .. }`.
-    Fn(Defaultness, FnSig, Generics, Option<P<Block>>),
+    Fn(Box<FnKind>),
     /// A module declaration (`mod`).
     ///
     /// E.g., `mod foo;` or `mod foo { .. }`.
@@ -2721,7 +2720,7 @@ pub enum ItemKind {
     /// A type alias (`type`).
     ///
     /// E.g., `type Foo = Bar<u8>;`.
-    TyAlias(Defaultness, Generics, GenericBounds, Option<P<Ty>>),
+    TyAlias(Box<TyAliasKind>),
     /// An enum definition (`enum`).
     ///
     /// E.g., `enum Foo<A, B> { C<A>, D<B> }`.
@@ -2737,7 +2736,7 @@ pub enum ItemKind {
     /// A trait declaration (`trait`).
     ///
     /// E.g., `trait Foo { .. }`, `trait Foo<T> { .. }` or `auto trait Foo {}`.
-    Trait(IsAuto, Unsafe, Generics, GenericBounds, Vec<P<AssocItem>>),
+    Trait(Box<TraitKind>),
     /// Trait alias
     ///
     /// E.g., `trait Foo = Bar + Quux;`.
@@ -2745,19 +2744,7 @@ pub enum ItemKind {
     /// An implementation.
     ///
     /// E.g., `impl<A> Foo<A> { .. }` or `impl<A> Trait for Foo<A> { .. }`.
-    Impl {
+    Impl(Box<ImplKind>),
-        unsafety: Unsafe,
-        polarity: ImplPolarity,
-        defaultness: Defaultness,
-        constness: Const,
-        generics: Generics,
-
-        /// The trait being implemented, if any.
-        of_trait: Option<TraitRef>,
-
-        self_ty: P<Ty>,
-        items: Vec<P<AssocItem>>,
-    },
     /// A macro invocation.
     ///
     /// E.g., `foo!(..)`.
@@ -2767,6 +2754,9 @@ pub enum ItemKind {
     MacroDef(MacroDef),
 }
 
+#[cfg(target_arch = "x86_64")]
+rustc_data_structures::static_assert_size!(ItemKind, 112);
+
 impl ItemKind {
     pub fn article(&self) -> &str {
         use ItemKind::*;
@@ -2801,14 +2791,14 @@ impl ItemKind {
 
     pub fn generics(&self) -> Option<&Generics> {
         match self {
-            Self::Fn(_, _, generics, _)
+            Self::Fn(box FnKind(_, _, generics, _))
-            | Self::TyAlias(_, generics, ..)
+            | Self::TyAlias(box TyAliasKind(_, generics, ..))
             | Self::Enum(_, generics)
             | Self::Struct(_, generics)
             | Self::Union(_, generics)
-            | Self::Trait(_, _, generics, ..)
+            | Self::Trait(box TraitKind(_, _, generics, ..))
             | Self::TraitAlias(generics, _)
-            | Self::Impl { generics, .. } => Some(generics),
+            | Self::Impl(box ImplKind { generics, .. }) => Some(generics),
             _ => None,
         }
     }
@@ -2831,17 +2821,22 @@ pub enum AssocItemKind {
     /// If `def` is parsed, then the constant is provided, and otherwise required.
     Const(Defaultness, P<Ty>, Option<P<Expr>>),
     /// An associated function.
-    Fn(Defaultness, FnSig, Generics, Option<P<Block>>),
+    Fn(Box<FnKind>),
     /// An associated type.
-    TyAlias(Defaultness, Generics, GenericBounds, Option<P<Ty>>),
+    TyAlias(Box<TyAliasKind>),
     /// A macro expanding to associated items.
     MacCall(MacCall),
 }
 
+#[cfg(target_arch = "x86_64")]
+rustc_data_structures::static_assert_size!(AssocItemKind, 72);
+
 impl AssocItemKind {
     pub fn defaultness(&self) -> Defaultness {
         match *self {
-            Self::Const(def, ..) | Self::Fn(def, ..) | Self::TyAlias(def, ..) => def,
+            Self::Const(def, ..)
+            | Self::Fn(box FnKind(def, ..))
+            | Self::TyAlias(box TyAliasKind(def, ..)) => def,
             Self::MacCall(..) => Defaultness::Final,
         }
     }
@@ -2851,8 +2846,8 @@ impl From<AssocItemKind> for ItemKind {
     fn from(assoc_item_kind: AssocItemKind) -> ItemKind {
         match assoc_item_kind {
             AssocItemKind::Const(a, b, c) => ItemKind::Const(a, b, c),
-            AssocItemKind::Fn(a, b, c, d) => ItemKind::Fn(a, b, c, d),
+            AssocItemKind::Fn(fn_kind) => ItemKind::Fn(fn_kind),
-            AssocItemKind::TyAlias(a, b, c, d) => ItemKind::TyAlias(a, b, c, d),
+            AssocItemKind::TyAlias(ty_alias_kind) => ItemKind::TyAlias(ty_alias_kind),
             AssocItemKind::MacCall(a) => ItemKind::MacCall(a),
         }
     }
@@ -2864,8 +2859,8 @@ impl TryFrom<ItemKind> for AssocItemKind {
     fn try_from(item_kind: ItemKind) -> Result<AssocItemKind, ItemKind> {
         Ok(match item_kind {
             ItemKind::Const(a, b, c) => AssocItemKind::Const(a, b, c),
-            ItemKind::Fn(a, b, c, d) => AssocItemKind::Fn(a, b, c, d),
+            ItemKind::Fn(fn_kind) => AssocItemKind::Fn(fn_kind),
-            ItemKind::TyAlias(a, b, c, d) => AssocItemKind::TyAlias(a, b, c, d),
+            ItemKind::TyAlias(ty_alias_kind) => AssocItemKind::TyAlias(ty_alias_kind),
             ItemKind::MacCall(a) => AssocItemKind::MacCall(a),
             _ => return Err(item_kind),
         })
@@ -2877,20 +2872,23 @@ impl TryFrom<ItemKind> for AssocItemKind {
 pub enum ForeignItemKind {
     /// A foreign static item (`static FOO: u8`).
     Static(P<Ty>, Mutability, Option<P<Expr>>),
-    /// A foreign function.
+    /// An foreign function.
-    Fn(Defaultness, FnSig, Generics, Option<P<Block>>),
+    Fn(Box<FnKind>),
-    /// A foreign type.
+    /// An foreign type.
-    TyAlias(Defaultness, Generics, GenericBounds, Option<P<Ty>>),
+    TyAlias(Box<TyAliasKind>),
     /// A macro expanding to foreign items.
     MacCall(MacCall),
 }
 
+#[cfg(target_arch = "x86_64")]
+rustc_data_structures::static_assert_size!(ForeignItemKind, 72);
+
 impl From<ForeignItemKind> for ItemKind {
     fn from(foreign_item_kind: ForeignItemKind) -> ItemKind {
         match foreign_item_kind {
             ForeignItemKind::Static(a, b, c) => ItemKind::Static(a, b, c),
-            ForeignItemKind::Fn(a, b, c, d) => ItemKind::Fn(a, b, c, d),
+            ForeignItemKind::Fn(fn_kind) => ItemKind::Fn(fn_kind),
-            ForeignItemKind::TyAlias(a, b, c, d) => ItemKind::TyAlias(a, b, c, d),
+            ForeignItemKind::TyAlias(ty_alias_kind) => ItemKind::TyAlias(ty_alias_kind),
             ForeignItemKind::MacCall(a) => ItemKind::MacCall(a),
         }
     }
@@ -2902,8 +2900,8 @@ impl TryFrom<ItemKind> for ForeignItemKind {
     fn try_from(item_kind: ItemKind) -> Result<ForeignItemKind, ItemKind> {
         Ok(match item_kind {
             ItemKind::Static(a, b, c) => ForeignItemKind::Static(a, b, c),
-            ItemKind::Fn(a, b, c, d) => ForeignItemKind::Fn(a, b, c, d),
+            ItemKind::Fn(fn_kind) => ForeignItemKind::Fn(fn_kind),
-            ItemKind::TyAlias(a, b, c, d) => ForeignItemKind::TyAlias(a, b, c, d),
+            ItemKind::TyAlias(ty_alias_kind) => ForeignItemKind::TyAlias(ty_alias_kind),
             ItemKind::MacCall(a) => ForeignItemKind::MacCall(a),
             _ => return Err(item_kind),
         })
@@ -2911,3 +2909,69 @@ impl TryFrom<ItemKind> for ForeignItemKind {
 }
 
 pub type ForeignItem = Item<ForeignItemKind>;
+
+pub trait HasTokens {
+    /// Called by `Parser::collect_tokens` to store the collected
+    /// tokens inside an AST node
+    fn finalize_tokens(&mut self, tokens: LazyTokenStream);
+}
+
+impl<T: HasTokens + 'static> HasTokens for P<T> {
+    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
+        (**self).finalize_tokens(tokens);
+    }
+}
+
+impl<T: HasTokens> HasTokens for Option<T> {
+    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
+        if let Some(inner) = self {
+            inner.finalize_tokens(tokens);
+        }
+    }
+}
+
+impl HasTokens for Attribute {
+    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
+        match &mut self.kind {
+            AttrKind::Normal(_, attr_tokens) => {
+                if attr_tokens.is_none() {
+                    *attr_tokens = Some(tokens);
+                }
+            }
+            AttrKind::DocComment(..) => {
+                panic!("Called finalize_tokens on doc comment attr {:?}", self)
+            }
+        }
+    }
+}
+
+impl HasTokens for Stmt {
+    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
+        let stmt_tokens = match self.kind {
+            StmtKind::Local(ref mut local) => &mut local.tokens,
+            StmtKind::Item(ref mut item) => &mut item.tokens,
+            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => &mut expr.tokens,
+            StmtKind::Empty => return,
+            StmtKind::MacCall(ref mut mac) => &mut mac.tokens,
+        };
+        if stmt_tokens.is_none() {
+            *stmt_tokens = Some(tokens);
+        }
+    }
+}
+
+macro_rules! derive_has_tokens {
+    ($($ty:path),*) => { $(
+        impl HasTokens for $ty {
+            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
+                if self.tokens.is_none() {
+                    self.tokens = Some(tokens);
+                }
+            }
+        }
+    )* }
+}
+
+derive_has_tokens! {
+    Item, Expr, Ty, AttrItem, Visibility, Path, Block, Pat
+}

compiler/rustc_ast/src/attr/mod.rs

@@ -234,10 +234,7 @@ impl MetaItem {
     }
 
     pub fn is_word(&self) -> bool {
-        match self.kind {
+        matches!(self.kind, MetaItemKind::Word)
-            MetaItemKind::Word => true,
-            _ => false,
-        }
     }
 
     pub fn has_name(&self, name: Symbol) -> bool {
@@ -479,7 +476,7 @@ impl MetaItemKind {
     pub fn mac_args(&self, span: Span) -> MacArgs {
         match self {
             MetaItemKind::Word => MacArgs::Empty,
-            MetaItemKind::NameValue(lit) => MacArgs::Eq(span, lit.token_tree().into()),
+            MetaItemKind::NameValue(lit) => MacArgs::Eq(span, lit.to_token()),
             MetaItemKind::List(list) => {
                 let mut tts = Vec::new();
                 for (i, item) in list.iter().enumerate() {
@@ -501,7 +498,10 @@ impl MetaItemKind {
         match *self {
             MetaItemKind::Word => vec![],
             MetaItemKind::NameValue(ref lit) => {
-                vec![TokenTree::token(token::Eq, span).into(), lit.token_tree().into()]
+                vec![
+                    TokenTree::token(token::Eq, span).into(),
+                    TokenTree::Token(lit.to_token()).into(),
+                ]
             }
             MetaItemKind::List(ref list) => {
                 let mut tokens = Vec::new();
@@ -526,7 +526,7 @@ impl MetaItemKind {
     fn list_from_tokens(tokens: TokenStream) -> Option<MetaItemKind> {
         let mut tokens = tokens.into_trees().peekable();
         let mut result = Vec::new();
-        while let Some(..) = tokens.peek() {
+        while tokens.peek().is_some() {
             let item = NestedMetaItem::from_tokens(&mut tokens)?;
             result.push(item);
             match tokens.next() {
@@ -557,10 +557,7 @@ impl MetaItemKind {
                 MetaItemKind::list_from_tokens(tokens.clone())
             }
             MacArgs::Delimited(..) => None,
-            MacArgs::Eq(_, tokens) => {
+            MacArgs::Eq(_, token) => Lit::from_token(token).ok().map(MetaItemKind::NameValue),
-                assert!(tokens.len() == 1);
-                MetaItemKind::name_value_from_tokens(&mut tokens.trees())
-            }
             MacArgs::Empty => Some(MetaItemKind::Word),
         }
     }
@@ -595,7 +592,7 @@ impl NestedMetaItem {
     fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
         match *self {
             NestedMetaItem::MetaItem(ref item) => item.token_trees_and_spacings(),
-            NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()],
+            NestedMetaItem::Literal(ref lit) => vec![TokenTree::Token(lit.to_token()).into()],
         }
     }
 

compiler/rustc_ast/src/lib.rs

@@ -9,6 +9,7 @@
     test(attr(deny(warnings)))
 )]
 #![feature(box_syntax)]
+#![feature(box_patterns)]
 #![feature(const_fn)] // For the `transmute` in `P::new`
 #![feature(const_fn_transmute)]
 #![feature(const_panic)]

compiler/rustc_ast/src/mut_visit.rs

@@ -28,7 +28,7 @@ pub trait ExpectOne<A: Array> {
 
 impl<A: Array> ExpectOne<A> for SmallVec<A> {
     fn expect_one(self, err: &'static str) -> A::Item {
-        assert!(self.len() == 1, err);
+        assert!(self.len() == 1, "{}", err);
         self.into_iter().next().unwrap()
     }
 }
@@ -365,18 +365,16 @@ pub fn visit_mac_args<T: MutVisitor>(args: &mut MacArgs, vis: &mut T) {
             visit_delim_span(dspan, vis);
             visit_tts(tokens, vis);
         }
-        MacArgs::Eq(eq_span, tokens) => {
+        MacArgs::Eq(eq_span, token) => {
             vis.visit_span(eq_span);
-            visit_tts(tokens, vis);
+            if vis.token_visiting_enabled() {
-            // The value in `#[key = VALUE]` must be visited as an expression for backward
+                visit_token(token, vis);
-            // compatibility, so that macros can be expanded in that position.
+            } else {
-            if !vis.token_visiting_enabled() {
+                // The value in `#[key = VALUE]` must be visited as an expression for backward
-                match Lrc::make_mut(&mut tokens.0).get_mut(0) {
+                // compatibility, so that macros can be expanded in that position.
-                    Some((TokenTree::Token(token), _spacing)) => match &mut token.kind {
+                match &mut token.kind {
-                        token::Interpolated(nt) => match Lrc::make_mut(nt) {
+                    token::Interpolated(nt) => match Lrc::make_mut(nt) {
-                            token::NtExpr(expr) => vis.visit_expr(expr),
+                        token::NtExpr(expr) => vis.visit_expr(expr),
-                            t => panic!("unexpected token in key-value attribute: {:?}", t),
-                        },
                         t => panic!("unexpected token in key-value attribute: {:?}", t),
                     },
                     t => panic!("unexpected token in key-value attribute: {:?}", t),
@@ -567,7 +565,7 @@ pub fn noop_visit_parenthesized_parameter_data<T: MutVisitor>(
     args: &mut ParenthesizedArgs,
     vis: &mut T,
 ) {
-    let ParenthesizedArgs { inputs, output, span } = args;
+    let ParenthesizedArgs { inputs, output, span, .. } = args;
     visit_vec(inputs, |input| vis.visit_ty(input));
     noop_visit_fn_ret_ty(output, vis);
     vis.visit_span(span);
@@ -790,8 +788,9 @@ pub fn noop_flat_map_generic_param<T: MutVisitor>(
         GenericParamKind::Type { default } => {
             visit_opt(default, |default| vis.visit_ty(default));
         }
-        GenericParamKind::Const { ty, kw_span: _ } => {
+        GenericParamKind::Const { ty, kw_span: _, default } => {
             vis.visit_ty(ty);
+            visit_opt(default, |default| vis.visit_anon_const(default));
         }
     }
     smallvec![param]
@@ -913,7 +912,7 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
             vis.visit_ty(ty);
             visit_opt(expr, |expr| vis.visit_expr(expr));
         }
-        ItemKind::Fn(_, sig, generics, body) => {
+        ItemKind::Fn(box FnKind(_, sig, generics, body)) => {
             visit_fn_sig(sig, vis);
             vis.visit_generics(generics);
             visit_opt(body, |body| vis.visit_block(body));
@@ -921,7 +920,7 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
         ItemKind::Mod(m) => vis.visit_mod(m),
         ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
         ItemKind::GlobalAsm(_ga) => {}
-        ItemKind::TyAlias(_, generics, bounds, ty) => {
+        ItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
             vis.visit_generics(generics);
             visit_bounds(bounds, vis);
             visit_opt(ty, |ty| vis.visit_ty(ty));
@@ -934,7 +933,7 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
             vis.visit_variant_data(variant_data);
             vis.visit_generics(generics);
         }
-        ItemKind::Impl {
+        ItemKind::Impl(box ImplKind {
             unsafety: _,
             polarity: _,
             defaultness: _,
@@ -943,13 +942,13 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
             of_trait,
             self_ty,
             items,
-        } => {
+        }) => {
             vis.visit_generics(generics);
             visit_opt(of_trait, |trait_ref| vis.visit_trait_ref(trait_ref));
             vis.visit_ty(self_ty);
             items.flat_map_in_place(|item| vis.flat_map_impl_item(item));
         }
-        ItemKind::Trait(_is_auto, _unsafety, generics, bounds, items) => {
+        ItemKind::Trait(box TraitKind(.., generics, bounds, items)) => {
             vis.visit_generics(generics);
             visit_bounds(bounds, vis);
             items.flat_map_in_place(|item| vis.flat_map_trait_item(item));
@@ -977,12 +976,12 @@ pub fn noop_flat_map_assoc_item<T: MutVisitor>(
             visitor.visit_ty(ty);
             visit_opt(expr, |expr| visitor.visit_expr(expr));
         }
-        AssocItemKind::Fn(_, sig, generics, body) => {
+        AssocItemKind::Fn(box FnKind(_, sig, generics, body)) => {
             visitor.visit_generics(generics);
             visit_fn_sig(sig, visitor);
             visit_opt(body, |body| visitor.visit_block(body));
         }
-        AssocItemKind::TyAlias(_, generics, bounds, ty) => {
+        AssocItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
             visitor.visit_generics(generics);
             visit_bounds(bounds, visitor);
             visit_opt(ty, |ty| visitor.visit_ty(ty));
@@ -1067,12 +1066,12 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
             visitor.visit_ty(ty);
             visit_opt(expr, |expr| visitor.visit_expr(expr));
         }
-        ForeignItemKind::Fn(_, sig, generics, body) => {
+        ForeignItemKind::Fn(box FnKind(_, sig, generics, body)) => {
             visitor.visit_generics(generics);
             visit_fn_sig(sig, visitor);
             visit_opt(body, |body| visitor.visit_block(body));
         }
-        ForeignItemKind::TyAlias(_, generics, bounds, ty) => {
+        ForeignItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
             visitor.visit_generics(generics);
             visit_bounds(bounds, visitor);
             visit_opt(ty, |ty| visitor.visit_ty(ty));

compiler/rustc_ast/src/token.rs

@@ -15,7 +15,7 @@ use rustc_span::hygiene::ExpnKind;
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::symbol::{Ident, Symbol};
-use rustc_span::{self, FileName, RealFileName, Span, DUMMY_SP};
+use rustc_span::{self, edition::Edition, FileName, RealFileName, Span, DUMMY_SP};
 use std::borrow::Cow;
 use std::{fmt, mem};
 
@@ -130,10 +130,7 @@ impl LitKind {
     }
 
     crate fn may_have_suffix(self) -> bool {
-        match self {
+        matches!(self, Integer | Float | Err)
-            Integer | Float | Err => true,
-            _ => false,
-        }
     }
 }
 
@@ -305,10 +302,7 @@ impl TokenKind {
     }
 
     pub fn should_end_const_arg(&self) -> bool {
-        match self {
+        matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr))
-            Gt | Ge | BinOp(Shr) | BinOpEq(Shr) => true,
-            _ => false,
-        }
     }
 }
 
@@ -346,18 +340,21 @@ impl Token {
     }
 
     pub fn is_op(&self) -> bool {
-        match self.kind {
+        !matches!(
-            OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
+            self.kind,
-            | Lifetime(..) | Interpolated(..) | Eof => false,
+            OpenDelim(..)
-            _ => true,
+                | CloseDelim(..)
-        }
+                | Literal(..)
+                | DocComment(..)
+                | Ident(..)
+                | Lifetime(..)
+                | Interpolated(..)
+                | Eof
+        )
     }
 
     pub fn is_like_plus(&self) -> bool {
-        match self.kind {
+        matches!(self.kind, BinOp(Plus) | BinOpEq(Plus))
-            BinOp(Plus) | BinOpEq(Plus) => true,
-            _ => false,
-        }
     }
 
     /// Returns `true` if the token can appear at the start of an expression.
@@ -379,13 +376,10 @@ impl Token {
             ModSep                            | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => match **nt {
+            Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
-                NtLiteral(..) |
                 NtExpr(..)    |
                 NtBlock(..)   |
-                NtPath(..) => true,
+                NtPath(..)),
-                _ => false,
-            },
             _ => false,
         }
     }
@@ -405,10 +399,7 @@ impl Token {
             Lifetime(..)                | // lifetime bound in trait object
             Lt | BinOp(Shl)             | // associated path
             ModSep                      => true, // global path
-            Interpolated(ref nt) => match **nt {
+            Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)),
-                NtTy(..) | NtPath(..) => true,
-                _ => false,
-            },
             _ => false,
         }
     }
@@ -417,10 +408,7 @@ impl Token {
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
             OpenDelim(Brace) => true,
-            Interpolated(ref nt) => match **nt {
+            Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
-                NtExpr(..) | NtBlock(..) | NtLiteral(..) => true,
-                _ => false,
-            },
             _ => self.can_begin_literal_maybe_minus(),
         }
     }
@@ -436,10 +424,7 @@ impl Token {
 
     /// Returns `true` if the token is any literal.
     pub fn is_lit(&self) -> bool {
-        match self.kind {
+        matches!(self.kind, Literal(..))
-            Literal(..) => true,
-            _ => false,
-        }
     }
 
     /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
@@ -705,7 +690,16 @@ pub enum NonterminalKind {
     Item,
     Block,
     Stmt,
-    Pat,
+    Pat2018 {
+        /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the
+        /// edition of the span. This is used for diagnostics.
+        inferred: bool,
+    },
+    Pat2021 {
+        /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the
+        /// edition of the span. This is used for diagnostics.
+        inferred: bool,
+    },
     Expr,
     Ty,
     Ident,
@@ -718,12 +712,24 @@ pub enum NonterminalKind {
 }
 
 impl NonterminalKind {
-    pub fn from_symbol(symbol: Symbol) -> Option<NonterminalKind> {
+    /// The `edition` closure is used to get the edition for the given symbol. Doing
+    /// `span.edition()` is expensive, so we do it lazily.
+    pub fn from_symbol(
+        symbol: Symbol,
+        edition: impl FnOnce() -> Edition,
+    ) -> Option<NonterminalKind> {
         Some(match symbol {
             sym::item => NonterminalKind::Item,
             sym::block => NonterminalKind::Block,
             sym::stmt => NonterminalKind::Stmt,
-            sym::pat => NonterminalKind::Pat,
+            sym::pat => match edition() {
+                Edition::Edition2015 | Edition::Edition2018 => {
+                    NonterminalKind::Pat2018 { inferred: true }
+                }
+                Edition::Edition2021 => NonterminalKind::Pat2021 { inferred: true },
+            },
+            sym::pat2018 => NonterminalKind::Pat2018 { inferred: false },
+            sym::pat2021 => NonterminalKind::Pat2021 { inferred: false },
             sym::expr => NonterminalKind::Expr,
             sym::ty => NonterminalKind::Ty,
             sym::ident => NonterminalKind::Ident,
@@ -741,7 +747,10 @@ impl NonterminalKind {
             NonterminalKind::Item => sym::item,
             NonterminalKind::Block => sym::block,
             NonterminalKind::Stmt => sym::stmt,
-            NonterminalKind::Pat => sym::pat,
+            NonterminalKind::Pat2018 { inferred: false } => sym::pat2018,
+            NonterminalKind::Pat2021 { inferred: false } => sym::pat2021,
+            NonterminalKind::Pat2018 { inferred: true }
+            | NonterminalKind::Pat2021 { inferred: true } => sym::pat,
             NonterminalKind::Expr => sym::expr,
             NonterminalKind::Ty => sym::ty,
             NonterminalKind::Ident => sym::ident,
@@ -762,7 +771,7 @@ impl fmt::Display for NonterminalKind {
 }
 
 impl Nonterminal {
-    fn span(&self) -> Span {
+    pub fn span(&self) -> Span {
         match self {
             NtItem(item) => item.span,
             NtBlock(block) => block.span,

compiler/rustc_ast/src/tokenstream.rs

@@ -1,15 +1,15 @@
 //! # Token Streams
 //!
 //! `TokenStream`s represent syntactic objects before they are converted into ASTs.
-//! A `TokenStream` is, roughly speaking, a sequence (eg stream) of `TokenTree`s,
+//! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s,
-//! which are themselves a single `Token` or a `Delimited` subsequence of tokens.
+//! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens.
 //!
 //! ## Ownership
 //!
 //! `TokenStream`s are persistent data structures constructed as ropes with reference
 //! counted-children. In general, this means that calling an operation on a `TokenStream`
 //! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to
-//! the original. This essentially coerces `TokenStream`s into 'views' of their subparts,
+//! the original. This essentially coerces `TokenStream`s into "views" of their subparts,
 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
 //! ownership of the original.
 
@@ -24,9 +24,9 @@ use smallvec::{smallvec, SmallVec};
 
 use std::{fmt, iter, mem};
 
-/// When the main rust parser encounters a syntax-extension invocation, it
+/// When the main Rust parser encounters a syntax-extension invocation, it
-/// parses the arguments to the invocation as a token-tree. This is a very
+/// parses the arguments to the invocation as a token tree. This is a very
-/// loose structure, such that all sorts of different AST-fragments can
+/// loose structure, such that all sorts of different AST fragments can
 /// be passed to syntax extensions using a uniform type.
 ///
 /// If the syntax extension is an MBE macro, it will attempt to match its
@@ -38,12 +38,18 @@ use std::{fmt, iter, mem};
 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
-    /// A single token
+    /// A single token.
     Token(Token),
-    /// A delimited sequence of token trees
+    /// A delimited sequence of token trees.
     Delimited(DelimSpan, DelimToken, TokenStream),
 }
 
+#[derive(Copy, Clone)]
+pub enum CanSynthesizeMissingTokens {
+    Yes,
+    No,
+}
+
 // Ensure all fields of `TokenTree` is `Send` and `Sync`.
 #[cfg(parallel_compiler)]
 fn _dummy()
@@ -56,7 +62,7 @@ where
 }
 
 impl TokenTree {
-    /// Checks if this TokenTree is equal to the other, regardless of span information.
+    /// Checks if this `TokenTree` is equal to the other, regardless of span information.
     pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
         match (self, other) {
             (TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind,
@@ -67,7 +73,7 @@ impl TokenTree {
         }
     }
 
-    /// Retrieves the TokenTree's span.
+    /// Retrieves the `TokenTree`'s span.
     pub fn span(&self) -> Span {
         match self {
             TokenTree::Token(token) => token.span,
@@ -121,20 +127,16 @@ where
 }
 
 pub trait CreateTokenStream: sync::Send + sync::Sync {
-    fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream>;
     fn create_token_stream(&self) -> TokenStream;
 }
 
 impl CreateTokenStream for TokenStream {
-    fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream> {
-        panic!("Cannot call `add_trailing_semi` on a `TokenStream`!");
-    }
     fn create_token_stream(&self) -> TokenStream {
         self.clone()
     }
 }
 
-/// A lazy version of `TokenStream`, which defers creation
+/// A lazy version of [`TokenStream`], which defers creation
 /// of an actual `TokenStream` until it is needed.
 /// `Box` is here only to reduce the structure size.
 #[derive(Clone)]
@@ -145,13 +147,6 @@ impl LazyTokenStream {
         LazyTokenStream(Lrc::new(Box::new(inner)))
     }
 
-    /// Extends the captured stream by one token,
-    /// which must be a trailing semicolon. This
-    /// affects the `TokenStream` created by `make_tokenstream`.
-    pub fn add_trailing_semi(&self) -> LazyTokenStream {
-        LazyTokenStream(Lrc::new(self.0.add_trailing_semi()))
-    }
-
     pub fn create_token_stream(&self) -> TokenStream {
         self.0.create_token_stream()
     }
@@ -182,11 +177,12 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
     }
 }
 
-/// A `TokenStream` is an abstract sequence of tokens, organized into `TokenTree`s.
+/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
 ///
 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
 /// instead of a representation of the abstract syntax tree.
-/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
+/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
+/// backwards compatability.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>);
 
@@ -423,7 +419,7 @@ impl TokenStreamBuilder {
     }
 }
 
-/// By-reference iterator over a `TokenStream`.
+/// By-reference iterator over a [`TokenStream`].
 #[derive(Clone)]
 pub struct CursorRef<'t> {
     stream: &'t TokenStream,
@@ -451,8 +447,8 @@ impl<'t> Iterator for CursorRef<'t> {
     }
 }
 
-/// Owning by-value iterator over a `TokenStream`.
+/// Owning by-value iterator over a [`TokenStream`].
-/// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
+// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
 #[derive(Clone)]
 pub struct Cursor {
     pub stream: TokenStream,

compiler/rustc_ast/src/util/classify.rs

@@ -12,14 +12,14 @@ use crate::ast;
 ///      |x| 5
 /// isn't parsed as (if true {...} else {...} | x) | 5
 pub fn expr_requires_semi_to_be_stmt(e: &ast::Expr) -> bool {
-    match e.kind {
+    !matches!(
+        e.kind,
         ast::ExprKind::If(..)
-        | ast::ExprKind::Match(..)
+            | ast::ExprKind::Match(..)
-        | ast::ExprKind::Block(..)
+            | ast::ExprKind::Block(..)
-        | ast::ExprKind::While(..)
+            | ast::ExprKind::While(..)
-        | ast::ExprKind::Loop(..)
+            | ast::ExprKind::Loop(..)
-        | ast::ExprKind::ForLoop(..)
+            | ast::ExprKind::ForLoop(..)
-        | ast::ExprKind::TryBlock(..) => false,
+            | ast::ExprKind::TryBlock(..)
-        _ => true,
+    )
-    }
 }

compiler/rustc_ast/src/util/comments.rs

@@ -180,10 +180,8 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec<Comme
             }
             rustc_lexer::TokenKind::BlockComment { doc_style, .. } => {
                 if doc_style.is_none() {
-                    let code_to_the_right = match text[pos + token.len..].chars().next() {
+                    let code_to_the_right =
-                        Some('\r' | '\n') => false,
+                        !matches!(text[pos + token.len..].chars().next(), Some('\r' | '\n'));
-                        _ => true,
-                    };
                     let style = match (code_to_the_left, code_to_the_right) {
                         (_, true) => CommentStyle::Mixed,
                         (false, false) => CommentStyle::Isolated,

compiler/rustc_ast/src/util/literal.rs

@@ -2,7 +2,6 @@
 
 use crate::ast::{self, Lit, LitKind};
 use crate::token::{self, Token};
-use crate::tokenstream::TokenTree;
 
 use rustc_lexer::unescape::{unescape_byte, unescape_char};
 use rustc_lexer::unescape::{unescape_byte_literal, unescape_literal, Mode};
@@ -88,7 +87,6 @@ impl LitKind {
                             }
                         });
                         error?;
-                        buf.shrink_to_fit();
                         Symbol::intern(&buf)
                     } else {
                         symbol
@@ -106,7 +104,6 @@ impl LitKind {
                     }
                 });
                 error?;
-                buf.shrink_to_fit();
                 LitKind::ByteStr(buf.into())
             }
             token::ByteStrRaw(_) => {
@@ -121,7 +118,6 @@ impl LitKind {
                         }
                     });
                     error?;
-                    buf.shrink_to_fit();
                     buf
                 } else {
                     symbol.to_string().into_bytes()
@@ -225,13 +221,13 @@ impl Lit {
         Lit { token: kind.to_lit_token(), kind, span }
     }
 
-    /// Losslessly convert an AST literal into a token stream.
+    /// Losslessly convert an AST literal into a token.
-    pub fn token_tree(&self) -> TokenTree {
+    pub fn to_token(&self) -> Token {
-        let token = match self.token.kind {
+        let kind = match self.token.kind {
             token::Bool => token::Ident(self.token.symbol, false),
             _ => token::Literal(self.token),
         };
-        TokenTree::token(token, self.span)
+        Token::new(kind, self.span)
     }
 }
 

compiler/rustc_ast/src/visit.rs

@@ -15,7 +15,6 @@
 
 use crate::ast::*;
 use crate::token;
-use crate::tokenstream::TokenTree;
 
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::Span;
@@ -293,7 +292,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
             visitor.visit_ty(typ);
             walk_list!(visitor, visit_expr, expr);
         }
-        ItemKind::Fn(_, ref sig, ref generics, ref body) => {
+        ItemKind::Fn(box FnKind(_, ref sig, ref generics, ref body)) => {
             visitor.visit_generics(generics);
             let kind = FnKind::Fn(FnCtxt::Free, item.ident, sig, &item.vis, body.as_deref());
             visitor.visit_fn(kind, item.span, item.id)
@@ -303,7 +302,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
             walk_list!(visitor, visit_foreign_item, &foreign_module.items);
         }
         ItemKind::GlobalAsm(ref ga) => visitor.visit_global_asm(ga),
-        ItemKind::TyAlias(_, ref generics, ref bounds, ref ty) => {
+        ItemKind::TyAlias(box TyAliasKind(_, ref generics, ref bounds, ref ty)) => {
             visitor.visit_generics(generics);
             walk_list!(visitor, visit_param_bound, bounds);
             walk_list!(visitor, visit_ty, ty);
@@ -312,7 +311,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
             visitor.visit_generics(generics);
             visitor.visit_enum_def(enum_definition, generics, item.id, item.span)
         }
-        ItemKind::Impl {
+        ItemKind::Impl(box ImplKind {
             unsafety: _,
             polarity: _,
             defaultness: _,
@@ -321,7 +320,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
             ref of_trait,
             ref self_ty,
             ref items,
-        } => {
+        }) => {
             visitor.visit_generics(generics);
             walk_list!(visitor, visit_trait_ref, of_trait);
             visitor.visit_ty(self_ty);
@@ -332,7 +331,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
             visitor.visit_generics(generics);
             visitor.visit_variant_data(struct_definition);
         }
-        ItemKind::Trait(.., ref generics, ref bounds, ref items) => {
+        ItemKind::Trait(box TraitKind(.., ref generics, ref bounds, ref items)) => {
             visitor.visit_generics(generics);
             walk_list!(visitor, visit_param_bound, bounds);
             walk_list!(visitor, visit_assoc_item, items, AssocCtxt::Trait);
@@ -544,12 +543,12 @@ pub fn walk_foreign_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a ForeignI
             visitor.visit_ty(ty);
             walk_list!(visitor, visit_expr, expr);
         }
-        ForeignItemKind::Fn(_, sig, generics, body) => {
+        ForeignItemKind::Fn(box FnKind(_, sig, generics, body)) => {
             visitor.visit_generics(generics);
             let kind = FnKind::Fn(FnCtxt::Foreign, ident, sig, vis, body.as_deref());
             visitor.visit_fn(kind, span, id);
         }
-        ForeignItemKind::TyAlias(_, generics, bounds, ty) => {
+        ForeignItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
             visitor.visit_generics(generics);
             walk_list!(visitor, visit_param_bound, bounds);
             walk_list!(visitor, visit_ty, ty);
@@ -578,7 +577,12 @@ pub fn walk_generic_param<'a, V: Visitor<'a>>(visitor: &mut V, param: &'a Generi
     match param.kind {
         GenericParamKind::Lifetime => (),
         GenericParamKind::Type { ref default } => walk_list!(visitor, visit_ty, default),
-        GenericParamKind::Const { ref ty, .. } => visitor.visit_ty(ty),
+        GenericParamKind::Const { ref ty, ref default, .. } => {
+            visitor.visit_ty(ty);
+            if let Some(default) = default {
+                visitor.visit_anon_const(default);
+            }
+        }
     }
 }
 
@@ -649,12 +653,12 @@ pub fn walk_assoc_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a AssocItem,
             visitor.visit_ty(ty);
             walk_list!(visitor, visit_expr, expr);
         }
-        AssocItemKind::Fn(_, sig, generics, body) => {
+        AssocItemKind::Fn(box FnKind(_, sig, generics, body)) => {
             visitor.visit_generics(generics);
             let kind = FnKind::Fn(FnCtxt::Assoc(ctxt), ident, sig, vis, body.as_deref());
             visitor.visit_fn(kind, span, id);
         }
-        AssocItemKind::TyAlias(_, generics, bounds, ty) => {
+        AssocItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
             visitor.visit_generics(generics);
             walk_list!(visitor, visit_param_bound, bounds);
             walk_list!(visitor, visit_ty, ty);
@@ -900,12 +904,9 @@ pub fn walk_mac_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a MacArgs) {
         MacArgs::Delimited(_dspan, _delim, _tokens) => {}
         // The value in `#[key = VALUE]` must be visited as an expression for backward
         // compatibility, so that macros can be expanded in that position.
-        MacArgs::Eq(_eq_span, tokens) => match tokens.trees_ref().next() {
+        MacArgs::Eq(_eq_span, token) => match &token.kind {
-            Some(TokenTree::Token(token)) => match &token.kind {
+            token::Interpolated(nt) => match &**nt {
-                token::Interpolated(nt) => match &**nt {
+                token::NtExpr(expr) => visitor.visit_expr(expr),
-                    token::NtExpr(expr) => visitor.visit_expr(expr),
-                    t => panic!("unexpected token in key-value attribute: {:?}", t),
-                },
                 t => panic!("unexpected token in key-value attribute: {:?}", t),
             },
             t => panic!("unexpected token in key-value attribute: {:?}", t),

compiler/rustc_ast_lowering/src/expr.rs

@@ -10,9 +10,9 @@ use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::Res;
 use rustc_session::parse::feature_err;
-use rustc_span::hygiene::ForLoopLoc;
 use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned};
 use rustc_span::symbol::{sym, Ident, Symbol};
+use rustc_span::{hygiene::ForLoopLoc, DUMMY_SP};
 use rustc_target::asm;
 use std::collections::hash_map::Entry;
 use std::fmt::Write;
@@ -87,9 +87,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ExprKind::Let(ref pat, ref scrutinee) => {
                     self.lower_expr_let(e.span, pat, scrutinee)
                 }
-                ExprKind::If(ref cond, ref then, ref else_opt) => {
+                ExprKind::If(ref cond, ref then, ref else_opt) => match cond.kind {
-                    self.lower_expr_if(e.span, cond, then, else_opt.as_deref())
+                    ExprKind::Let(ref pat, ref scrutinee) => {
-                }
+                        self.lower_expr_if_let(e.span, pat, scrutinee, then, else_opt.as_deref())
+                    }
+                    _ => self.lower_expr_if(cond, then, else_opt.as_deref()),
+                },
                 ExprKind::While(ref cond, ref body, opt_label) => self
                     .with_loop_scope(e.id, |this| {
                         this.lower_expr_while_in_loop_scope(e.span, cond, body, opt_label)
@@ -99,6 +102,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         this.lower_block(body, false),
                         opt_label,
                         hir::LoopSource::Loop,
+                        DUMMY_SP,
                     )
                 }),
                 ExprKind::TryBlock(ref body) => self.lower_expr_try_block(body),
@@ -337,10 +341,30 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
     fn lower_expr_if(
         &mut self,
-        span: Span,
         cond: &Expr,
         then: &Block,
         else_opt: Option<&Expr>,
+    ) -> hir::ExprKind<'hir> {
+        macro_rules! make_if {
+            ($opt:expr) => {{
+                let then_expr = self.lower_block_expr(then);
+                hir::ExprKind::If(self.lower_expr(cond), self.arena.alloc(then_expr), $opt)
+            }};
+        }
+        if let Some(rslt) = else_opt {
+            make_if!(Some(self.lower_expr(rslt)))
+        } else {
+            make_if!(None)
+        }
+    }
+
+    fn lower_expr_if_let(
+        &mut self,
+        span: Span,
+        pat: &Pat,
+        scrutinee: &Expr,
+        then: &Block,
+        else_opt: Option<&Expr>,
     ) -> hir::ExprKind<'hir> {
         // FIXME(#53667): handle lowering of && and parens.
 
@@ -353,30 +377,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let else_arm = self.arm(else_pat, else_expr);
 
         // Handle then + scrutinee:
-        let (then_pat, scrutinee, desugar) = match cond.kind {
+        let scrutinee = self.lower_expr(scrutinee);
-            // `<pat> => <then>`:
+        let then_pat = self.lower_pat(pat);
-            ExprKind::Let(ref pat, ref scrutinee) => {
+
-                let scrutinee = self.lower_expr(scrutinee);
-                let pat = self.lower_pat(pat);
-                (pat, scrutinee, hir::MatchSource::IfLetDesugar { contains_else_clause })
-            }
-            // `true => <then>`:
-            _ => {
-                // Lower condition:
-                let cond = self.lower_expr(cond);
-                let span_block =
-                    self.mark_span_with_reason(DesugaringKind::CondTemporary, cond.span, None);
-                // Wrap in a construct equivalent to `{ let _t = $cond; _t }`
-                // to preserve drop semantics since `if cond { ... }` does not
-                // let temporaries live outside of `cond`.
-                let cond = self.expr_drop_temps(span_block, cond, ThinVec::new());
-                let pat = self.pat_bool(span, true);
-                (pat, cond, hir::MatchSource::IfDesugar { contains_else_clause })
-            }
-        };
         let then_expr = self.lower_block_expr(then);
         let then_arm = self.arm(then_pat, self.arena.alloc(then_expr));
 
+        let desugar = hir::MatchSource::IfLetDesugar { contains_else_clause };
         hir::ExprKind::Match(scrutinee, arena_vec![self; then_arm, else_arm], desugar)
     }
 
@@ -447,7 +454,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.expr_match(span, scrutinee, arena_vec![self; then_arm, else_arm], desugar);
 
         // `[opt_ident]: loop { ... }`
-        hir::ExprKind::Loop(self.block_expr(self.arena.alloc(match_expr)), opt_label, source)
+        hir::ExprKind::Loop(
+            self.block_expr(self.arena.alloc(match_expr)),
+            opt_label,
+            source,
+            span.with_hi(cond.span.hi()),
+        )
     }
 
     /// Desugar `try { <stmts>; <expr> }` into `{ <stmts>; ::std::ops::Try::from_ok(<expr>) }`,
@@ -742,7 +754,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // loop { .. }
         let loop_expr = self.arena.alloc(hir::Expr {
             hir_id: loop_hir_id,
-            kind: hir::ExprKind::Loop(loop_block, None, hir::LoopSource::Loop),
+            kind: hir::ExprKind::Loop(loop_block, None, hir::LoopSource::Loop, span),
             span,
             attrs: ThinVec::new(),
         });
@@ -764,10 +776,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         body: &Expr,
         fn_decl_span: Span,
     ) -> hir::ExprKind<'hir> {
-        // Lower outside new scope to preserve `is_in_loop_condition`.
+        let (body_id, generator_option) = self.with_new_scopes(move |this| {
-        let fn_decl = self.lower_fn_decl(decl, None, false, None);
-
-        self.with_new_scopes(move |this| {
             let prev = this.current_item;
             this.current_item = Some(fn_decl_span);
             let mut generator_kind = None;
@@ -779,8 +788,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let generator_option =
                 this.generator_movability_for_fn(&decl, fn_decl_span, generator_kind, movability);
             this.current_item = prev;
-            hir::ExprKind::Closure(capture_clause, fn_decl, body_id, fn_decl_span, generator_option)
+            (body_id, generator_option)
-        })
+        });
+
+        // Lower outside new scope to preserve `is_in_loop_condition`.
+        let fn_decl = self.lower_fn_decl(decl, None, false, None);
+
+        hir::ExprKind::Closure(capture_clause, fn_decl, body_id, fn_decl_span, generator_option)
     }
 
     fn generator_movability_for_fn(
@@ -826,12 +840,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
     ) -> hir::ExprKind<'hir> {
         let outer_decl =
             FnDecl { inputs: decl.inputs.clone(), output: FnRetTy::Default(fn_decl_span) };
-        // We need to lower the declaration outside the new scope, because we
-        // have to conserve the state of being inside a loop condition for the
-        // closure argument types.
-        let fn_decl = self.lower_fn_decl(&outer_decl, None, false, None);
 
-        self.with_new_scopes(move |this| {
+        let body_id = self.with_new_scopes(|this| {
             // FIXME(cramertj): allow `async` non-`move` closures with arguments.
             if capture_clause == CaptureBy::Ref && !decl.inputs.is_empty() {
                 struct_span_err!(
@@ -862,8 +872,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 );
                 this.expr(fn_decl_span, async_body, ThinVec::new())
             });
-            hir::ExprKind::Closure(capture_clause, fn_decl, body_id, fn_decl_span, None)
+            body_id
-        })
+        });
+
+        // We need to lower the declaration outside the new scope, because we
+        // have to conserve the state of being inside a loop condition for the
+        // closure argument types.
+        let fn_decl = self.lower_fn_decl(&outer_decl, None, false, None);
+
+        hir::ExprKind::Closure(capture_clause, fn_decl, body_id, fn_decl_span, None)
     }
 
     /// Destructure the LHS of complex assignments.
@@ -1703,7 +1720,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         );
 
         // `[opt_ident]: loop { ... }`
-        let kind = hir::ExprKind::Loop(loop_block, opt_label, hir::LoopSource::ForLoop);
+        let kind = hir::ExprKind::Loop(
+            loop_block,
+            opt_label,
+            hir::LoopSource::ForLoop,
+            e.span.with_hi(orig_head_span.hi()),
+        );
         let loop_expr = self.arena.alloc(hir::Expr {
             hir_id: self.lower_node_id(e.id),
             kind,

compiler/rustc_ast_lowering/src/item.rs

@@ -67,7 +67,7 @@ impl<'a> Visitor<'a> for ItemLowerer<'a, '_, '_> {
         if let Some(hir_id) = item_hir_id {
             self.lctx.with_parent_item_lifetime_defs(hir_id, |this| {
                 let this = &mut ItemLowerer { lctx: this };
-                if let ItemKind::Impl { ref of_trait, .. } = item.kind {
+                if let ItemKind::Impl(box ImplKind { ref of_trait, .. }) = item.kind {
                     this.with_trait_impl_ref(of_trait, |this| visit::walk_item(this, item));
                 } else {
                     visit::walk_item(this, item);
@@ -134,7 +134,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let old_len = self.in_scope_lifetimes.len();
 
         let parent_generics = match self.items.get(&parent_hir_id).unwrap().kind {
-            hir::ItemKind::Impl { ref generics, .. }
+            hir::ItemKind::Impl(hir::Impl { ref generics, .. })
             | hir::ItemKind::Trait(_, _, ref generics, ..) => &generics.params[..],
             _ => &[],
         };
@@ -189,7 +189,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 vec
             }
             ItemKind::MacroDef(..) => SmallVec::new(),
-            ItemKind::Fn(..) | ItemKind::Impl { of_trait: None, .. } => smallvec![i.id],
+            ItemKind::Fn(..) | ItemKind::Impl(box ImplKind { of_trait: None, .. }) => {
+                smallvec![i.id]
+            }
             _ => smallvec![i.id],
         };
 
@@ -276,12 +278,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let (ty, body_id) = self.lower_const_item(t, span, e.as_deref());
                 hir::ItemKind::Const(ty, body_id)
             }
-            ItemKind::Fn(
+            ItemKind::Fn(box FnKind(
                 _,
                 FnSig { ref decl, header, span: fn_sig_span },
                 ref generics,
                 ref body,
-            ) => {
+            )) => {
                 let fn_def_id = self.resolver.local_def_id(id);
                 self.with_new_scopes(|this| {
                     this.current_item = Some(ident.span);
@@ -310,21 +312,26 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     );
                     let sig = hir::FnSig {
                         decl,
-                        header: this.lower_fn_header(header),
+                        header: this.lower_fn_header(header, fn_sig_span, id),
                         span: fn_sig_span,
                     };
                     hir::ItemKind::Fn(sig, generics, body_id)
                 })
             }
             ItemKind::Mod(ref m) => hir::ItemKind::Mod(self.lower_mod(m)),
-            ItemKind::ForeignMod(ref fm) => hir::ItemKind::ForeignMod {
+            ItemKind::ForeignMod(ref fm) => {
-                abi: fm.abi.map_or(abi::Abi::C, |abi| self.lower_abi(abi)),
+                if fm.abi.is_none() {
-                items: self
+                    self.maybe_lint_missing_abi(span, id, abi::Abi::C);
-                    .arena
+                }
-                    .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
+                hir::ItemKind::ForeignMod {
-            },
+                    abi: fm.abi.map_or(abi::Abi::C, |abi| self.lower_abi(abi)),
+                    items: self
+                        .arena
+                        .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
+                }
+            }
             ItemKind::GlobalAsm(ref ga) => hir::ItemKind::GlobalAsm(self.lower_global_asm(ga)),
-            ItemKind::TyAlias(_, ref gen, _, Some(ref ty)) => {
+            ItemKind::TyAlias(box TyAliasKind(_, ref gen, _, Some(ref ty))) => {
                 // We lower
                 //
                 // type Foo = impl Trait
@@ -343,7 +350,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let generics = self.lower_generics(gen, ImplTraitContext::disallowed());
                 hir::ItemKind::TyAlias(ty, generics)
             }
-            ItemKind::TyAlias(_, ref generics, _, None) => {
+            ItemKind::TyAlias(box TyAliasKind(_, ref generics, _, None)) => {
                 let ty = self.arena.alloc(self.ty(span, hir::TyKind::Err));
                 let generics = self.lower_generics(generics, ImplTraitContext::disallowed());
                 hir::ItemKind::TyAlias(ty, generics)
@@ -370,7 +377,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     self.lower_generics(generics, ImplTraitContext::disallowed()),
                 )
             }
-            ItemKind::Impl {
+            ItemKind::Impl(box ImplKind {
                 unsafety,
                 polarity,
                 defaultness,
@@ -379,7 +386,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 of_trait: ref trait_ref,
                 self_ty: ref ty,
                 items: ref impl_items,
-            } => {
+            }) => {
                 let def_id = self.resolver.local_def_id(id);
 
                 // Lower the "impl header" first. This ordering is important
@@ -431,7 +438,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // to not cause an assertion failure inside the `lower_defaultness` function.
                 let has_val = true;
                 let (defaultness, defaultness_span) = self.lower_defaultness(defaultness, has_val);
-                hir::ItemKind::Impl {
+                hir::ItemKind::Impl(hir::Impl {
                     unsafety: self.lower_unsafety(unsafety),
                     polarity,
                     defaultness,
@@ -441,9 +448,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     of_trait: trait_ref,
                     self_ty: lowered_ty,
                     items: new_impl_items,
-                }
+                })
             }
-            ItemKind::Trait(is_auto, unsafety, ref generics, ref bounds, ref items) => {
+            ItemKind::Trait(box TraitKind(
+                is_auto,
+                unsafety,
+                ref generics,
+                ref bounds,
+                ref items,
+            )) => {
                 let bounds = self.lower_param_bounds(bounds, ImplTraitContext::disallowed());
                 let items = self
                     .arena
@@ -693,7 +706,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             ident: i.ident,
             attrs: self.lower_attrs(&i.attrs),
             kind: match i.kind {
-                ForeignItemKind::Fn(_, ref sig, ref generics, _) => {
+                ForeignItemKind::Fn(box FnKind(_, ref sig, ref generics, _)) => {
                     let fdec = &sig.decl;
                     let (generics, (fn_dec, fn_args)) = self.add_in_band_defs(
                         generics,
@@ -798,19 +811,19 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let body = default.as_ref().map(|x| self.lower_const_body(i.span, Some(x)));
                 (hir::Generics::empty(), hir::TraitItemKind::Const(ty, body))
             }
-            AssocItemKind::Fn(_, ref sig, ref generics, None) => {
+            AssocItemKind::Fn(box FnKind(_, ref sig, ref generics, None)) => {
                 let names = self.lower_fn_params_to_names(&sig.decl);
                 let (generics, sig) =
-                    self.lower_method_sig(generics, sig, trait_item_def_id, false, None);
+                    self.lower_method_sig(generics, sig, trait_item_def_id, false, None, i.id);
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)))
             }
-            AssocItemKind::Fn(_, ref sig, ref generics, Some(ref body)) => {
+            AssocItemKind::Fn(box FnKind(_, ref sig, ref generics, Some(ref body))) => {
                 let body_id = self.lower_fn_body_block(i.span, &sig.decl, Some(body));
                 let (generics, sig) =
-                    self.lower_method_sig(generics, sig, trait_item_def_id, false, None);
+                    self.lower_method_sig(generics, sig, trait_item_def_id, false, None, i.id);
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)))
             }
-            AssocItemKind::TyAlias(_, ref generics, ref bounds, ref default) => {
+            AssocItemKind::TyAlias(box TyAliasKind(_, ref generics, ref bounds, ref default)) => {
                 let ty = default.as_ref().map(|x| self.lower_ty(x, ImplTraitContext::disallowed()));
                 let generics = self.lower_generics(generics, ImplTraitContext::disallowed());
                 let kind = hir::TraitItemKind::Type(
@@ -836,10 +849,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_trait_item_ref(&mut self, i: &AssocItem) -> hir::TraitItemRef {
         let (kind, has_default) = match &i.kind {
             AssocItemKind::Const(_, _, default) => (hir::AssocItemKind::Const, default.is_some()),
-            AssocItemKind::TyAlias(_, _, _, default) => {
+            AssocItemKind::TyAlias(box TyAliasKind(_, _, _, default)) => {
                 (hir::AssocItemKind::Type, default.is_some())
             }
-            AssocItemKind::Fn(_, sig, _, default) => {
+            AssocItemKind::Fn(box FnKind(_, sig, _, default)) => {
                 (hir::AssocItemKind::Fn { has_self: sig.decl.has_self() }, default.is_some())
             }
             AssocItemKind::MacCall(..) => unimplemented!(),
@@ -865,7 +878,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     hir::ImplItemKind::Const(ty, self.lower_const_body(i.span, expr.as_deref())),
                 )
             }
-            AssocItemKind::Fn(_, sig, generics, body) => {
+            AssocItemKind::Fn(box FnKind(_, sig, generics, body)) => {
                 self.current_item = Some(i.span);
                 let asyncness = sig.header.asyncness;
                 let body_id =
@@ -877,11 +890,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     impl_item_def_id,
                     impl_trait_return_allow,
                     asyncness.opt_return_id(),
+                    i.id,
                 );
 
                 (generics, hir::ImplItemKind::Fn(sig, body_id))
             }
-            AssocItemKind::TyAlias(_, generics, _, ty) => {
+            AssocItemKind::TyAlias(box TyAliasKind(_, generics, _, ty)) => {
                 let generics = self.lower_generics(generics, ImplTraitContext::disallowed());
                 let kind = match ty {
                     None => {
@@ -932,7 +946,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             kind: match &i.kind {
                 AssocItemKind::Const(..) => hir::AssocItemKind::Const,
                 AssocItemKind::TyAlias(..) => hir::AssocItemKind::Type,
-                AssocItemKind::Fn(_, sig, ..) => {
+                AssocItemKind::Fn(box FnKind(_, sig, ..)) => {
                     hir::AssocItemKind::Fn { has_self: sig.decl.has_self() }
                 }
                 AssocItemKind::MacCall(..) => unimplemented!(),
@@ -1270,8 +1284,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
         fn_def_id: LocalDefId,
         impl_trait_return_allow: bool,
         is_async: Option<NodeId>,
+        id: NodeId,
     ) -> (hir::Generics<'hir>, hir::FnSig<'hir>) {
-        let header = self.lower_fn_header(sig.header);
+        let header = self.lower_fn_header(sig.header, sig.span, id);
         let (generics, decl) = self.add_in_band_defs(
             generics,
             fn_def_id,
@@ -1288,12 +1303,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         (generics, hir::FnSig { header, decl, span: sig.span })
     }
 
-    fn lower_fn_header(&mut self, h: FnHeader) -> hir::FnHeader {
+    fn lower_fn_header(&mut self, h: FnHeader, span: Span, id: NodeId) -> hir::FnHeader {
         hir::FnHeader {
             unsafety: self.lower_unsafety(h.unsafety),
             asyncness: self.lower_asyncness(h.asyncness),
             constness: self.lower_constness(h.constness),
-            abi: self.lower_extern(h.ext),
+            abi: self.lower_extern(h.ext, span, id),
         }
     }
 
@@ -1304,10 +1319,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         })
     }
 
-    pub(super) fn lower_extern(&mut self, ext: Extern) -> abi::Abi {
+    pub(super) fn lower_extern(&mut self, ext: Extern, span: Span, id: NodeId) -> abi::Abi {
         match ext {
             Extern::None => abi::Abi::Rust,
-            Extern::Implicit => abi::Abi::C,
+            Extern::Implicit => {
+                self.maybe_lint_missing_abi(span, id, abi::Abi::C);
+                abi::Abi::C
+            }
             Extern::Explicit(abi) => self.lower_abi(abi),
         }
     }

compiler/rustc_ast_lowering/src/lib.rs

@@ -30,14 +30,14 @@
 //! get confused if the spans from leaf AST nodes occur in multiple places
 //! in the HIR, especially for multiple identifiers.
 
-#![feature(array_value_iter)]
 #![feature(crate_visibility_modifier)]
 #![feature(or_patterns)]
+#![feature(box_patterns)]
 #![recursion_limit = "256"]
 
 use rustc_ast::node_id::NodeMap;
 use rustc_ast::token::{self, DelimToken, Nonterminal, Token};
-use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, DelimSpan, TokenStream, TokenTree};
 use rustc_ast::visit::{self, AssocCtxt, Visitor};
 use rustc_ast::walk_list;
 use rustc_ast::{self as ast, *};
@@ -53,13 +53,15 @@ use rustc_hir::definitions::{DefKey, DefPathData, Definitions};
 use rustc_hir::intravisit;
 use rustc_hir::{ConstArg, GenericArg, ParamName};
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_session::lint::{builtin::BARE_TRAIT_OBJECTS, BuiltinLintDiagnostics, LintBuffer};
+use rustc_session::lint::builtin::{BARE_TRAIT_OBJECTS, MISSING_ABI};
+use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer};
 use rustc_session::parse::ParseSess;
 use rustc_session::Session;
 use rustc_span::hygiene::ExpnId;
-use rustc_span::source_map::{respan, DesugaringKind, ExpnData, ExpnKind};
+use rustc_span::source_map::{respan, DesugaringKind};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
+use rustc_target::spec::abi::Abi;
 
 use smallvec::{smallvec, SmallVec};
 use std::collections::BTreeMap;
@@ -206,7 +208,7 @@ pub trait ResolverAstLowering {
     ) -> LocalDefId;
 }
 
-type NtToTokenstream = fn(&Nonterminal, &ParseSess, Span) -> TokenStream;
+type NtToTokenstream = fn(&Nonterminal, &ParseSess, CanSynthesizeMissingTokens) -> TokenStream;
 
 /// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
 /// and if so, what meaning it has.
@@ -393,6 +395,42 @@ enum AnonymousLifetimeMode {
     PassThrough,
 }
 
+struct TokenStreamLowering<'a> {
+    parse_sess: &'a ParseSess,
+    synthesize_tokens: CanSynthesizeMissingTokens,
+    nt_to_tokenstream: NtToTokenstream,
+}
+
+impl<'a> TokenStreamLowering<'a> {
+    fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
+        tokens.into_trees().flat_map(|tree| self.lower_token_tree(tree).into_trees()).collect()
+    }
+
+    fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
+        match tree {
+            TokenTree::Token(token) => self.lower_token(token),
+            TokenTree::Delimited(span, delim, tts) => {
+                TokenTree::Delimited(span, delim, self.lower_token_stream(tts)).into()
+            }
+        }
+    }
+
+    fn lower_token(&mut self, token: Token) -> TokenStream {
+        match token.kind {
+            token::Interpolated(nt) => {
+                let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
+                TokenTree::Delimited(
+                    DelimSpan::from_single(token.span),
+                    DelimToken::NoDelim,
+                    self.lower_token_stream(tts),
+                )
+                .into()
+            }
+            _ => TokenTree::Token(token).into(),
+        }
+    }
+}
+
 struct ImplTraitTypeIdVisitor<'a> {
     ids: &'a mut SmallVec<[NodeId; 1]>,
 }
@@ -463,13 +501,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     ItemKind::Struct(_, ref generics)
                     | ItemKind::Union(_, ref generics)
                     | ItemKind::Enum(_, ref generics)
-                    | ItemKind::TyAlias(_, ref generics, ..)
+                    | ItemKind::TyAlias(box TyAliasKind(_, ref generics, ..))
-                    | ItemKind::Trait(_, _, ref generics, ..) => {
+                    | ItemKind::Trait(box TraitKind(_, _, ref generics, ..)) => {
                         let def_id = self.lctx.resolver.local_def_id(item.id);
                         let count = generics
                             .params
                             .iter()
-                            .filter(|param| matches!(param.kind, ast::GenericParamKind::Lifetime { .. }))
+                            .filter(|param| {
+                                matches!(param.kind, ast::GenericParamKind::Lifetime { .. })
+                            })
                             .count();
                         self.lctx.type_def_lifetime_params.insert(def_id.to_def_id(), count);
                     }
@@ -701,10 +741,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         span: Span,
         allow_internal_unstable: Option<Lrc<[Symbol]>>,
     ) -> Span {
-        span.fresh_expansion(ExpnData {
+        span.mark_with_reason(allow_internal_unstable, reason, self.sess.edition())
-            allow_internal_unstable,
-            ..ExpnData::default(ExpnKind::Desugaring(reason), span, self.sess.edition(), None)
-        })
     }
 
     fn with_anonymous_lifetime_mode<R>(
@@ -955,42 +992,77 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         match *args {
             MacArgs::Empty => MacArgs::Empty,
             MacArgs::Delimited(dspan, delim, ref tokens) => {
-                MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone()))
+                // This is either a non-key-value attribute, or a `macro_rules!` body.
-            }
+                // We either not have any nonterminals present (in the case of an attribute),
-            MacArgs::Eq(eq_span, ref tokens) => {
+                // or have tokens available for all nonterminals in the case of a nested
-                MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone()))
+                // `macro_rules`: e.g:
-            }
+                //
-        }
+                // ```rust
-    }
+                // macro_rules! outer {
-
+                //     ($e:expr) => {
-    fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
+                //         macro_rules! inner {
-        tokens.into_trees().flat_map(|tree| self.lower_token_tree(tree).into_trees()).collect()
+                //             () => { $e }
-    }
+                //         }
-
+                //     }
-    fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
+                // }
-        match tree {
+                // ```
-            TokenTree::Token(token) => self.lower_token(token),
+                //
-            TokenTree::Delimited(span, delim, tts) => {
+                // In both cases, we don't want to synthesize any tokens
-                TokenTree::Delimited(span, delim, self.lower_token_stream(tts)).into()
+                MacArgs::Delimited(
-            }
+                    dspan,
-        }
+                    delim,
-    }
+                    self.lower_token_stream(tokens.clone(), CanSynthesizeMissingTokens::No),
-
-    fn lower_token(&mut self, token: Token) -> TokenStream {
-        match token.kind {
-            token::Interpolated(nt) => {
-                let tts = (self.nt_to_tokenstream)(&nt, &self.sess.parse_sess, token.span);
-                TokenTree::Delimited(
-                    DelimSpan::from_single(token.span),
-                    DelimToken::NoDelim,
-                    self.lower_token_stream(tts),
                 )
-                .into()
             }
-            _ => TokenTree::Token(token).into(),
+            // This is an inert key-value attribute - it will never be visible to macros
+            // after it gets lowered to HIR. Therefore, we can synthesize tokens with fake
+            // spans to handle nonterminals in `#[doc]` (e.g. `#[doc = $e]`).
+            MacArgs::Eq(eq_span, ref token) => {
+                // In valid code the value is always representable as a single literal token.
+                fn unwrap_single_token(sess: &Session, tokens: TokenStream, span: Span) -> Token {
+                    if tokens.len() != 1 {
+                        sess.diagnostic()
+                            .delay_span_bug(span, "multiple tokens in key-value attribute's value");
+                    }
+                    match tokens.into_trees().next() {
+                        Some(TokenTree::Token(token)) => token,
+                        Some(TokenTree::Delimited(_, delim, tokens)) => {
+                            if delim != token::NoDelim {
+                                sess.diagnostic().delay_span_bug(
+                                    span,
+                                    "unexpected delimiter in key-value attribute's value",
+                                )
+                            }
+                            unwrap_single_token(sess, tokens, span)
+                        }
+                        None => Token::dummy(),
+                    }
+                }
+
+                let tokens = TokenStreamLowering {
+                    parse_sess: &self.sess.parse_sess,
+                    synthesize_tokens: CanSynthesizeMissingTokens::Yes,
+                    nt_to_tokenstream: self.nt_to_tokenstream,
+                }
+                .lower_token(token.clone());
+                MacArgs::Eq(eq_span, unwrap_single_token(self.sess, tokens, token.span))
+            }
         }
     }
 
+    fn lower_token_stream(
+        &self,
+        tokens: TokenStream,
+        synthesize_tokens: CanSynthesizeMissingTokens,
+    ) -> TokenStream {
+        TokenStreamLowering {
+            parse_sess: &self.sess.parse_sess,
+            synthesize_tokens,
+            nt_to_tokenstream: self.nt_to_tokenstream,
+        }
+        .lower_token_stream(tokens)
+    }
+
     /// Given an associated type constraint like one of these:
     ///
     /// ```
@@ -1004,16 +1076,40 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn lower_assoc_ty_constraint(
         &mut self,
         constraint: &AssocTyConstraint,
-        itctx: ImplTraitContext<'_, 'hir>,
+        mut itctx: ImplTraitContext<'_, 'hir>,
     ) -> hir::TypeBinding<'hir> {
         debug!("lower_assoc_ty_constraint(constraint={:?}, itctx={:?})", constraint, itctx);
 
-        if let Some(ref gen_args) = constraint.gen_args {
+        // lower generic arguments of identifier in constraint
-            self.sess.span_fatal(
+        let gen_args = if let Some(ref gen_args) = constraint.gen_args {
-                gen_args.span(),
+            let gen_args_ctor = match gen_args {
-                "generic associated types in trait paths are currently not implemented",
+                GenericArgs::AngleBracketed(ref data) => {
-            );
+                    self.lower_angle_bracketed_parameter_data(
-        }
+                        data,
+                        ParamMode::Explicit,
+                        itctx.reborrow(),
+                    )
+                    .0
+                }
+                GenericArgs::Parenthesized(ref data) => {
+                    let mut err = self.sess.struct_span_err(
+                        gen_args.span(),
+                        "parenthesized generic arguments cannot be used in associated type constraints"
+                    );
+                    // FIXME: try to write a suggestion here
+                    err.emit();
+                    self.lower_angle_bracketed_parameter_data(
+                        &data.as_angle_bracketed_args(),
+                        ParamMode::Explicit,
+                        itctx.reborrow(),
+                    )
+                    .0
+                }
+            };
+            self.arena.alloc(gen_args_ctor.into_generic_args(&self.arena))
+        } else {
+            self.arena.alloc(hir::GenericArgs::none())
+        };
 
         let kind = match constraint.kind {
             AssocTyConstraintKind::Equality { ref ty } => {
@@ -1110,6 +1206,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         hir::TypeBinding {
             hir_id: self.lower_node_id(constraint.id),
             ident: constraint.ident,
+            gen_args,
             kind,
             span: constraint.span,
         }
@@ -1220,6 +1317,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
             TyKind::BareFn(ref f) => self.with_in_scope_lifetime_defs(&f.generic_params, |this| {
                 this.with_anonymous_lifetime_mode(AnonymousLifetimeMode::PassThrough, |this| {
+                    let span = this.sess.source_map().next_point(t.span.shrink_to_lo());
                     hir::TyKind::BareFn(this.arena.alloc(hir::BareFnTy {
                         generic_params: this.lower_generic_params(
                             &f.generic_params,
@@ -1227,7 +1325,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             ImplTraitContext::disallowed(),
                         ),
                         unsafety: this.lower_unsafety(f.unsafety),
-                        abi: this.lower_extern(f.ext),
+                        abi: this.lower_extern(f.ext, span, t.id),
                         decl: this.lower_fn_decl(&f.decl, None, false, None),
                         param_names: this.lower_fn_params_to_names(&f.decl),
                     }))
@@ -1716,7 +1814,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         }
         self.arena.alloc_from_iter(inputs.iter().map(|param| match param.pat.kind {
             PatKind::Ident(_, ident, _) => ident,
-            _ => Ident::new(kw::Invalid, param.pat.span),
+            _ => Ident::new(kw::Empty, param.pat.span),
         }))
     }
 
@@ -1806,12 +1904,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             output,
             c_variadic,
             implicit_self: decl.inputs.get(0).map_or(hir::ImplicitSelfKind::None, |arg| {
-                let is_mutable_pat = match arg.pat.kind {
+                use BindingMode::{ByRef, ByValue};
-                    PatKind::Ident(BindingMode::ByValue(mt) | BindingMode::ByRef(mt), _, _) => {
+                let is_mutable_pat = matches!(
-                        mt == Mutability::Mut
+                    arg.pat.kind,
-                    }
+                    PatKind::Ident(ByValue(Mutability::Mut) | ByRef(Mutability::Mut), ..)
-                    _ => false,
+                );
-                };
 
                 match arg.ty.kind {
                     TyKind::ImplicitSelf if is_mutable_pat => hir::ImplicitSelfKind::Mut,
@@ -2192,13 +2289,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                 (hir::ParamName::Plain(param.ident), kind)
             }
-            GenericParamKind::Const { ref ty, kw_span: _ } => {
+            GenericParamKind::Const { ref ty, kw_span: _, ref default } => {
                 let ty = self
                     .with_anonymous_lifetime_mode(AnonymousLifetimeMode::ReportError, |this| {
                         this.lower_ty(&ty, ImplTraitContext::disallowed())
                     });
+                let default = default.as_ref().map(|def| self.lower_anon_const(def));
 
-                (hir::ParamName::Plain(param.ident), hir::GenericParamKind::Const { ty })
+                (hir::ParamName::Plain(param.ident), hir::GenericParamKind::Const { ty, default })
             }
         };
 
@@ -2709,6 +2807,26 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             )
         }
     }
+
+    fn maybe_lint_missing_abi(&mut self, span: Span, id: NodeId, default: Abi) {
+        // FIXME(davidtwco): This is a hack to detect macros which produce spans of the
+        // call site which do not have a macro backtrace. See #61963.
+        let is_macro_callsite = self
+            .sess
+            .source_map()
+            .span_to_snippet(span)
+            .map(|snippet| snippet.starts_with("#["))
+            .unwrap_or(true);
+        if !is_macro_callsite {
+            self.resolver.lint_buffer().buffer_lint_with_diagnostic(
+                MISSING_ABI,
+                id,
+                span,
+                "extern declarations without an explicit ABI are deprecated",
+                BuiltinLintDiagnostics::MissingAbi(span, default),
+            )
+        }
+    }
 }
 
 fn body_ids(bodies: &BTreeMap<hir::BodyId, hir::Body<'_>>) -> Vec<hir::BodyId> {

compiler/rustc_ast_lowering/src/path.rs

@@ -273,7 +273,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         if !generic_args.parenthesized && !has_lifetimes {
             generic_args.args = self
                 .elided_path_lifetimes(
-                    first_generic_span.map(|s| s.shrink_to_lo()).unwrap_or(segment.ident.span),
+                    first_generic_span.map_or(segment.ident.span, |s| s.shrink_to_lo()),
                     expected_lifetimes,
                 )
                 .map(GenericArg::Lifetime)
@@ -362,7 +362,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         }
     }
 
-    fn lower_angle_bracketed_parameter_data(
+    pub(crate) fn lower_angle_bracketed_parameter_data(
         &mut self,
         data: &AngleBracketedArgs,
         param_mode: ParamMode,
@@ -401,15 +401,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         // compatibility, even in contexts like an impl header where
         // we generally don't permit such things (see #51008).
         self.with_anonymous_lifetime_mode(AnonymousLifetimeMode::PassThrough, |this| {
-            let &ParenthesizedArgs { ref inputs, ref output, span } = data;
+            let ParenthesizedArgs { span, inputs, inputs_span, output } = data;
             let inputs = this.arena.alloc_from_iter(
                 inputs.iter().map(|ty| this.lower_ty_direct(ty, ImplTraitContext::disallowed())),
             );
             let output_ty = match output {
                 FnRetTy::Ty(ty) => this.lower_ty(&ty, ImplTraitContext::disallowed()),
-                FnRetTy::Default(_) => this.arena.alloc(this.ty_tup(span, &[])),
+                FnRetTy::Default(_) => this.arena.alloc(this.ty_tup(*span, &[])),
             };
-            let args = smallvec![GenericArg::Type(this.ty_tup(span, inputs))];
+            let args = smallvec![GenericArg::Type(this.ty_tup(*inputs_span, inputs))];
             let binding = this.output_ty_binding(output_ty.span, output_ty);
             (
                 GenericArgsCtor { args, bindings: arena_vec![this; binding], parenthesized: true },
@@ -426,6 +426,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     ) -> hir::TypeBinding<'hir> {
         let ident = Ident::with_dummy_span(hir::FN_OUTPUT_NAME);
         let kind = hir::TypeBindingKind::Equality { ty };
-        hir::TypeBinding { hir_id: self.next_id(), span, ident, kind }
+        let args = arena_vec![self;];
+        let bindings = arena_vec![self;];
+        let gen_args = self.arena.alloc(hir::GenericArgs { args, bindings, parenthesized: false });
+        hir::TypeBinding { hir_id: self.next_id(), gen_args, span, ident, kind }
     }
 }

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -16,7 +16,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{error_code, pluralize, struct_span_err, Applicability};
 use rustc_parse::validate_attr;
 use rustc_session::lint::builtin::PATTERNS_IN_FNS_WITHOUT_BODY;
-use rustc_session::lint::LintBuffer;
+use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer};
 use rustc_session::Session;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Span;
@@ -184,7 +184,7 @@ impl<'a> AstValidator<'a> {
     }
 
     fn check_lifetime(&self, ident: Ident) {
-        let valid_names = [kw::UnderscoreLifetime, kw::StaticLifetime, kw::Invalid];
+        let valid_names = [kw::UnderscoreLifetime, kw::StaticLifetime, kw::Empty];
         if !valid_names.contains(&ident.name) && ident.without_first_quote().is_reserved() {
             self.err_handler().span_err(ident.span, "lifetimes cannot use keyword names");
         }
@@ -213,14 +213,14 @@ impl<'a> AstValidator<'a> {
         err.emit();
     }
 
-    fn check_decl_no_pat(decl: &FnDecl, mut report_err: impl FnMut(Span, bool)) {
+    fn check_decl_no_pat(decl: &FnDecl, mut report_err: impl FnMut(Span, Option<Ident>, bool)) {
         for Param { pat, .. } in &decl.inputs {
             match pat.kind {
                 PatKind::Ident(BindingMode::ByValue(Mutability::Not), _, None) | PatKind::Wild => {}
-                PatKind::Ident(BindingMode::ByValue(Mutability::Mut), _, None) => {
+                PatKind::Ident(BindingMode::ByValue(Mutability::Mut), ident, None) => {
-                    report_err(pat.span, true)
+                    report_err(pat.span, Some(ident), true)
                 }
-                _ => report_err(pat.span, false),
+                _ => report_err(pat.span, None, false),
             }
         }
     }
@@ -717,35 +717,46 @@ impl<'a> AstValidator<'a> {
 
 /// Checks that generic parameters are in the correct order,
 /// which is lifetimes, then types and then consts. (`<'a, T, const N: usize>`)
-fn validate_generic_param_order<'a>(
+fn validate_generic_param_order(
     sess: &Session,
     handler: &rustc_errors::Handler,
-    generics: impl Iterator<Item = (ParamKindOrd, Option<&'a [GenericBound]>, Span, Option<String>)>,
+    generics: &[GenericParam],
     span: Span,
 ) {
     let mut max_param: Option<ParamKindOrd> = None;
     let mut out_of_order = FxHashMap::default();
     let mut param_idents = vec![];
 
-    for (kind, bounds, span, ident) in generics {
+    for param in generics {
+        let ident = Some(param.ident.to_string());
+        let (kind, bounds, span) = (&param.kind, Some(&*param.bounds), param.ident.span);
+        let (ord_kind, ident) = match &param.kind {
+            GenericParamKind::Lifetime => (ParamKindOrd::Lifetime, ident),
+            GenericParamKind::Type { default: _ } => (ParamKindOrd::Type, ident),
+            GenericParamKind::Const { ref ty, kw_span: _, default: _ } => {
+                let ty = pprust::ty_to_string(ty);
+                let unordered = sess.features_untracked().const_generics;
+                (ParamKindOrd::Const { unordered }, Some(format!("const {}: {}", param.ident, ty)))
+            }
+        };
         if let Some(ident) = ident {
-            param_idents.push((kind, bounds, param_idents.len(), ident));
+            param_idents.push((kind, ord_kind, bounds, param_idents.len(), ident));
         }
         let max_param = &mut max_param;
         match max_param {
-            Some(max_param) if *max_param > kind => {
+            Some(max_param) if *max_param > ord_kind => {
-                let entry = out_of_order.entry(kind).or_insert((*max_param, vec![]));
+                let entry = out_of_order.entry(ord_kind).or_insert((*max_param, vec![]));
                 entry.1.push(span);
             }
-            Some(_) | None => *max_param = Some(kind),
+            Some(_) | None => *max_param = Some(ord_kind),
         };
     }
 
     let mut ordered_params = "<".to_string();
     if !out_of_order.is_empty() {
-        param_idents.sort_by_key(|&(po, _, i, _)| (po, i));
+        param_idents.sort_by_key(|&(_, po, _, i, _)| (po, i));
         let mut first = true;
-        for (_, bounds, _, ident) in param_idents {
+        for (kind, _, bounds, _, ident) in param_idents {
             if !first {
                 ordered_params += ", ";
             }
@@ -756,6 +767,16 @@ fn validate_generic_param_order<'a>(
                     ordered_params += &pprust::bounds_to_string(&bounds);
                 }
             }
+            match kind {
+                GenericParamKind::Type { default: Some(default) } => {
+                    ordered_params += " = ";
+                    ordered_params += &pprust::ty_to_string(default);
+                }
+                GenericParamKind::Type { default: None } => (),
+                GenericParamKind::Lifetime => (),
+                // FIXME(const_generics_defaults)
+                GenericParamKind::Const { ty: _, kw_span: _, default: _ } => (),
+            }
             first = false;
         }
     }
@@ -773,14 +794,12 @@ fn validate_generic_param_order<'a>(
         err.span_suggestion(
             span,
             &format!(
-                "reorder the parameters: lifetimes{}",
+                "reorder the parameters: lifetimes, {}",
                 if sess.features_untracked().const_generics {
-                    ", then consts and types"
+                    "then consts and types"
-                } else if sess.features_untracked().min_const_generics {
-                    ", then types, then consts"
                 } else {
-                    ", then types"
+                    "then types, then consts"
-                },
+                }
             ),
             ordered_params.clone(),
             Applicability::MachineApplicable,
@@ -815,7 +834,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         match ty.kind {
             TyKind::BareFn(ref bfty) => {
                 self.check_fn_decl(&bfty.decl, SelfSemantic::No);
-                Self::check_decl_no_pat(&bfty.decl, |span, _| {
+                Self::check_decl_no_pat(&bfty.decl, |span, _, _| {
                     struct_span_err!(
                         self.session,
                         span,
@@ -901,7 +920,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         }
 
         match item.kind {
-            ItemKind::Impl {
+            ItemKind::Impl(box ImplKind {
                 unsafety,
                 polarity,
                 defaultness: _,
@@ -910,7 +929,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 of_trait: Some(ref t),
                 ref self_ty,
                 items: _,
-            } => {
+            }) => {
                 self.with_in_trait_impl(true, |this| {
                     this.invalid_visibility(&item.vis, None);
                     if let TyKind::Err = self_ty.kind {
@@ -938,7 +957,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 });
                 return; // Avoid visiting again.
             }
-            ItemKind::Impl {
+            ItemKind::Impl(box ImplKind {
                 unsafety,
                 polarity,
                 defaultness,
@@ -947,7 +966,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 of_trait: None,
                 ref self_ty,
                 items: _,
-            } => {
+            }) => {
                 let error = |annotation_span, annotation| {
                     let mut err = self.err_handler().struct_span_err(
                         self_ty.span,
@@ -979,7 +998,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         .emit();
                 }
             }
-            ItemKind::Fn(def, _, _, ref body) => {
+            ItemKind::Fn(box FnKind(def, _, _, ref body)) => {
                 self.check_defaultness(item.span, def);
 
                 if body.is_none() {
@@ -1008,7 +1027,13 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     }
                 }
             }
-            ItemKind::Trait(is_auto, _, ref generics, ref bounds, ref trait_items) => {
+            ItemKind::Trait(box TraitKind(
+                is_auto,
+                _,
+                ref generics,
+                ref bounds,
+                ref trait_items,
+            )) => {
                 if is_auto == IsAuto::Yes {
                     // Auto traits cannot have generics, super traits nor contain items.
                     self.deny_generic_params(generics, item.ident.span);
@@ -1056,7 +1081,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 let msg = "free static item without body";
                 self.error_item_without_body(item.span, "static", msg, " = <expr>;");
             }
-            ItemKind::TyAlias(def, _, ref bounds, ref body) => {
+            ItemKind::TyAlias(box TyAliasKind(def, _, ref bounds, ref body)) => {
                 self.check_defaultness(item.span, def);
                 if body.is_none() {
                     let msg = "free type alias without body";
@@ -1072,12 +1097,12 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
     fn visit_foreign_item(&mut self, fi: &'a ForeignItem) {
         match &fi.kind {
-            ForeignItemKind::Fn(def, sig, _, body) => {
+            ForeignItemKind::Fn(box FnKind(def, sig, _, body)) => {
                 self.check_defaultness(fi.span, *def);
                 self.check_foreign_fn_bodyless(fi.ident, body.as_deref());
                 self.check_foreign_fn_headerless(fi.ident, fi.span, sig.header);
             }
-            ForeignItemKind::TyAlias(def, generics, bounds, body) => {
+            ForeignItemKind::TyAlias(box TyAliasKind(def, generics, bounds, body)) => {
                 self.check_defaultness(fi.span, *def);
                 self.check_foreign_kind_bodyless(fi.ident, "type", body.as_ref().map(|b| b.span));
                 self.check_type_no_bounds(bounds, "`extern` blocks");
@@ -1152,22 +1177,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         validate_generic_param_order(
             self.session,
             self.err_handler(),
-            generics.params.iter().map(|param| {
+            &generics.params,
-                let ident = Some(param.ident.to_string());
-                let (kind, ident) = match &param.kind {
-                    GenericParamKind::Lifetime => (ParamKindOrd::Lifetime, ident),
-                    GenericParamKind::Type { default: _ } => (ParamKindOrd::Type, ident),
-                    GenericParamKind::Const { ref ty, kw_span: _ } => {
-                        let ty = pprust::ty_to_string(ty);
-                        let unordered = self.session.features_untracked().const_generics;
-                        (
-                            ParamKindOrd::Const { unordered },
-                            Some(format!("const {}: {}", param.ident, ty)),
-                        )
-                    }
-                };
-                (kind, Some(&*param.bounds), param.ident.span, ident)
-            }),
             generics.span,
         );
 
@@ -1208,11 +1218,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
     }
 
     fn visit_pat(&mut self, pat: &'a Pat) {
-        match pat.kind {
+        match &pat.kind {
-            PatKind::Lit(ref expr) => {
+            PatKind::Lit(expr) => {
                 self.check_expr_within_pat(expr, false);
             }
-            PatKind::Range(ref start, ref end, _) => {
+            PatKind::Range(start, end, _) => {
                 if let Some(expr) = start {
                     self.check_expr_within_pat(expr, true);
                 }
@@ -1285,7 +1295,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
         // Functions without bodies cannot have patterns.
         if let FnKind::Fn(ctxt, _, sig, _, None) = fk {
-            Self::check_decl_no_pat(&sig.decl, |span, mut_ident| {
+            Self::check_decl_no_pat(&sig.decl, |span, ident, mut_ident| {
                 let (code, msg, label) = match ctxt {
                     FnCtxt::Foreign => (
                         error_code!(E0130),
@@ -1299,7 +1309,16 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     ),
                 };
                 if mut_ident && matches!(ctxt, FnCtxt::Assoc(_)) {
-                    self.lint_buffer.buffer_lint(PATTERNS_IN_FNS_WITHOUT_BODY, id, span, msg);
+                    if let Some(ident) = ident {
+                        let diag = BuiltinLintDiagnostics::PatternsInFnsWithoutBody(span, ident);
+                        self.lint_buffer.buffer_lint_with_diagnostic(
+                            PATTERNS_IN_FNS_WITHOUT_BODY,
+                            id,
+                            span,
+                            msg,
+                            diag,
+                        )
+                    }
                 } else {
                     self.err_handler()
                         .struct_span_err(span, msg)
@@ -1323,10 +1342,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 AssocItemKind::Const(_, _, body) => {
                     self.check_impl_item_provided(item.span, body, "constant", " = <expr>;");
                 }
-                AssocItemKind::Fn(_, _, _, body) => {
+                AssocItemKind::Fn(box FnKind(_, _, _, body)) => {
                     self.check_impl_item_provided(item.span, body, "function", " { <body> }");
                 }
-                AssocItemKind::TyAlias(_, _, bounds, body) => {
+                AssocItemKind::TyAlias(box TyAliasKind(_, _, bounds, body)) => {
                     self.check_impl_item_provided(item.span, body, "type", " = <type>;");
                     self.check_type_no_bounds(bounds, "`impl`s");
                 }
@@ -1336,7 +1355,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
         if ctxt == AssocCtxt::Trait || self.in_trait_impl {
             self.invalid_visibility(&item.vis, None);
-            if let AssocItemKind::Fn(_, sig, _, _) = &item.kind {
+            if let AssocItemKind::Fn(box FnKind(_, sig, _, _)) = &item.kind {
                 self.check_trait_fn_not_const(sig.header.constness);
                 self.check_trait_fn_not_async(item.span, sig.header.asyncness);
             }

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -1,7 +1,7 @@
 use rustc_ast as ast;
 use rustc_ast::visit::{self, AssocCtxt, FnCtxt, FnKind, Visitor};
 use rustc_ast::{AssocTyConstraint, AssocTyConstraintKind, NodeId};
-use rustc_ast::{GenericParam, GenericParamKind, PatKind, RangeEnd, VariantData};
+use rustc_ast::{PatKind, RangeEnd, VariantData};
 use rustc_errors::struct_span_err;
 use rustc_feature::{AttributeGate, BUILTIN_ATTRIBUTE_MAP};
 use rustc_feature::{Features, GateIssue};
@@ -14,6 +14,17 @@ use rustc_span::Span;
 use tracing::debug;
 
 macro_rules! gate_feature_fn {
+    ($visitor: expr, $has_feature: expr, $span: expr, $name: expr, $explain: expr, $help: expr) => {{
+        let (visitor, has_feature, span, name, explain, help) =
+            (&*$visitor, $has_feature, $span, $name, $explain, $help);
+        let has_feature: bool = has_feature(visitor.features);
+        debug!("gate_feature(feature = {:?}, span = {:?}); has? {}", name, span, has_feature);
+        if !has_feature && !span.allows_unstable($name) {
+            feature_err_issue(&visitor.sess.parse_sess, name, span, GateIssue::Language, explain)
+                .help(help)
+                .emit();
+        }
+    }};
     ($visitor: expr, $has_feature: expr, $span: expr, $name: expr, $explain: expr) => {{
         let (visitor, has_feature, span, name, explain) =
             (&*$visitor, $has_feature, $span, $name, $explain);
@@ -27,6 +38,9 @@ macro_rules! gate_feature_fn {
 }
 
 macro_rules! gate_feature_post {
+    ($visitor: expr, $feature: ident, $span: expr, $explain: expr, $help: expr) => {
+        gate_feature_fn!($visitor, |x: &Features| x.$feature, $span, sym::$feature, $explain, $help)
+    };
     ($visitor: expr, $feature: ident, $span: expr, $explain: expr) => {
         gate_feature_fn!($visitor, |x: &Features| x.$feature, $span, sym::$feature, $explain)
     };
@@ -142,6 +156,14 @@ impl<'a> PostExpansionVisitor<'a> {
                     "efiapi ABI is experimental and subject to change"
                 );
             }
+            "C-cmse-nonsecure-call" => {
+                gate_feature_post!(
+                    &self,
+                    abi_c_cmse_nonsecure_call,
+                    span,
+                    "C-cmse-nonsecure-call ABI is experimental and subject to change"
+                );
+            }
             abi => self
                 .sess
                 .parse_sess
@@ -351,12 +373,14 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 }
             }
 
-            ast::ItemKind::Impl { polarity, defaultness, ref of_trait, .. } => {
+            ast::ItemKind::Impl(box ast::ImplKind {
+                polarity, defaultness, ref of_trait, ..
+            }) => {
                 if let ast::ImplPolarity::Negative(span) = polarity {
                     gate_feature_post!(
                         &self,
                         negative_impls,
-                        span.to(of_trait.as_ref().map(|t| t.path.span).unwrap_or(span)),
+                        span.to(of_trait.as_ref().map_or(span, |t| t.path.span)),
                         "negative trait bounds are not yet fully implemented; \
                          use marker types for now"
                     );
@@ -367,7 +391,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 }
             }
 
-            ast::ItemKind::Trait(ast::IsAuto::Yes, ..) => {
+            ast::ItemKind::Trait(box ast::TraitKind(ast::IsAuto::Yes, ..)) => {
                 gate_feature_post!(
                     &self,
                     auto_traits,
@@ -385,7 +409,9 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 gate_feature_post!(&self, decl_macro, i.span, msg);
             }
 
-            ast::ItemKind::TyAlias(_, _, _, Some(ref ty)) => self.check_impl_trait(&ty),
+            ast::ItemKind::TyAlias(box ast::TyAliasKind(_, _, _, Some(ref ty))) => {
+                self.check_impl_trait(&ty)
+            }
 
             _ => {}
         }
@@ -397,10 +423,8 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
         match i.kind {
             ast::ForeignItemKind::Fn(..) | ast::ForeignItemKind::Static(..) => {
                 let link_name = self.sess.first_attr_value_str_by_name(&i.attrs, sym::link_name);
-                let links_to_llvm = match link_name {
+                let links_to_llvm =
-                    Some(val) => val.as_str().starts_with("llvm."),
+                    link_name.map_or(false, |val| val.as_str().starts_with("llvm."));
-                    _ => false,
-                };
                 if links_to_llvm {
                     gate_feature_post!(
                         &self,
@@ -529,19 +553,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
         visit::walk_fn(self, fn_kind, span)
     }
 
-    fn visit_generic_param(&mut self, param: &'a GenericParam) {
-        if let GenericParamKind::Const { .. } = param.kind {
-            gate_feature_fn!(
-                &self,
-                |x: &Features| x.const_generics || x.min_const_generics,
-                param.ident.span,
-                sym::min_const_generics,
-                "const generics are unstable"
-            );
-        }
-        visit::walk_generic_param(self, param)
-    }
-
     fn visit_assoc_ty_constraint(&mut self, constraint: &'a AssocTyConstraint) {
         if let AssocTyConstraintKind::Bound { .. } = constraint.kind {
             gate_feature_post!(
@@ -556,13 +567,13 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
     fn visit_assoc_item(&mut self, i: &'a ast::AssocItem, ctxt: AssocCtxt) {
         let is_fn = match i.kind {
-            ast::AssocItemKind::Fn(_, ref sig, _, _) => {
+            ast::AssocItemKind::Fn(box ast::FnKind(_, ref sig, _, _)) => {
                 if let (ast::Const::Yes(_), AssocCtxt::Trait) = (sig.header.constness, ctxt) {
                     gate_feature_post!(&self, const_fn, i.span, "const fn is unstable");
                 }
                 true
             }
-            ast::AssocItemKind::TyAlias(_, ref generics, _, ref ty) => {
+            ast::AssocItemKind::TyAlias(box ast::TyAliasKind(_, ref generics, _, ref ty)) => {
                 if let (Some(_), AssocCtxt::Trait) = (ty, ctxt) {
                     gate_feature_post!(
                         &self,
@@ -612,6 +623,13 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
 
     let spans = sess.parse_sess.gated_spans.spans.borrow();
     macro_rules! gate_all {
+        ($gate:ident, $msg:literal, $help:literal) => {
+            if let Some(spans) = spans.get(&sym::$gate) {
+                for span in spans {
+                    gate_feature_post!(&visitor, $gate, *span, $msg, $help);
+                }
+            }
+        };
         ($gate:ident, $msg:literal) => {
             if let Some(spans) = spans.get(&sym::$gate) {
                 for span in spans {
@@ -622,7 +640,11 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
     }
     gate_all!(if_let_guard, "`if let` guards are experimental");
     gate_all!(let_chains, "`let` expressions in this position are experimental");
-    gate_all!(async_closure, "async closures are unstable");
+    gate_all!(
+        async_closure,
+        "async closures are unstable",
+        "to use an async block, remove the `||`: `async {`"
+    );
     gate_all!(generators, "yield syntax is experimental");
     gate_all!(or_patterns, "or-patterns syntax is experimental");
     gate_all!(raw_ref_op, "raw address of syntax is experimental");
@@ -634,6 +656,10 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
         extended_key_value_attributes,
         "arbitrary expressions in key-value attributes are unstable"
     );
+    gate_all!(
+        const_generics_defaults,
+        "default values for const generic parameters are experimental"
+    );
     if sess.parse_sess.span_diagnostic.err_count() == 0 {
         // Errors for `destructuring_assignment` can get quite noisy, especially where `_` is
         // involved, so we only emit errors where there are no other parsing errors.

compiler/rustc_ast_passes/src/lib.rs

@@ -6,6 +6,8 @@
 
 #![feature(bindings_after_at)]
 #![feature(iter_is_partitioned)]
+#![feature(box_syntax)]
+#![feature(box_patterns)]
 #![recursion_limit = "256"]
 
 pub mod ast_validation;

compiler/rustc_ast_passes/src/node_count.rs

@@ -68,7 +68,7 @@ impl<'ast> Visitor<'ast> for NodeCounter {
         self.count += 1;
         walk_generics(self, g)
     }
-    fn visit_fn(&mut self, fk: FnKind<'_>, s: Span, _: NodeId) {
+    fn visit_fn(&mut self, fk: visit::FnKind<'_>, s: Span, _: NodeId) {
         self.count += 1;
         walk_fn(self, fk, s)
     }

compiler/rustc_ast_pretty/Cargo.toml

@@ -11,4 +11,3 @@ doctest = false
 tracing = "0.1"
 rustc_span = { path = "../rustc_span" }
 rustc_ast = { path = "../rustc_ast" }
-rustc_target = { path = "../rustc_target" }

compiler/rustc_ast_pretty/src/lib.rs

@@ -1,6 +1,7 @@
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(or_patterns)]
+#![feature(box_patterns)]
 #![recursion_limit = "256"]
 
 mod helpers;

compiler/rustc_ast_pretty/src/pp.rs

@@ -75,7 +75,7 @@
 //! breaking inconsistently to become
 //!
 //! ```
-//! foo(hello, there
+//! foo(hello, there,
 //!     good, friends);
 //! ```
 //!
@@ -83,7 +83,7 @@
 //!
 //! ```
 //! foo(hello,
-//!     there
+//!     there,
 //!     good,
 //!     friends);
 //! ```

compiler/rustc_ast_pretty/src/pprust/mod.rs

@@ -8,11 +8,6 @@ use rustc_ast as ast;
 use rustc_ast::token::{Nonterminal, Token, TokenKind};
 use rustc_ast::tokenstream::{TokenStream, TokenTree};
 
-pub fn nonterminal_to_string_no_extra_parens(nt: &Nonterminal) -> String {
-    let state = State::without_insert_extra_parens();
-    state.nonterminal_to_string(nt)
-}
-
 pub fn nonterminal_to_string(nt: &Nonterminal) -> String {
     State::new().nonterminal_to_string(nt)
 }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -88,13 +88,6 @@ pub struct State<'a> {
     comments: Option<Comments<'a>>,
     ann: &'a (dyn PpAnn + 'a),
     is_expanded: bool,
-    // If `true`, additional parenthesis (separate from `ExprKind::Paren`)
-    // are inserted to ensure that proper precedence is preserved
-    // in the pretty-printed output.
-    //
-    // This is usually `true`, except when performing the pretty-print/reparse
-    // check in `nt_to_tokenstream`
-    insert_extra_parens: bool,
 }
 
 crate const INDENT_UNIT: usize = 4;
@@ -115,7 +108,6 @@ pub fn print_crate<'a>(
         comments: Some(Comments::new(sm, filename, input)),
         ann,
         is_expanded,
-        insert_extra_parens: true,
     };
 
     if is_expanded && !krate.attrs.iter().any(|attr| attr.has_name(sym::no_core)) {
@@ -235,7 +227,6 @@ impl std::ops::DerefMut for State<'_> {
 }
 
 pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::DerefMut {
-    fn insert_extra_parens(&self) -> bool;
     fn comments(&mut self) -> &mut Option<Comments<'a>>;
     fn print_ident(&mut self, ident: Ident);
     fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool);
@@ -454,10 +445,11 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             ),
             MacArgs::Empty | MacArgs::Eq(..) => {
                 self.print_path(&item.path, false, 0);
-                if let MacArgs::Eq(_, tokens) = &item.args {
+                if let MacArgs::Eq(_, token) = &item.args {
                     self.space();
                     self.word_space("=");
-                    self.print_tts(tokens, true);
+                    let token_str = self.token_to_string_ext(token, true);
+                    self.word(token_str);
                 }
             }
         }
@@ -819,16 +811,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
 
     fn to_string(&self, f: impl FnOnce(&mut State<'_>)) -> String {
         let mut printer = State::new();
-        printer.insert_extra_parens = self.insert_extra_parens();
         f(&mut printer);
         printer.s.eof()
     }
 }
 
 impl<'a> PrintState<'a> for State<'a> {
-    fn insert_extra_parens(&self) -> bool {
-        self.insert_extra_parens
-    }
     fn comments(&mut self) -> &mut Option<Comments<'a>> {
         &mut self.comments
     }
@@ -865,17 +853,7 @@ impl<'a> PrintState<'a> for State<'a> {
 
 impl<'a> State<'a> {
     pub fn new() -> State<'a> {
-        State {
+        State { s: pp::mk_printer(), comments: None, ann: &NoAnn, is_expanded: false }
-            s: pp::mk_printer(),
-            comments: None,
-            ann: &NoAnn,
-            is_expanded: false,
-            insert_extra_parens: true,
-        }
-    }
-
-    pub(super) fn without_insert_extra_parens() -> State<'a> {
-        State { insert_extra_parens: false, ..State::new() }
     }
 
     // Synthesizes a comment that was not textually present in the original source
@@ -1044,14 +1022,14 @@ impl<'a> State<'a> {
         self.maybe_print_comment(span.lo());
         self.print_outer_attributes(attrs);
         match kind {
-            ast::ForeignItemKind::Fn(def, sig, gen, body) => {
+            ast::ForeignItemKind::Fn(box ast::FnKind(def, sig, gen, body)) => {
                 self.print_fn_full(sig, ident, gen, vis, *def, body.as_deref(), attrs);
             }
             ast::ForeignItemKind::Static(ty, mutbl, body) => {
                 let def = ast::Defaultness::Final;
                 self.print_item_const(ident, Some(*mutbl), ty, body.as_deref(), vis, def);
             }
-            ast::ForeignItemKind::TyAlias(def, generics, bounds, ty) => {
+            ast::ForeignItemKind::TyAlias(box ast::TyAliasKind(def, generics, bounds, ty)) => {
                 self.print_associated_type(ident, generics, bounds, ty.as_deref(), vis, *def);
             }
             ast::ForeignItemKind::MacCall(m) => {
@@ -1156,7 +1134,7 @@ impl<'a> State<'a> {
             ast::ItemKind::Const(def, ref ty, ref body) => {
                 self.print_item_const(item.ident, None, ty, body.as_deref(), &item.vis, def);
             }
-            ast::ItemKind::Fn(def, ref sig, ref gen, ref body) => {
+            ast::ItemKind::Fn(box ast::FnKind(def, ref sig, ref gen, ref body)) => {
                 let body = body.as_deref();
                 self.print_fn_full(sig, item.ident, gen, &item.vis, def, body, &item.attrs);
             }
@@ -1197,7 +1175,7 @@ impl<'a> State<'a> {
                 self.s.word(ga.asm.to_string());
                 self.end();
             }
-            ast::ItemKind::TyAlias(def, ref generics, ref bounds, ref ty) => {
+            ast::ItemKind::TyAlias(box ast::TyAliasKind(def, ref generics, ref bounds, ref ty)) => {
                 let ty = ty.as_deref();
                 self.print_associated_type(item.ident, generics, bounds, ty, &item.vis, def);
             }
@@ -1212,7 +1190,7 @@ impl<'a> State<'a> {
                 self.head(visibility_qualified(&item.vis, "union"));
                 self.print_struct(struct_def, generics, item.ident, item.span, true);
             }
-            ast::ItemKind::Impl {
+            ast::ItemKind::Impl(box ast::ImplKind {
                 unsafety,
                 polarity,
                 defaultness,
@@ -1221,7 +1199,7 @@ impl<'a> State<'a> {
                 ref of_trait,
                 ref self_ty,
                 ref items,
-            } => {
+            }) => {
                 self.head("");
                 self.print_visibility(&item.vis);
                 self.print_defaultness(defaultness);
@@ -1255,7 +1233,13 @@ impl<'a> State<'a> {
                 }
                 self.bclose(item.span);
             }
-            ast::ItemKind::Trait(is_auto, unsafety, ref generics, ref bounds, ref trait_items) => {
+            ast::ItemKind::Trait(box ast::TraitKind(
+                is_auto,
+                unsafety,
+                ref generics,
+                ref bounds,
+                ref trait_items,
+            )) => {
                 self.head("");
                 self.print_visibility(&item.vis);
                 self.print_unsafety(unsafety);
@@ -1475,13 +1459,13 @@ impl<'a> State<'a> {
         self.maybe_print_comment(span.lo());
         self.print_outer_attributes(attrs);
         match kind {
-            ast::AssocItemKind::Fn(def, sig, gen, body) => {
+            ast::AssocItemKind::Fn(box ast::FnKind(def, sig, gen, body)) => {
                 self.print_fn_full(sig, ident, gen, vis, *def, body.as_deref(), attrs);
             }
             ast::AssocItemKind::Const(def, ty, body) => {
                 self.print_item_const(ident, None, ty, body.as_deref(), vis, *def);
             }
-            ast::AssocItemKind::TyAlias(def, generics, bounds, ty) => {
+            ast::AssocItemKind::TyAlias(box ast::TyAliasKind(def, generics, bounds, ty)) => {
                 self.print_associated_type(ident, generics, bounds, ty.as_deref(), vis, *def);
             }
             ast::AssocItemKind::MacCall(m) => {
@@ -1680,8 +1664,7 @@ impl<'a> State<'a> {
     }
 
     /// Prints `expr` or `(expr)` when `needs_par` holds.
-    fn print_expr_cond_paren(&mut self, expr: &ast::Expr, mut needs_par: bool) {
+    fn print_expr_cond_paren(&mut self, expr: &ast::Expr, needs_par: bool) {
-        needs_par &= self.insert_extra_parens;
         if needs_par {
             self.popen();
         }
@@ -2668,13 +2651,16 @@ impl<'a> State<'a> {
                         s.print_type(default)
                     }
                 }
-                ast::GenericParamKind::Const { ref ty, kw_span: _ } => {
+                ast::GenericParamKind::Const { ref ty, kw_span: _, ref default } => {
                     s.word_space("const");
                     s.print_ident(param.ident);
                     s.s.space();
                     s.word_space(":");
                     s.print_type(ty);
-                    s.print_type_bounds(":", &param.bounds)
+                    s.print_type_bounds(":", &param.bounds);
+                    if let Some(ref _default) = default {
+                        // FIXME(const_generics_defaults): print the `default` value here
+                    }
                 }
             }
         });
@@ -2787,7 +2773,7 @@ impl<'a> State<'a> {
                     self.print_explicit_self(&eself);
                 } else {
                     let invalid = if let PatKind::Ident(_, ident, _) = input.pat.kind {
-                        ident.name == kw::Invalid
+                        ident.name == kw::Empty
                     } else {
                         false
                     };

compiler/rustc_attr/Cargo.toml

@@ -18,4 +18,3 @@ rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_session = { path = "../rustc_session" }
 rustc_ast = { path = "../rustc_ast" }
-version_check = "0.9"

compiler/rustc_attr/src/builtin.rs

@@ -10,7 +10,6 @@ use rustc_session::Session;
 use rustc_span::hygiene::Transparency;
 use rustc_span::{symbol::sym, symbol::Symbol, Span};
 use std::num::NonZeroU32;
-use version_check::Version;
 
 pub fn is_builtin_attr(attr: &Attribute) -> bool {
     attr.is_doc_comment() || attr.ident().filter(|ident| is_builtin_attr_name(ident.name)).is_some()
@@ -67,7 +66,7 @@ fn handle_errors(sess: &ParseSess, span: Span, error: AttrError) {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Copy, Clone, PartialEq, Encodable, Decodable, Debug)]
 pub enum InlineAttr {
     None,
     Hint,
@@ -75,13 +74,13 @@ pub enum InlineAttr {
     Never,
 }
 
-#[derive(Clone, Encodable, Decodable)]
+#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
 pub enum InstructionSetAttr {
     ArmA32,
     ArmT32,
 }
 
-#[derive(Clone, Encodable, Decodable)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub enum OptimizeAttr {
     None,
     Speed,
@@ -526,6 +525,26 @@ fn gate_cfg(gated_cfg: &GatedCfg, cfg_span: Span, sess: &ParseSess, features: &F
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct Version {
+    major: u16,
+    minor: u16,
+    patch: u16,
+}
+
+fn parse_version(s: &str, allow_appendix: bool) -> Option<Version> {
+    let mut components = s.split('-');
+    let d = components.next()?;
+    if !allow_appendix && components.next().is_some() {
+        return None;
+    }
+    let mut digits = d.splitn(3, '.');
+    let major = digits.next()?.parse().ok()?;
+    let minor = digits.next()?.parse().ok()?;
+    let patch = digits.next().unwrap_or("0").parse().ok()?;
+    Some(Version { major, minor, patch })
+}
+
 /// Evaluate a cfg-like condition (with `any` and `all`), using `eval` to
 /// evaluate individual items.
 pub fn eval_condition(
@@ -555,19 +574,26 @@ pub fn eval_condition(
                     return false;
                 }
             };
-            let min_version = match Version::parse(&min_version.as_str()) {
+            let min_version = match parse_version(&min_version.as_str(), false) {
                 Some(ver) => ver,
                 None => {
-                    sess.span_diagnostic.struct_span_err(*span, "invalid version literal").emit();
+                    sess.span_diagnostic
+                        .struct_span_warn(
+                            *span,
+                            "unknown version literal format, assuming it refers to a future version",
+                        )
+                        .emit();
                     return false;
                 }
             };
-            let channel = env!("CFG_RELEASE_CHANNEL");
+            let rustc_version = parse_version(env!("CFG_RELEASE"), true).unwrap();
-            let nightly = channel == "nightly" || channel == "dev";
-            let rustc_version = Version::parse(env!("CFG_RELEASE")).unwrap();
 
-            // See https://github.com/rust-lang/rust/issues/64796#issuecomment-625474439 for details
+            // See https://github.com/rust-lang/rust/issues/64796#issuecomment-640851454 for details
-            if nightly { rustc_version > min_version } else { rustc_version >= min_version }
+            if sess.assume_incomplete_release {
+                rustc_version > min_version
+            } else {
+                rustc_version >= min_version
+            }
         }
         ast::MetaItemKind::List(ref mis) => {
             for mi in mis.iter() {

compiler/rustc_builtin_macros/src/assert.rs

@@ -12,27 +12,43 @@ use rustc_span::{Span, DUMMY_SP};
 
 pub fn expand_assert<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
-    sp: Span,
+    span: Span,
     tts: TokenStream,
 ) -> Box<dyn MacResult + 'cx> {
-    let Assert { cond_expr, custom_message } = match parse_assert(cx, sp, tts) {
+    let Assert { cond_expr, custom_message } = match parse_assert(cx, span, tts) {
         Ok(assert) => assert,
         Err(mut err) => {
             err.emit();
-            return DummyResult::any(sp);
+            return DummyResult::any(span);
         }
     };
 
     // `core::panic` and `std::panic` are different macros, so we use call-site
     // context to pick up whichever is currently in scope.
-    let sp = cx.with_call_site_ctxt(sp);
+    let sp = cx.with_call_site_ctxt(span);
 
     let panic_call = if let Some(tokens) = custom_message {
+        let path = if span.rust_2021() {
+            // On edition 2021, we always call `$crate::panic::panic_2021!()`.
+            Path {
+                span: sp,
+                segments: cx
+                    .std_path(&[sym::panic, sym::panic_2021])
+                    .into_iter()
+                    .map(|ident| PathSegment::from_ident(ident))
+                    .collect(),
+                tokens: None,
+            }
+        } else {
+            // Before edition 2021, we call `panic!()` unqualified,
+            // such that it calls either `std::panic!()` or `core::panic!()`.
+            Path::from_ident(Ident::new(sym::panic, sp))
+        };
         // Pass the custom message to panic!().
         cx.expr(
             sp,
             ExprKind::MacCall(MacCall {
-                path: Path::from_ident(Ident::new(sym::panic, sp)),
+                path,
                 args: P(MacArgs::Delimited(
                     DelimSpan::from_single(sp),
                     MacDelimiter::Parenthesis,

compiler/rustc_builtin_macros/src/deriving/clone.rs

@@ -38,10 +38,9 @@ pub fn expand_deriving_clone(
             | ItemKind::Enum(_, Generics { ref params, .. }) => {
                 let container_id = cx.current_expansion.id.expn_data().parent;
                 if cx.resolver.has_derive_copy(container_id)
-                    && !params.iter().any(|param| match param.kind {
+                    && !params
-                        ast::GenericParamKind::Type { .. } => true,
+                        .iter()
-                        _ => false,
+                        .any(|param| matches!(param.kind, ast::GenericParamKind::Type { .. }))
-                    })
                 {
                     bounds = vec![];
                     is_shallow = true;

compiler/rustc_builtin_macros/src/deriving/debug.rs

@@ -8,6 +8,10 @@ use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{Span, DUMMY_SP};
 
+fn make_mut_borrow(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<Expr>) -> P<Expr> {
+    cx.expr(sp, ast::ExprKind::AddrOf(ast::BorrowKind::Ref, ast::Mutability::Mut, expr))
+}
+
 pub fn expand_deriving_debug(
     cx: &mut ExtCtxt<'_>,
     span: Span,
@@ -67,11 +71,12 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
     let fmt = substr.nonself_args[0].clone();
 
     let mut stmts = Vec::with_capacity(fields.len() + 2);
+    let fn_path_finish;
     match vdata {
         ast::VariantData::Tuple(..) | ast::VariantData::Unit(..) => {
             // tuple struct/"normal" variant
-            let expr =
+            let fn_path_debug_tuple = cx.std_path(&[sym::fmt, sym::Formatter, sym::debug_tuple]);
-                cx.expr_method_call(span, fmt, Ident::new(sym::debug_tuple, span), vec![name]);
+            let expr = cx.expr_call_global(span, fn_path_debug_tuple, vec![fmt, name]);
             stmts.push(cx.stmt_let(span, true, builder, expr));
 
             for field in fields {
@@ -79,22 +84,21 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
                 let field = cx.expr_addr_of(field.span, field.self_.clone());
                 let field = cx.expr_addr_of(field.span, field);
 
-                let expr = cx.expr_method_call(
+                let fn_path_field = cx.std_path(&[sym::fmt, sym::DebugTuple, sym::field]);
-                    span,
+                let builder_recv = make_mut_borrow(cx, span, builder_expr.clone());
-                    builder_expr.clone(),
+                let expr = cx.expr_call_global(span, fn_path_field, vec![builder_recv, field]);
-                    Ident::new(sym::field, span),
-                    vec![field],
-                );
 
                 // Use `let _ = expr;` to avoid triggering the
                 // unused_results lint.
                 stmts.push(stmt_let_underscore(cx, span, expr));
             }
+
+            fn_path_finish = cx.std_path(&[sym::fmt, sym::DebugTuple, sym::finish]);
         }
         ast::VariantData::Struct(..) => {
             // normal struct/struct variant
-            let expr =
+            let fn_path_debug_struct = cx.std_path(&[sym::fmt, sym::Formatter, sym::debug_struct]);
-                cx.expr_method_call(span, fmt, Ident::new(sym::debug_struct, span), vec![name]);
+            let expr = cx.expr_call_global(span, fn_path_debug_struct, vec![fmt, name]);
             stmts.push(cx.stmt_let(DUMMY_SP, true, builder, expr));
 
             for field in fields {
@@ -104,20 +108,20 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
                 );
 
                 // Use double indirection to make sure this works for unsized types
+                let fn_path_field = cx.std_path(&[sym::fmt, sym::DebugStruct, sym::field]);
                 let field = cx.expr_addr_of(field.span, field.self_.clone());
                 let field = cx.expr_addr_of(field.span, field);
-                let expr = cx.expr_method_call(
+                let builder_recv = make_mut_borrow(cx, span, builder_expr.clone());
-                    span,
+                let expr =
-                    builder_expr.clone(),
+                    cx.expr_call_global(span, fn_path_field, vec![builder_recv, name, field]);
-                    Ident::new(sym::field, span),
-                    vec![name, field],
-                );
                 stmts.push(stmt_let_underscore(cx, span, expr));
             }
+            fn_path_finish = cx.std_path(&[sym::fmt, sym::DebugStruct, sym::finish]);
         }
     }
 
-    let expr = cx.expr_method_call(span, builder_expr, Ident::new(sym::finish, span), vec![]);
+    let builder_recv = make_mut_borrow(cx, span, builder_expr);
+    let expr = cx.expr_call_global(span, fn_path_finish, vec![builder_recv]);
 
     stmts.push(cx.stmt_expr(expr));
     let block = cx.block(span, stmts);

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -404,12 +404,10 @@ impl<'a> TraitDef<'a> {
                 let has_no_type_params = match item.kind {
                     ast::ItemKind::Struct(_, ref generics)
                     | ast::ItemKind::Enum(_, ref generics)
-                    | ast::ItemKind::Union(_, ref generics) => {
+                    | ast::ItemKind::Union(_, ref generics) => !generics
-                        !generics.params.iter().any(|param| match param.kind {
+                        .params
-                            ast::GenericParamKind::Type { .. } => true,
+                        .iter()
-                            _ => false,
+                        .any(|param| matches!(param.kind, ast::GenericParamKind::Type { .. })),
-                        })
-                    }
                     _ => unreachable!(),
                 };
                 let container_id = cx.current_expansion.id.expn_data().parent;
@@ -529,12 +527,12 @@ impl<'a> TraitDef<'a> {
                     tokens: None,
                 },
                 attrs: Vec::new(),
-                kind: ast::AssocItemKind::TyAlias(
+                kind: ast::AssocItemKind::TyAlias(box ast::TyAliasKind(
                     ast::Defaultness::Final,
                     Generics::default(),
                     Vec::new(),
                     Some(type_def.to_ty(cx, self.span, type_ident, generics)),
-                ),
+                )),
                 tokens: None,
             })
         });
@@ -600,7 +598,7 @@ impl<'a> TraitDef<'a> {
 
             let mut ty_params = params
                 .iter()
-                .filter(|param| matches!(param.kind,  ast::GenericParamKind::Type{..}))
+                .filter(|param| matches!(param.kind, ast::GenericParamKind::Type { .. }))
                 .peekable();
 
             if ty_params.peek().is_some() {
@@ -689,7 +687,7 @@ impl<'a> TraitDef<'a> {
             self.span,
             Ident::invalid(),
             a,
-            ast::ItemKind::Impl {
+            ast::ItemKind::Impl(box ast::ImplKind {
                 unsafety,
                 polarity: ast::ImplPolarity::Positive,
                 defaultness: ast::Defaultness::Final,
@@ -698,7 +696,7 @@ impl<'a> TraitDef<'a> {
                 of_trait: opt_trait_ref,
                 self_ty: self_type,
                 items: methods.into_iter().chain(associated_types).collect(),
-            },
+            }),
         )
     }
 
@@ -868,7 +866,7 @@ impl<'a> MethodDef<'a> {
                 Self_ if nonstatic => {
                     self_args.push(arg_expr);
                 }
-                Ptr(ref ty, _) if (if let Self_ = **ty { true } else { false }) && nonstatic => {
+                Ptr(ref ty, _) if matches!(**ty, Self_) && nonstatic => {
                     self_args.push(cx.expr_deref(trait_.span, arg_expr))
                 }
                 _ => {
@@ -931,7 +929,7 @@ impl<'a> MethodDef<'a> {
                 tokens: None,
             },
             ident: method_ident,
-            kind: ast::AssocItemKind::Fn(def, sig, fn_generics, Some(body_block)),
+            kind: ast::AssocItemKind::Fn(box ast::FnKind(def, sig, fn_generics, Some(body_block))),
             tokens: None,
         })
     }

compiler/rustc_builtin_macros/src/deriving/hash.rs

@@ -48,8 +48,8 @@ pub fn expand_deriving_hash(
 }
 
 fn hash_substructure(cx: &mut ExtCtxt<'_>, trait_span: Span, substr: &Substructure<'_>) -> P<Expr> {
-    let state_expr = match &substr.nonself_args {
+    let state_expr = match substr.nonself_args {
-        &[o_f] => o_f,
+        [o_f] => o_f,
         _ => cx.span_bug(trait_span, "incorrect number of arguments in `derive(Hash)`"),
     };
     let call_hash = |span, thing_expr| {
@@ -64,9 +64,9 @@ fn hash_substructure(cx: &mut ExtCtxt<'_>, trait_span: Span, substr: &Substructu
     };
     let mut stmts = Vec::new();
 
-    let fields = match *substr.fields {
+    let fields = match substr.fields {
-        Struct(_, ref fs) | EnumMatching(_, 1, .., ref fs) => fs,
+        Struct(_, fs) | EnumMatching(_, 1, .., fs) => fs,
-        EnumMatching(.., ref fs) => {
+        EnumMatching(.., fs) => {
             let variant_value = deriving::call_intrinsic(
                 cx,
                 trait_span,

compiler/rustc_builtin_macros/src/deriving/mod.rs

@@ -2,7 +2,7 @@
 
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
-use rustc_ast::{ItemKind, MetaItem};
+use rustc_ast::{ImplKind, ItemKind, MetaItem};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, MultiItemModifier};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
@@ -145,7 +145,8 @@ fn inject_impl_of_structural_trait(
                 *default = None;
                 ast::GenericArg::Type(cx.ty_ident(span, param.ident))
             }
-            ast::GenericParamKind::Const { ty: _, kw_span: _ } => {
+            ast::GenericParamKind::Const { ty: _, kw_span: _, default } => {
+                *default = None;
                 ast::GenericArg::Const(cx.const_ident(span, param.ident))
             }
         })
@@ -178,7 +179,7 @@ fn inject_impl_of_structural_trait(
         span,
         Ident::invalid(),
         attrs,
-        ItemKind::Impl {
+        ItemKind::Impl(box ImplKind {
             unsafety: ast::Unsafe::No,
             polarity: ast::ImplPolarity::Positive,
             defaultness: ast::Defaultness::Final,
@@ -187,7 +188,7 @@ fn inject_impl_of_structural_trait(
             of_trait: Some(trait_ref),
             self_ty: self_type,
             items: Vec::new(),
-        },
+        }),
     );
 
     push(Annotatable::Item(newitem));

compiler/rustc_builtin_macros/src/format.rs

@@ -1044,10 +1044,7 @@ pub fn expand_preparsed_format_args(
 
     let numbered_position_args = pieces.iter().any(|arg: &parse::Piece<'_>| match *arg {
         parse::String(_) => false,
-        parse::NextArgument(arg) => match arg.position {
+        parse::NextArgument(arg) => matches!(arg.position, parse::Position::ArgumentIs(_)),
-            parse::Position::ArgumentIs(_) => true,
-            _ => false,
-        },
     });
 
     cx.build_index_map();

compiler/rustc_builtin_macros/src/format_foreign.rs

@@ -580,10 +580,7 @@ pub mod printf {
     }
 
     fn is_flag(c: &char) -> bool {
-        match c {
+        matches!(c, '0' | '-' | '+' | ' ' | '#' | '\'')
-            '0' | '-' | '+' | ' ' | '#' | '\'' => true,
-            _ => false,
-        }
     }
 
     #[cfg(test)]

compiler/rustc_builtin_macros/src/global_allocator.rs

@@ -5,7 +5,7 @@ use rustc_ast::expand::allocator::{
 };
 use rustc_ast::ptr::P;
 use rustc_ast::{self as ast, Attribute, Expr, FnHeader, FnSig, Generics, Param, StmtKind};
-use rustc_ast::{ItemKind, Mutability, Stmt, Ty, TyKind, Unsafe};
+use rustc_ast::{FnKind, ItemKind, Mutability, Stmt, Ty, TyKind, Unsafe};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
@@ -85,7 +85,8 @@ impl AllocFnFactory<'_, '_> {
         let header = FnHeader { unsafety: Unsafe::Yes(self.span), ..FnHeader::default() };
         let sig = FnSig { decl, header, span: self.span };
         let block = Some(self.cx.block_expr(output_expr));
-        let kind = ItemKind::Fn(ast::Defaultness::Final, sig, Generics::default(), block);
+        let kind =
+            ItemKind::Fn(box FnKind(ast::Defaultness::Final, sig, Generics::default(), block));
         let item = self.cx.item(
             self.span,
             Ident::from_str_and_span(&self.kind.fn_name(method.name), self.span),

compiler/rustc_builtin_macros/src/lib.rs

@@ -2,6 +2,8 @@
 //! injecting code into the crate before it is lowered to HIR.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(box_patterns)]
+#![feature(box_syntax)]
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
@@ -14,10 +16,9 @@ extern crate proc_macro;
 
 use crate::deriving::*;
 
-use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtension, SyntaxExtensionKind};
+use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
 use rustc_expand::proc_macro::BangProcMacro;
-use rustc_span::edition::Edition;
+use rustc_span::symbol::sym;
-use rustc_span::symbol::{sym, Ident};
 
 mod asm;
 mod assert;
@@ -34,6 +35,7 @@ mod global_allocator;
 mod global_asm;
 mod llvm_asm;
 mod log_syntax;
+mod panic;
 mod source_util;
 mod test;
 mod trace_macros;
@@ -44,13 +46,8 @@ pub mod proc_macro_harness;
 pub mod standard_library_imports;
 pub mod test_harness;
 
-pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand, edition: Edition) {
+pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
-    let mut register = |name, kind| {
+    let mut register = |name, kind| resolver.register_builtin_macro(name, kind);
-        resolver.register_builtin_macro(
-            Ident::with_dummy_span(name),
-            SyntaxExtension { is_builtin: true, ..SyntaxExtension::default(kind, edition) },
-        )
-    };
     macro register_bang($($name:ident: $f:expr,)*) {
         $(register(sym::$name, SyntaxExtensionKind::LegacyBang(Box::new($f as MacroExpanderFn)));)*
     }
@@ -82,6 +79,8 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand, edition: Editi
         log_syntax: log_syntax::expand_log_syntax,
         module_path: source_util::expand_mod,
         option_env: env::expand_option_env,
+        core_panic: panic::expand_panic,
+        std_panic: panic::expand_panic,
         stringify: source_util::expand_stringify,
         trace_macros: trace_macros::expand_trace_macros,
     }

compiler/rustc_builtin_macros/src/llvm_asm.rs

@@ -87,13 +87,15 @@ fn parse_inline_asm<'a>(
     // parsed as `llvm_asm!(z)` with `z = "x": y` which is type ascription.
     let first_colon = tts
         .trees()
-        .position(|tt| match tt {
+        .position(|tt| {
-            tokenstream::TokenTree::Token(Token { kind: token::Colon | token::ModSep, .. }) => true,
+            matches!(
-            _ => false,
+                tt,
+                tokenstream::TokenTree::Token(Token { kind: token::Colon | token::ModSep, .. })
+            )
         })
         .unwrap_or(tts.len());
     let mut p = cx.new_parser_from_tts(tts.trees().skip(first_colon).collect());
-    let mut asm = kw::Invalid;
+    let mut asm = kw::Empty;
     let mut asm_str_style = None;
     let mut outputs = Vec::new();
     let mut inputs = Vec::new();

compiler/rustc_builtin_macros/src/panic.rs

@@ -0,0 +1,48 @@
+use rustc_ast::ptr::P;
+use rustc_ast::tokenstream::{DelimSpan, TokenStream};
+use rustc_ast::*;
+use rustc_expand::base::*;
+use rustc_span::symbol::sym;
+use rustc_span::Span;
+
+// This expands to either
+// - `$crate::panic::panic_2015!(...)` or
+// - `$crate::panic::panic_2021!(...)`
+// depending on the edition.
+//
+// This is used for both std::panic!() and core::panic!().
+//
+// `$crate` will refer to either the `std` or `core` crate depending on which
+// one we're expanding from.
+pub fn expand_panic<'cx>(
+    cx: &'cx mut ExtCtxt<'_>,
+    sp: Span,
+    tts: TokenStream,
+) -> Box<dyn MacResult + 'cx> {
+    let panic = if sp.rust_2021() { sym::panic_2021 } else { sym::panic_2015 };
+
+    let sp = cx.with_call_site_ctxt(sp);
+
+    MacEager::expr(
+        cx.expr(
+            sp,
+            ExprKind::MacCall(MacCall {
+                path: Path {
+                    span: sp,
+                    segments: cx
+                        .std_path(&[sym::panic, panic])
+                        .into_iter()
+                        .map(|ident| PathSegment::from_ident(ident))
+                        .collect(),
+                    tokens: None,
+                },
+                args: P(MacArgs::Delimited(
+                    DelimSpan::from_single(sp),
+                    MacDelimiter::Parenthesis,
+                    tts,
+                )),
+                prior_type_ascription: None,
+            }),
+        ),
+    )
+}

compiler/rustc_builtin_macros/src/proc_macro_harness.rs

@@ -256,10 +256,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
         // we're just not interested in this item.
         //
         // If we find one, try to locate a `#[proc_macro_derive]` attribute on it.
-        let is_fn = match item.kind {
+        let is_fn = matches!(item.kind, ast::ItemKind::Fn(..));
-            ast::ItemKind::Fn(..) => true,
-            _ => false,
-        };
 
         let mut found_attr: Option<&'a ast::Attribute> = None;
 

compiler/rustc_builtin_macros/src/source_util.rs

@@ -5,7 +5,8 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{self, *};
 use rustc_expand::module::DirectoryOwnership;
-use rustc_parse::{self, new_parser_from_file, parser::Parser};
+use rustc_parse::parser::{ForceCollect, Parser};
+use rustc_parse::{self, new_parser_from_file};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
 use rustc_span::symbol::Symbol;
 use rustc_span::{self, Pos, Span};
@@ -139,7 +140,7 @@ pub fn expand_include<'cx>(
         fn make_items(mut self: Box<ExpandResult<'a>>) -> Option<SmallVec<[P<ast::Item>; 1]>> {
             let mut ret = SmallVec::new();
             while self.p.token != token::Eof {
-                match self.p.parse_item() {
+                match self.p.parse_item(ForceCollect::No) {
                     Err(mut err) => {
                         err.emit();
                         break;

compiler/rustc_builtin_macros/src/test.rs

@@ -425,7 +425,7 @@ fn test_type(cx: &ExtCtxt<'_>) -> TestType {
 fn has_test_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool {
     let has_should_panic_attr = cx.sess.contains_name(&i.attrs, sym::should_panic);
     let sd = &cx.sess.parse_sess.span_diagnostic;
-    if let ast::ItemKind::Fn(_, ref sig, ref generics, _) = i.kind {
+    if let ast::ItemKind::Fn(box ast::FnKind(_, ref sig, ref generics, _)) = i.kind {
         if let ast::Unsafe::Yes(span) = sig.header.unsafety {
             sd.struct_span_err(i.span, "unsafe functions cannot be used for tests")
                 .span_label(span, "`unsafe` because of this")
@@ -474,7 +474,7 @@ fn has_test_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool {
 }
 
 fn has_bench_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool {
-    let has_sig = if let ast::ItemKind::Fn(_, ref sig, _, _) = i.kind {
+    let has_sig = if let ast::ItemKind::Fn(box ast::FnKind(_, ref sig, _, _)) = i.kind {
         // N.B., inadequate check, but we're running
         // well before resolve, can't get too deep.
         sig.decl.inputs.len() == 1

compiler/rustc_builtin_macros/src/test_harness.rs

@@ -311,7 +311,8 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
     let decl = ecx.fn_decl(vec![], ast::FnRetTy::Ty(main_ret_ty));
     let sig = ast::FnSig { decl, header: ast::FnHeader::default(), span: sp };
     let def = ast::Defaultness::Final;
-    let main = ast::ItemKind::Fn(def, sig, ast::Generics::default(), Some(main_body));
+    let main =
+        ast::ItemKind::Fn(box ast::FnKind(def, sig, ast::Generics::default(), Some(main_body)));
 
     // Honor the reexport_test_harness_main attribute
     let main_id = match cx.reexport_test_harness_main {

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -12,6 +12,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
+        env:
+          - BACKEND: ""
+          - BACKEND: --oldbe
 
     steps:
     - uses: actions/checkout@v2
@@ -51,7 +54,7 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
-        ./test.sh
+        ./test.sh $BACKEND
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz build

compiler/rustc_codegen_cranelift/.vscode/settings.json

@@ -1,6 +1,7 @@
 {
     // source for rustc_* is not included in the rust-src component; disable the errors about this
     "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"],
+    "rust-analyzer.assist.importMergeBehavior": "last",
     "rust-analyzer.cargo.loadOutDirsFromCheck": true,
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -2,9 +2,9 @@
 # It is not intended for manual editing.
 [[package]]
 name = "anyhow"
-version = "1.0.34"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7"
+checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1"
 
 [[package]]
 name = "ar"
@@ -25,15 +25,15 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
 [[package]]
 name = "byteorder"
-version = "1.3.4"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
 [[package]]
 name = "cc"
-version = "1.0.62"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40"
+checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
 
 [[package]]
 name = "cfg-if"
@@ -49,16 +49,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -75,8 +75,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -84,18 +84,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -104,45 +104,11 @@ dependencies = [
 ]
 
 [[package]]
-name = "cranelift-module"
+name = "cranelift-jit"
-version = "0.68.0"
+version = "0.69.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
 dependencies = [
  "anyhow",
- "cranelift-codegen",
- "cranelift-entity",
- "log",
- "thiserror",
-]
-
-[[package]]
-name = "cranelift-native"
-version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
-dependencies = [
- "cranelift-codegen",
- "raw-cpuid",
- "target-lexicon",
-]
-
-[[package]]
-name = "cranelift-object"
-version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
-dependencies = [
- "anyhow",
- "cranelift-codegen",
- "cranelift-module",
- "log",
- "object",
- "target-lexicon",
-]
-
-[[package]]
-name = "cranelift-simplejit"
-version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
-dependencies = [
  "cranelift-codegen",
  "cranelift-entity",
  "cranelift-module",
@@ -155,6 +121,41 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "cranelift-module"
+version = "0.69.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-entity",
+ "log",
+ "thiserror",
+]
+
+[[package]]
+name = "cranelift-native"
+version = "0.69.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+dependencies = [
+ "cranelift-codegen",
+ "raw-cpuid",
+ "target-lexicon",
+]
+
+[[package]]
+name = "cranelift-object"
+version = "0.69.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-module",
+ "log",
+ "object",
+ "target-lexicon",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.2.1"
@@ -208,9 +209,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
 [[package]]
 name = "indexmap"
-version = "1.6.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2"
+checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -218,15 +219,15 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.80"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
+checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
 
 [[package]]
 name = "libloading"
-version = "0.6.5"
+version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1090080fe06ec2648d0da3881d9453d97e71a45f00eb179af7fdd7e3f686fdb0"
+checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883"
 dependencies = [
  "cfg-if 1.0.0",
  "winapi",
@@ -234,9 +235,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.11"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
+checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2"
 dependencies = [
  "cfg-if 0.1.10",
 ]
@@ -271,9 +272,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
+checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
 dependencies = [
  "proc-macro2",
 ]
@@ -325,13 +326,14 @@ dependencies = [
  "ar",
  "cranelift-codegen",
  "cranelift-frontend",
+ "cranelift-jit",
  "cranelift-module",
  "cranelift-object",
- "cranelift-simplejit",
  "gimli",
  "indexmap",
  "libloading",
  "object",
+ "smallvec",
  "target-lexicon",
 ]
 
@@ -361,15 +363,15 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "smallvec"
-version = "1.4.2"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
+checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
 [[package]]
 name = "syn"
-version = "1.0.48"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
+checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -384,18 +386,18 @@ checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9"
 
 [[package]]
 name = "thiserror"
-version = "1.0.22"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e"
+checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.22"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56"
+checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
 dependencies = [
  "proc-macro2",
  "quote",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -9,10 +9,10 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] }
+cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x86", "x64"] }
 cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
-cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
+cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
@@ -21,13 +21,14 @@ object = { version = "0.22.0", default-features = false, features = ["std", "rea
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
 libloading = { version = "0.6.0", optional = true }
+smallvec = "1.6.1"
 
 # Uncomment to use local checkout of cranelift
 #[patch."https://github.com/bytecodealliance/wasmtime/"]
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
 #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" }
 #cranelift-module = { path = "../wasmtime/cranelift/module" }
-#cranelift-simplejit = { path = "../wasmtime/cranelift/simplejit" }
+#cranelift-jit = { path = "../wasmtime/cranelift/jit" }
 #cranelift-object = { path = "../wasmtime/cranelift/object" }
 
 #[patch.crates-io]
@@ -35,8 +36,9 @@ libloading = { version = "0.6.0", optional = true }
 
 [features]
 default = ["jit", "inline_asm"]
-jit = ["cranelift-simplejit", "libloading"]
+jit = ["cranelift-jit", "libloading"]
 inline_asm = []
+oldbe = []
 
 [profile.dev]
 # By compiling dependencies with optimizations, performing tests gets much faster.

compiler/rustc_codegen_cranelift/Readme.md

@@ -1,8 +1,6 @@
-# WIP Cranelift codegen backend for rust
+# Cranelift codegen backend for rust
 
-> ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠
+The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift).
-
-The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift).
 This has the potential to improve compilation times in debug mode.
 If your project doesn't use any of the things listed under "Not yet supported", it should work fine.
 If not please open an issue.
@@ -68,7 +66,15 @@ $ $cg_clif_dir/build/cargo.sh jit
 or
 
 ```bash
-$ $cg_clif_dir/build/bin/cg_clif --jit my_crate.rs
+$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+```
+
+There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
+first called. It currently does not work with multi-threaded programs. When a not yet compiled
+function is called from another thread than the main thread, you will get an ICE.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh lazy-jit
 ```
 
 ### Shell
@@ -77,7 +83,7 @@ These are a few functions that allow you to easily run rust code from the shell
 
 ```bash
 function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - --jit
+    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
 }
 
 function jit() {
@@ -95,8 +101,7 @@ function jit_calc() {
 
 ## Not yet supported
 
-* Good non-rust abi support ([several problems](https://github.com/bjorn3/rustc_codegen_cranelift/issues/10))
+* Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041))
-* Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041)
     * On Linux there is support for invoking an external assembler for `global_asm!` and `asm!`.
       `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You
       have to specify specific registers instead.

compiler/rustc_codegen_cranelift/build.sh

@@ -3,23 +3,29 @@ set -e
 
 # Settings
 export CHANNEL="release"
-build_sysroot=1
+build_sysroot="clif"
 target_dir='build'
+oldbe=''
 while [[ $# != 0 ]]; do
     case $1 in
         "--debug")
             export CHANNEL="debug"
             ;;
-        "--without-sysroot")
+        "--sysroot")
-            build_sysroot=0
+            build_sysroot=$2
+            shift
             ;;
         "--target-dir")
             target_dir=$2
             shift
             ;;
+        "--oldbe")
+            oldbe='--features oldbe'
+            ;;
         *)
             echo "Unknown flag '$1'"
-            echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR]"
+            echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--oldbe]"
+            exit 1
             ;;
     esac
     shift
@@ -27,23 +33,24 @@ done
 
 # Build cg_clif
 unset CARGO_TARGET_DIR
-export RUSTFLAGS="-Zrun_dsymutil=no"
 unamestr=$(uname)
 if [[ "$unamestr" == 'Linux' ]]; then
    export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS
 elif [[ "$unamestr" == 'Darwin' ]]; then
-   export RUSTFLAGS='-Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS
+   export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS
    dylib_ext='dylib'
 else
    echo "Unsupported os"
    exit 1
 fi
 if [[ "$CHANNEL" == "release" ]]; then
-    cargo build --release
+    cargo build $oldbe --release
 else
-    cargo build
+    cargo build $oldbe
 fi
 
+source scripts/ext_config.sh
+
 rm -rf "$target_dir"
 mkdir "$target_dir"
 mkdir "$target_dir"/bin "$target_dir"/lib
@@ -51,10 +58,29 @@ ln target/$CHANNEL/cg_clif{,_build_sysroot} "$target_dir"/bin
 ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib
 ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir"
 
-if [[ "$build_sysroot" == "1" ]]; then
+mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
-    echo "[BUILD] sysroot"
+if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
-    export CG_CLIF_INCR_CACHE_DISABLED=1
+    cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
-    dir=$(pwd)
-    cd "$target_dir"
-    time "$dir/build_sysroot/build_sysroot.sh"
 fi
+
+case "$build_sysroot" in
+    "none")
+        ;;
+    "llvm")
+        cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/"
+        ;;
+    "clif")
+        echo "[BUILD] sysroot"
+        dir=$(pwd)
+        cd "$target_dir"
+        time "$dir/build_sysroot/build_sysroot.sh"
+        cp lib/rustlib/*/lib/libstd-* lib/
+        ;;
+    *)
+        echo "Unknown sysroot kind \`$build_sysroot\`."
+        echo "The allowed values are:"
+        echo "    none A sysroot that doesn't contain the standard library"
+        echo "    llvm Copy the sysroot from rustc compiled by cg_llvm"
+        echo "    clif Build a new sysroot using cg_clif"
+        exit 1
+esac

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -2,9 +2,9 @@
 # It is not intended for manual editing.
 [[package]]
 name = "addr2line"
-version = "0.14.0"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423"
+checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
 dependencies = [
  "compiler_builtins",
  "gimli",
@@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "cc"
-version = "1.0.65"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15"
+checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
 
 [[package]]
 name = "cfg-if"
@@ -63,9 +63,7 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.36"
+version = "0.1.39"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cd0782e0a7da7598164153173e5a5d4d9b1da094473c98dce0ff91406112369"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -130,9 +128,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8"
+checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
 dependencies = [
  "compiler_builtins",
  "libc",
@@ -141,9 +139,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.80"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
+checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
 dependencies = [
  "rustc-std-workspace-core",
 ]

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml

@@ -5,17 +5,19 @@ version = "0.0.0"
 
 [dependencies]
 core = { path = "./sysroot_src/library/core" }
-compiler_builtins = "0.1"
 alloc = { path = "./sysroot_src/library/alloc" }
 std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
 test = { path = "./sysroot_src/library/test" }
 
 alloc_system = { path = "./alloc_system" }
 
+compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
+
 [patch.crates-io]
 rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" }
 rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" }
 rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" }
+compiler_builtins = { path = "./compiler-builtins" }
 
 [profile.dev]
 lto = "off"

compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh

@@ -24,17 +24,16 @@ export CARGO_TARGET_DIR=target
 
 # Build libs
 export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort"
+export __CARGO_DEFAULT_LIB_METADATA="cg_clif"
 if [[ "$1" != "--debug" ]]; then
     sysroot_channel='release'
     # FIXME Enable incremental again once rust-lang/rust#74946 is fixed
-    # FIXME Enable -Zmir-opt-level=2 again once it doesn't ice anymore
+    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release
-    CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS" cargo build --target "$TARGET_TRIPLE" --release
 else
     sysroot_channel='debug'
     cargo build --target "$TARGET_TRIPLE"
 fi
 
 # Copy files to sysroot
-mkdir -p "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"
 ln "target/$TARGET_TRIPLE/$sysroot_channel/deps/"* "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"
 rm "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"*.{rmeta,d}

compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh

@@ -29,4 +29,11 @@ git commit --no-gpg-sign -m "Patch $file"
 done
 popd
 
-echo "Successfully prepared libcore for building"
+git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned"
+pushd compiler-builtins
+git checkout -- .
+git checkout 0.1.39
+git apply ../../crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch
+popd
+
+echo "Successfully prepared sysroot source for building"

compiler/rustc_codegen_cranelift/clean_all.sh

@@ -1,5 +1,5 @@
 #!/bin/bash --verbose
 set -e
 
-rm -rf target/ build/ build_sysroot/{sysroot_src/,target/} perf.data{,.old}
+rm -rf target/ build/ build_sysroot/{sysroot_src/,target/,compiler-builtins/} perf.data{,.old}
 rm -rf rand/ regex/ simple-raytracer/

compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch

@@ -0,0 +1,35 @@
+From 7078cca3cb614e1e82da428380b4e16fc3afef46 Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 21 Jan 2021 14:46:36 +0100
+Subject: [PATCH] Remove rotate_left from Int
+
+---
+ src/int/mod.rs | 5 -----
+ 1 file changed, 5 deletions(-)
+
+diff --git a/src/int/mod.rs b/src/int/mod.rs
+index 06054c8..3bea17b 100644
+--- a/src/int/mod.rs
++++ b/src/int/mod.rs
+@@ -85,7 +85,6 @@ pub trait Int:
+     fn wrapping_sub(self, other: Self) -> Self;
+     fn wrapping_shl(self, other: u32) -> Self;
+     fn wrapping_shr(self, other: u32) -> Self;
+-    fn rotate_left(self, other: u32) -> Self;
+     fn overflowing_add(self, other: Self) -> (Self, bool);
+     fn aborting_div(self, other: Self) -> Self;
+     fn aborting_rem(self, other: Self) -> Self;
+@@ -209,10 +208,6 @@ macro_rules! int_impl_common {
+             <Self>::wrapping_shr(self, other)
+         }
+ 
+-        fn rotate_left(self, other: u32) -> Self {
+-            <Self>::rotate_left(self, other)
+-        }
+-
+         fn overflowing_add(self, other: Self) -> (Self, bool) {
+             <Self>::overflowing_add(self, other)
+         }
+-- 
+2.26.2.7.g19db9cfb68
+

compiler/rustc_codegen_cranelift/example/alloc_example.rs

@@ -11,7 +11,8 @@ use alloc_system::System;
 #[global_allocator]
 static ALLOC: System = System;
 
-#[link(name = "c")]
+#[cfg_attr(unix, link(name = "c"))]
+#[cfg_attr(target_env = "msvc", link(name = "msvcrt"))]
 extern "C" {
     fn puts(s: *const u8) -> i32;
 }

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -532,8 +532,8 @@ pub mod intrinsics {
 }
 
 pub mod libc {
-    #[cfg_attr(not(windows), link(name = "c"))]
+    #[cfg_attr(unix, link(name = "c"))]
-    #[cfg_attr(windows, link(name = "msvcrt"))]
+    #[cfg_attr(target_env = "msvc", link(name = "msvcrt"))]
     extern "C" {
         pub fn puts(s: *const i8) -> i32;
         pub fn printf(format: *const i8, ...) -> i32;

compiler/rustc_codegen_cranelift/example/mod_bench.rs

@@ -1,7 +1,8 @@
 #![feature(start, box_syntax, core_intrinsics, lang_items)]
 #![no_std]
 
-#[link(name = "c")]
+#[cfg_attr(unix, link(name = "c"))]
+#[cfg_attr(target_env = "msvc", link(name = "msvcrt"))]
 extern {}
 
 #[panic_handler]

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -15,6 +15,8 @@ fn main() {
     let stderr = ::std::io::stderr();
     let mut stderr = stderr.lock();
 
+    // FIXME support lazy jit when multi threading
+    #[cfg(not(lazy_jit))]
     std::thread::spawn(move || {
         println!("Hello from another thread!");
     });

compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch

@@ -119,5 +119,21 @@ index 6609bc3..241b497 100644
  
  #[test]
  #[should_panic(expected = "index 0 greater than length of slice")]
+diff --git a/library/core/tests/num/ops.rs b/library/core/tests/num/ops.rs
+index 9979cc8..d5d1d83 100644
+--- a/library/core/tests/num/ops.rs
++++ b/library/core/tests/num/ops.rs
+@@ -238,7 +238,7 @@ macro_rules! test_shift_assign {
+         }
+     };
+ }
+-test_shift!(test_shl_defined, Shl::shl);
+-test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
+-test_shift!(test_shr_defined, Shr::shr);
+-test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
++//test_shift!(test_shl_defined, Shl::shl);
++//test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
++//test_shift!(test_shr_defined, Shr::shr);
++//test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
 --
 2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1 +1 @@
-nightly-2020-11-27
+nightly-2021-01-30

compiler/rustc_codegen_cranelift/scripts/cargo.sh

@@ -10,7 +10,9 @@ cmd=$1
 shift || true
 
 if [[ "$cmd" = "jit" ]]; then
-cargo "+${TOOLCHAIN}" rustc "$@" -- --jit
+cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -Cprefer-dynamic
+elif [[ "$cmd" = "lazy-jit" ]]; then
+cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy -Cprefer-dynamic
 else
 cargo "+${TOOLCHAIN}" "$cmd" "$@"
 fi

compiler/rustc_codegen_cranelift/scripts/config.sh

@@ -12,28 +12,6 @@ else
    exit 1
 fi
 
-HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
-TARGET_TRIPLE=$HOST_TRIPLE
-#TARGET_TRIPLE="x86_64-pc-windows-gnu"
-#TARGET_TRIPLE="aarch64-unknown-linux-gnu"
-
-linker=''
-RUN_WRAPPER=''
-export JIT_SUPPORTED=1
-if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
-   export JIT_SUPPORTED=0
-   if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then
-      # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
-      linker='-Clinker=aarch64-linux-gnu-gcc'
-      RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu'
-   elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
-      # We are cross-compiling for Windows. Run tests in wine.
-      RUN_WRAPPER='wine'
-   else
-      echo "Unknown non-native platform"
-   fi
-fi
-
 if echo "$RUSTC_WRAPPER" | grep sccache; then
 echo
 echo -e "\x1b[1;93m=== Warning: Unset RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m"
@@ -44,16 +22,14 @@ fi
 dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 
 export RUSTC=$dir"/bin/cg_clif"
-export RUSTFLAGS=$linker" "$RUSTFLAGS
+
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
 '-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
 
 # FIXME remove once the atomic shim is gone
-if [[ $(uname) == 'Darwin' ]]; then
+if [[ "$unamestr" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
-export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib"
+export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib"
 export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
-
-export CG_CLIF_DISPLAY_CG_TIME=1

compiler/rustc_codegen_cranelift/scripts/ext_config.sh

@@ -0,0 +1,27 @@
+# Note to people running shellcheck: this file should only be sourced, not executed directly.
+
+# Various env vars that should only be set for the build system but not for cargo.sh
+
+set -e
+
+export CG_CLIF_DISPLAY_CG_TIME=1
+export CG_CLIF_INCR_CACHE_DISABLED=1
+
+export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
+export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE}
+
+export RUN_WRAPPER=''
+export JIT_SUPPORTED=1
+if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+   export JIT_SUPPORTED=0
+   if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then
+      # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+      export RUSTFLAGS='-Clinker=aarch64-linux-gnu-gcc '$RUSTFLAGS
+      export RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu'
+   elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
+      # We are cross-compiling for Windows. Run tests in wine.
+      export RUN_WRAPPER='wine'
+   else
+      echo "Unknown non-native platform"
+   fi
+fi

compiler/rustc_codegen_cranelift/scripts/filter_profile.rs

@@ -4,7 +4,7 @@
 pushd $(dirname "$0")/../
 source build/config.sh
 popd
-PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0
+PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0
 #*/
 
 //! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse

compiler/rustc_codegen_cranelift/scripts/tests.sh

@@ -3,7 +3,7 @@
 set -e
 
 source build/config.sh
-export CG_CLIF_INCR_CACHE_DISABLED=1
+source scripts/ext_config.sh
 MY_RUSTC="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
 
 function no_sysroot_tests() {
@@ -15,7 +15,10 @@ function no_sysroot_tests() {
 
     if [[ "$JIT_SUPPORTED" = "1" ]]; then
         echo "[JIT] mini_core_hello_world"
-        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
+        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
+
+        echo "[JIT-lazy] mini_core_hello_world"
+        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
     else
         echo "[JIT] mini_core_hello_world (skipped)"
     fi
@@ -37,7 +40,10 @@ function base_sysroot_tests() {
 
     if [[ "$JIT_SUPPORTED" = "1" ]]; then
         echo "[JIT] std_example"
-        $MY_RUSTC --jit example/std_example.rs --target "$HOST_TRIPLE"
+        $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE"
+
+        echo "[JIT-lazy] std_example"
+        $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --cfg lazy_jit --target "$HOST_TRIPLE"
     else
         echo "[JIT] std_example (skipped)"
     fi

compiler/rustc_codegen_cranelift/src/abi/comments.rs

@@ -4,10 +4,10 @@
 use std::borrow::Cow;
 
 use rustc_middle::mir;
+use rustc_target::abi::call::PassMode;
 
 use cranelift_codegen::entity::EntityRef;
 
-use crate::abi::pass_mode::*;
 use crate::prelude::*;
 
 pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
@@ -21,9 +21,9 @@ pub(super) fn add_arg_comment<'tcx>(
     kind: &str,
     local: Option<mir::Local>,
     local_field: Option<usize>,
-    params: EmptySinglePair<Value>,
+    params: &[Value],
-    pass_mode: PassMode,
+    arg_abi_mode: PassMode,
-    ty: Ty<'tcx>,
+    arg_layout: TyAndLayout<'tcx>,
 ) {
     let local = if let Some(local) = local {
         Cow::Owned(format!("{:?}", local))
@@ -37,12 +37,20 @@ pub(super) fn add_arg_comment<'tcx>(
     };
 
     let params = match params {
-        Empty => Cow::Borrowed("-"),
+        [] => Cow::Borrowed("-"),
-        Single(param) => Cow::Owned(format!("= {:?}", param)),
+        [param] => Cow::Owned(format!("= {:?}", param)),
-        Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)),
+        [param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)),
+        params => Cow::Owned(format!(
+            "= {}",
+            params
+                .iter()
+                .map(ToString::to_string)
+                .collect::<Vec<_>>()
+                .join(",")
+        )),
     };
 
-    let pass_mode = format!("{:?}", pass_mode);
+    let pass_mode = format!("{:?}", arg_abi_mode);
     fx.add_global_comment(format!(
         "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}",
         kind = kind,
@@ -50,7 +58,7 @@ pub(super) fn add_arg_comment<'tcx>(
         local_field = local_field,
         params = params,
         pass_mode = pass_mode,
-        ty = ty,
+        ty = arg_layout.ty,
     ));
 }
 

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -6,199 +6,51 @@ mod pass_mode;
 mod returning;
 
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::ty::layout::FnAbiExt;
+use rustc_target::abi::call::{Conv, FnAbi};
 use rustc_target::spec::abi::Abi;
 
-use cranelift_codegen::ir::{AbiParam, ArgumentPurpose};
+use cranelift_codegen::ir::AbiParam;
+use smallvec::smallvec;
 
 use self::pass_mode::*;
 use crate::prelude::*;
 
 pub(crate) use self::returning::{can_return_to_ssa_var, codegen_return};
 
-// Copied from https://github.com/rust-lang/rust/blob/f52c72948aa1dd718cc1f168d21c91c584c0a662/src/librustc_middle/ty/layout.rs#L2301
+fn clif_sig_from_fn_abi<'tcx>(
-#[rustfmt::skip]
-pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::PolyFnSig<'tcx> {
-    use rustc_middle::ty::subst::Subst;
-
-    // FIXME(davidtwco,eddyb): A `ParamEnv` should be passed through to this function.
-    let ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
-    match *ty.kind() {
-        ty::FnDef(..) => {
-            // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering
-            // parameters unused if they show up in the signature, but not in the `mir::Body`
-            // (i.e. due to being inside a projection that got normalized, see
-            // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping
-            // track of a polymorphization `ParamEnv` to allow normalizing later.
-            let mut sig = match *ty.kind() {
-                ty::FnDef(def_id, substs) => tcx
-                    .normalize_erasing_regions(tcx.param_env(def_id), tcx.fn_sig(def_id))
-                    .subst(tcx, substs),
-                _ => unreachable!(),
-            };
-
-            if let ty::InstanceDef::VtableShim(..) = instance.def {
-                // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
-                sig = sig.map_bound(|mut sig| {
-                    let mut inputs_and_output = sig.inputs_and_output.to_vec();
-                    inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
-                    sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
-                    sig
-                });
-            }
-            sig
-        }
-        ty::Closure(def_id, substs) => {
-            let sig = substs.as_closure().sig();
-
-            let env_ty = tcx.closure_env_ty(def_id, substs).unwrap();
-            sig.map_bound(|sig| {
-                tcx.mk_fn_sig(
-                    std::iter::once(env_ty.skip_binder()).chain(sig.inputs().iter().cloned()),
-                    sig.output(),
-                    sig.c_variadic,
-                    sig.unsafety,
-                    sig.abi,
-                )
-            })
-        }
-        ty::Generator(_, substs, _) => {
-            let sig = substs.as_generator().poly_sig();
-
-            let env_region = ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind: ty::BrEnv });
-            let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
-
-            let pin_did = tcx.require_lang_item(rustc_hir::LangItem::Pin, None);
-            let pin_adt_ref = tcx.adt_def(pin_did);
-            let pin_substs = tcx.intern_substs(&[env_ty.into()]);
-            let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
-
-            sig.map_bound(|sig| {
-                let state_did = tcx.require_lang_item(rustc_hir::LangItem::GeneratorState, None);
-                let state_adt_ref = tcx.adt_def(state_did);
-                let state_substs =
-                    tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
-                let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
-
-                tcx.mk_fn_sig(
-                    [env_ty, sig.resume_ty].iter(),
-                    &ret_ty,
-                    false,
-                    rustc_hir::Unsafety::Normal,
-                    rustc_target::spec::abi::Abi::Rust,
-                )
-            })
-        }
-        _ => bug!("unexpected type {:?} in Instance::fn_sig", ty),
-    }
-}
-
-fn clif_sig_from_fn_sig<'tcx>(
     tcx: TyCtxt<'tcx>,
     triple: &target_lexicon::Triple,
-    sig: FnSig<'tcx>,
+    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
-    span: Span,
-    is_vtable_fn: bool,
-    requires_caller_location: bool,
 ) -> Signature {
-    let abi = match sig.abi {
+    let call_conv = match fn_abi.conv {
-        Abi::System => Abi::C,
+        Conv::Rust | Conv::C => CallConv::triple_default(triple),
-        abi => abi,
+        Conv::X86_64SysV => CallConv::SystemV,
-    };
+        Conv::X86_64Win64 => CallConv::WindowsFastcall,
-    let (call_conv, inputs, output): (CallConv, Vec<Ty<'tcx>>, Ty<'tcx>) = match abi {
+        Conv::ArmAapcs
-        Abi::Rust => (
+        | Conv::CCmseNonSecureCall
-            CallConv::triple_default(triple),
+        | Conv::Msp430Intr
-            sig.inputs().to_vec(),
+        | Conv::PtxKernel
-            sig.output(),
+        | Conv::X86Fastcall
-        ),
+        | Conv::X86Intr
-        Abi::C | Abi::Unadjusted => (
+        | Conv::X86Stdcall
-            CallConv::triple_default(triple),
+        | Conv::X86ThisCall
-            sig.inputs().to_vec(),
+        | Conv::X86VectorCall
-            sig.output(),
+        | Conv::AmdGpuKernel
-        ),
+        | Conv::AvrInterrupt
-        Abi::SysV64 => (CallConv::SystemV, sig.inputs().to_vec(), sig.output()),
+        | Conv::AvrNonBlockingInterrupt => {
-        Abi::RustCall => {
+            todo!("{:?}", fn_abi.conv)
-            assert_eq!(sig.inputs().len(), 2);
-            let extra_args = match sig.inputs().last().unwrap().kind() {
-                ty::Tuple(ref tupled_arguments) => tupled_arguments,
-                _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"),
-            };
-            let mut inputs: Vec<Ty<'tcx>> = vec![sig.inputs()[0]];
-            inputs.extend(extra_args.types());
-            (CallConv::triple_default(triple), inputs, sig.output())
         }
-        Abi::System => unreachable!(),
-        Abi::RustIntrinsic => (
-            CallConv::triple_default(triple),
-            sig.inputs().to_vec(),
-            sig.output(),
-        ),
-        _ => unimplemented!("unsupported abi {:?}", sig.abi),
     };
-
+    let inputs = fn_abi
-    let inputs = inputs
+        .args
-        .into_iter()
+        .iter()
-        .enumerate()
+        .map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter())
-        .map(|(i, ty)| {
-            let mut layout = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap();
-            if i == 0 && is_vtable_fn {
-                // Virtual calls turn their self param into a thin pointer.
-                // See https://github.com/rust-lang/rust/blob/37b6a5e5e82497caf5353d9d856e4eb5d14cbe06/src/librustc/ty/layout.rs#L2519-L2572 for more info
-                layout = tcx
-                    .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit())))
-                    .unwrap();
-            }
-            let pass_mode = get_pass_mode(tcx, layout);
-            if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic {
-                match pass_mode {
-                    PassMode::NoPass | PassMode::ByVal(_) => {}
-                    PassMode::ByRef { size: Some(size) } => {
-                        let purpose = ArgumentPurpose::StructArgument(u32::try_from(size.bytes()).expect("struct too big to pass on stack"));
-                        return EmptySinglePair::Single(AbiParam::special(pointer_ty(tcx), purpose)).into_iter();
-                    }
-                    PassMode::ByValPair(_, _) | PassMode::ByRef { size: None } => {
-                        tcx.sess.span_warn(
-                            span,
-                            &format!(
-                                "Argument of type `{:?}` with pass mode `{:?}` is not yet supported \
-                                for non-rust abi `{}`. Calling this function may result in a crash.",
-                                layout.ty,
-                                pass_mode,
-                                abi,
-                            ),
-                        );
-                    }
-                }
-            }
-            pass_mode.get_param_ty(tcx).map(AbiParam::new).into_iter()
-        })
         .flatten();
 
-    let (mut params, returns): (Vec<_>, Vec<_>) = match get_pass_mode(
+    let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
-        tcx,
+    // Sometimes the first param is an pointer to the place where the return value needs to be stored.
-        tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(),
+    let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
-    ) {
-        PassMode::NoPass => (inputs.collect(), vec![]),
-        PassMode::ByVal(ret_ty) => (inputs.collect(), vec![AbiParam::new(ret_ty)]),
-        PassMode::ByValPair(ret_ty_a, ret_ty_b) => (
-            inputs.collect(),
-            vec![AbiParam::new(ret_ty_a), AbiParam::new(ret_ty_b)],
-        ),
-        PassMode::ByRef { size: Some(_) } => {
-            (
-                Some(pointer_ty(tcx)) // First param is place to put return val
-                    .into_iter()
-                    .map(|ty| AbiParam::special(ty, ArgumentPurpose::StructReturn))
-                    .chain(inputs)
-                    .collect(),
-                vec![],
-            )
-        }
-        PassMode::ByRef { size: None } => todo!(),
-    };
-
-    if requires_caller_location {
-        params.push(AbiParam::new(pointer_ty(tcx)));
-    }
 
     Signature {
         params,
@@ -207,30 +59,17 @@ fn clif_sig_from_fn_sig<'tcx>(
     }
 }
 
-pub(crate) fn get_function_name_and_sig<'tcx>(
+pub(crate) fn get_function_sig<'tcx>(
     tcx: TyCtxt<'tcx>,
     triple: &target_lexicon::Triple,
     inst: Instance<'tcx>,
-    support_vararg: bool,
+) -> Signature {
-) -> (String, Signature) {
     assert!(!inst.substs.needs_infer());
-    let fn_sig = tcx
+    clif_sig_from_fn_abi(
-        .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_sig_for_fn_abi(tcx, inst));
-    if fn_sig.c_variadic && !support_vararg {
-        tcx.sess.span_fatal(
-            tcx.def_span(inst.def_id()),
-            "Variadic function definitions are not yet supported",
-        );
-    }
-    let sig = clif_sig_from_fn_sig(
         tcx,
         triple,
-        fn_sig,
+        &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]),
-        tcx.def_span(inst.def_id()),
+    )
-        false,
-        inst.def.requires_caller_location(tcx),
-    );
-    (tcx.symbol_name(inst).name.to_string(), sig)
 }
 
 /// Instance must be monomorphized
@@ -239,7 +78,8 @@ pub(crate) fn import_function<'tcx>(
     module: &mut impl Module,
     inst: Instance<'tcx>,
 ) -> FuncId {
-    let (name, sig) = get_function_name_and_sig(tcx, module.isa().triple(), inst, true);
+    let name = tcx.symbol_name(inst).name.to_string();
+    let sig = get_function_sig(tcx, module.isa().triple(), inst);
     module
         .declare_function(&name, Linkage::Import, &sig)
         .unwrap()
@@ -263,13 +103,13 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
     pub(crate) fn lib_call(
         &mut self,
         name: &str,
-        input_tys: Vec<types::Type>,
+        params: Vec<AbiParam>,
-        output_tys: Vec<types::Type>,
+        returns: Vec<AbiParam>,
         args: &[Value],
     ) -> &[Value] {
         let sig = Signature {
-            params: input_tys.iter().cloned().map(AbiParam::new).collect(),
+            params,
-            returns: output_tys.iter().cloned().map(AbiParam::new).collect(),
+            returns,
             call_conv: CallConv::triple_default(self.triple()),
         };
         let func_id = self
@@ -301,16 +141,18 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
             .iter()
             .map(|arg| {
                 (
-                    self.clif_type(arg.layout().ty).unwrap(),
+                    AbiParam::new(self.clif_type(arg.layout().ty).unwrap()),
                     arg.load_scalar(self),
                 )
             })
             .unzip();
         let return_layout = self.layout_of(return_ty);
         let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
-            tup.types().map(|ty| self.clif_type(ty).unwrap()).collect()
+            tup.types()
+                .map(|ty| AbiParam::new(self.clif_type(ty).unwrap()))
+                .collect()
         } else {
-            vec![self.clif_type(return_ty).unwrap()]
+            vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
         };
         let ret_vals = self.lib_call(name, input_tys, return_tys, &args);
         match *ret_vals {
@@ -352,12 +194,25 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
     start_block: Block,
 ) {
+    fx.bcx.append_block_params_for_function_params(start_block);
+
+    fx.bcx.switch_to_block(start_block);
+    fx.bcx.ins().nop();
+
     let ssa_analyzed = crate::analyze::analyze(fx);
 
     #[cfg(debug_assertions)]
     self::comments::add_args_header_comment(fx);
 
-    let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, start_block);
+    let mut block_params_iter = fx
+        .bcx
+        .func
+        .dfg
+        .block_params(start_block)
+        .to_vec()
+        .into_iter();
+    let ret_place =
+        self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter);
     assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE);
 
     // None means pass_mode == NoPass
@@ -366,6 +221,9 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
         Spread(Vec<Option<CValue<'tcx>>>),
     }
 
+    let fn_abi = fx.fn_abi.take().unwrap();
+    let mut arg_abis_iter = fn_abi.args.iter();
+
     let func_params = fx
         .mir
         .args_iter()
@@ -385,14 +243,18 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
                 };
 
                 let mut params = Vec::new();
-                for (i, arg_ty) in tupled_arg_tys.types().enumerate() {
+                for (i, _arg_ty) in tupled_arg_tys.types().enumerate() {
-                    let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_ty);
+                    let arg_abi = arg_abis_iter.next().unwrap();
+                    let param =
+                        cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter);
                     params.push(param);
                 }
 
                 (local, ArgKind::Spread(params), arg_ty)
             } else {
-                let param = cvalue_for_param(fx, start_block, Some(local), None, arg_ty);
+                let arg_abi = arg_abis_iter.next().unwrap();
+                let param =
+                    cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter);
                 (local, ArgKind::Normal(param), arg_ty)
             }
         })
@@ -401,13 +263,14 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
     assert!(fx.caller_location.is_none());
     if fx.instance.def.requires_caller_location(fx.tcx) {
         // Store caller location for `#[track_caller]`.
-        fx.caller_location = Some(
+        let arg_abi = arg_abis_iter.next().unwrap();
-            cvalue_for_param(fx, start_block, None, None, fx.tcx.caller_location_ty()).unwrap(),
+        fx.caller_location =
-        );
+            Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap());
     }
 
-    fx.bcx.switch_to_block(start_block);
+    assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind");
-    fx.bcx.ins().nop();
+    fx.fn_abi = Some(fn_abi);
+    assert!(block_params_iter.next().is_none(), "arg_value left behind");
 
     #[cfg(debug_assertions)]
     self::comments::add_locals_header_comment(fx);
@@ -533,6 +396,21 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         None
     };
 
+    let extra_args = &args[fn_sig.inputs().len()..];
+    let extra_args = extra_args
+        .iter()
+        .map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx)))
+        .collect::<Vec<_>>();
+    let fn_abi = if let Some(instance) = instance {
+        FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
+    } else {
+        FnAbi::of_fn_ptr(
+            &RevealAllLayoutCx(fx.tcx),
+            fn_ty.fn_sig(fx.tcx),
+            &extra_args,
+        )
+    };
+
     let is_cold = instance
         .map(|inst| {
             fx.tcx
@@ -570,8 +448,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
     //   | indirect call target
     //   |         | the first argument to be passed
-    //   v         v          v virtual calls are special cased below
+    //   v         v
-    let (func_ref, first_arg, is_virtual_call) = match instance {
+    let (func_ref, first_arg) = match instance {
         // Trait object call
         Some(Instance {
             def: InstanceDef::Virtual(_, idx),
@@ -582,23 +460,19 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(
                     nop_inst,
-                    format!(
+                    format!("virtual call; self arg pass mode: {:?}", &fn_abi.args[0],),
-                        "virtual call; self arg pass mode: {:?}",
-                        get_pass_mode(fx.tcx, args[0].layout())
-                    ),
                 );
             }
             let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0], idx);
-            (Some(method), Single(ptr), true)
+            (Some(method), smallvec![ptr])
         }
 
         // Normal call
         Some(_) => (
             None,
             args.get(0)
-                .map(|arg| adjust_arg_for_abi(fx, *arg))
+                .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0]))
-                .unwrap_or(Empty),
+                .unwrap_or(smallvec![]),
-            false,
         ),
 
         // Indirect call
@@ -612,23 +486,27 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             (
                 Some(func),
                 args.get(0)
-                    .map(|arg| adjust_arg_for_abi(fx, *arg))
+                    .map(|arg| adjust_arg_for_abi(fx, *arg, &fn_abi.args[0]))
-                    .unwrap_or(Empty),
+                    .unwrap_or(smallvec![]),
-                false,
             )
         }
     };
 
     let ret_place = destination.map(|(place, _)| place);
-    let (call_inst, call_args) =
+    let (call_inst, call_args) = self::returning::codegen_with_call_return_arg(
-        self::returning::codegen_with_call_return_arg(fx, fn_sig, ret_place, |fx, return_ptr| {
+        fx,
+        &fn_abi.ret,
+        ret_place,
+        |fx, return_ptr| {
+            let regular_args_count = args.len();
             let mut call_args: Vec<Value> = return_ptr
                 .into_iter()
                 .chain(first_arg.into_iter())
                 .chain(
                     args.into_iter()
+                        .enumerate()
                         .skip(1)
-                        .map(|arg| adjust_arg_for_abi(fx, arg).into_iter())
+                        .map(|(i, arg)| adjust_arg_for_abi(fx, arg, &fn_abi.args[i]).into_iter())
                         .flatten(),
                 )
                 .collect::<Vec<_>>();
@@ -639,18 +517,17 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             {
                 // Pass the caller location for `#[track_caller]`.
                 let caller_location = fx.get_caller_location(span);
-                call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter());
+                call_args.extend(
+                    adjust_arg_for_abi(fx, caller_location, &fn_abi.args[regular_args_count])
+                        .into_iter(),
+                );
+                assert_eq!(fn_abi.args.len(), regular_args_count + 1);
+            } else {
+                assert_eq!(fn_abi.args.len(), regular_args_count);
             }
 
             let call_inst = if let Some(func_ref) = func_ref {
-                let sig = clif_sig_from_fn_sig(
+                let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
-                    fx.tcx,
-                    fx.triple(),
-                    fn_sig,
-                    span,
-                    is_virtual_call,
-                    false, // calls through function pointers never pass the caller location
-                );
                 let sig = fx.bcx.import_signature(sig);
                 fx.bcx.ins().call_indirect(sig, func_ref, &call_args)
             } else {
@@ -660,7 +537,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             };
 
             (call_inst, call_args)
-        });
+        },
+    );
 
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
@@ -701,37 +579,33 @@ pub(crate) fn codegen_drop<'tcx>(
     drop_place: CPlace<'tcx>,
 ) {
     let ty = drop_place.layout().ty;
-    let drop_fn = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
+    let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
 
-    if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def {
+    if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def {
         // we don't actually need to drop anything
     } else {
-        let drop_fn_ty = drop_fn.ty(fx.tcx, ParamEnv::reveal_all());
-        let fn_sig = fx.tcx.normalize_erasing_late_bound_regions(
-            ParamEnv::reveal_all(),
-            drop_fn_ty.fn_sig(fx.tcx),
-        );
-        assert_eq!(fn_sig.output(), fx.tcx.mk_unit());
-
         match ty.kind() {
             ty::Dynamic(..) => {
                 let (ptr, vtable) = drop_place.to_ptr_maybe_unsized();
                 let ptr = ptr.get_addr(fx);
                 let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap());
 
-                let sig = clif_sig_from_fn_sig(
+                // FIXME(eddyb) perhaps move some of this logic into
-                    fx.tcx,
+                // `Instance::resolve_drop_in_place`?
-                    fx.triple(),
+                let virtual_drop = Instance {
-                    fn_sig,
+                    def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
-                    span,
+                    substs: drop_instance.substs,
-                    true,
+                };
-                    false, // `drop_in_place` is never `#[track_caller]`
+                let fn_abi = FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), virtual_drop, &[]);
-                );
+
+                let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
                 let sig = fx.bcx.import_signature(sig);
                 fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
             }
             _ => {
-                assert!(!matches!(drop_fn.def, InstanceDef::Virtual(_, _)));
+                assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _)));
+
+                let fn_abi = FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), drop_instance, &[]);
 
                 let arg_value = drop_place.place_ref(
                     fx,
@@ -743,17 +617,19 @@ pub(crate) fn codegen_drop<'tcx>(
                         },
                     )),
                 );
-                let arg_value = adjust_arg_for_abi(fx, arg_value);
+                let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0]);
 
                 let mut call_args: Vec<Value> = arg_value.into_iter().collect::<Vec<_>>();
 
-                if drop_fn.def.requires_caller_location(fx.tcx) {
+                if drop_instance.def.requires_caller_location(fx.tcx) {
                     // Pass the caller location for `#[track_caller]`.
                     let caller_location = fx.get_caller_location(span);
-                    call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter());
+                    call_args.extend(
+                        adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1]).into_iter(),
+                    );
                 }
 
-                let func_ref = fx.get_function_ref(drop_fn);
+                let func_ref = fx.get_function_ref(drop_instance);
                 fx.bcx.ins().call(func_ref, &call_args);
             }
         }

compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

@@ -1,140 +1,281 @@
 //! Argument passing
 
 use crate::prelude::*;
+use crate::value_and_place::assert_assignable;
 
-pub(super) use EmptySinglePair::*;
+use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose};
+use rustc_target::abi::call::{
+    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind,
+};
+use smallvec::{smallvec, SmallVec};
 
-#[derive(Copy, Clone, Debug)]
+pub(super) trait ArgAbiExt<'tcx> {
-pub(super) enum PassMode {
+    fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>;
-    NoPass,
+    fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>);
-    ByVal(Type),
-    ByValPair(Type, Type),
-    ByRef { size: Option<Size> },
 }
 
-#[derive(Copy, Clone, Debug)]
+fn reg_to_abi_param(reg: Reg) -> AbiParam {
-pub(super) enum EmptySinglePair<T> {
+    let clif_ty = match (reg.kind, reg.size.bytes()) {
-    Empty,
+        (RegKind::Integer, 1) => types::I8,
-    Single(T),
+        (RegKind::Integer, 2) => types::I16,
-    Pair(T, T),
+        (RegKind::Integer, 4) => types::I32,
+        (RegKind::Integer, 8) => types::I64,
+        (RegKind::Integer, 16) => types::I128,
+        (RegKind::Float, 4) => types::F32,
+        (RegKind::Float, 8) => types::F64,
+        (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(),
+        _ => unreachable!("{:?}", reg),
+    };
+    AbiParam::new(clif_ty)
 }
 
-impl<T> EmptySinglePair<T> {
+fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam {
-    pub(super) fn into_iter(self) -> EmptySinglePairIter<T> {
+    match arg_attrs.arg_ext {
-        EmptySinglePairIter(self)
+        RustcArgExtension::None => {}
-    }
+        RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext,
-
+        RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext,
-    pub(super) fn map<U>(self, mut f: impl FnMut(T) -> U) -> EmptySinglePair<U> {
-        match self {
-            Empty => Empty,
-            Single(v) => Single(f(v)),
-            Pair(a, b) => Pair(f(a), f(b)),
-        }
     }
+    param
 }
 
-pub(super) struct EmptySinglePairIter<T>(EmptySinglePair<T>);
+fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> {
-
+    let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
-impl<T> Iterator for EmptySinglePairIter<T> {
+        (0, 0)
-    type Item = T;
-
-    fn next(&mut self) -> Option<T> {
-        match std::mem::replace(&mut self.0, Empty) {
-            Empty => None,
-            Single(v) => Some(v),
-            Pair(a, b) => {
-                self.0 = Single(b);
-                Some(a)
-            }
-        }
-    }
-}
-
-impl<T: std::fmt::Debug> EmptySinglePair<T> {
-    pub(super) fn assert_single(self) -> T {
-        match self {
-            Single(v) => v,
-            _ => panic!("Called assert_single on {:?}", self),
-        }
-    }
-
-    pub(super) fn assert_pair(self) -> (T, T) {
-        match self {
-            Pair(a, b) => (a, b),
-            _ => panic!("Called assert_pair on {:?}", self),
-        }
-    }
-}
-
-impl PassMode {
-    pub(super) fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair<Type> {
-        match self {
-            PassMode::NoPass => Empty,
-            PassMode::ByVal(clif_type) => Single(clif_type),
-            PassMode::ByValPair(a, b) => Pair(a, b),
-            PassMode::ByRef { size: Some(_) } => Single(pointer_ty(tcx)),
-            PassMode::ByRef { size: None } => Pair(pointer_ty(tcx), pointer_ty(tcx)),
-        }
-    }
-}
-
-pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode {
-    if layout.is_zst() {
-        // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer
-        PassMode::NoPass
     } else {
-        match &layout.abi {
+        (
-            Abi::Uninhabited => PassMode::NoPass,
+            cast.rest.total.bytes() / cast.rest.unit.size.bytes(),
-            Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())),
+            cast.rest.total.bytes() % cast.rest.unit.size.bytes(),
-            Abi::ScalarPair(a, b) => {
+        )
-                let a = scalar_to_clif_type(tcx, a.clone());
+    };
-                let b = scalar_to_clif_type(tcx, b.clone());
-                if a == types::I128 && b == types::I128 {
-                    // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are
-                    // available on x86_64. Cranelift gets confused when too many return params
-                    // are used.
-                    PassMode::ByRef {
-                        size: Some(layout.size),
-                    }
-                } else {
-                    PassMode::ByValPair(a, b)
-                }
-            }
 
-            // FIXME implement Vector Abi in a cg_llvm compatible way
+    if cast.prefix.iter().all(|x| x.is_none()) {
-            Abi::Vector { .. } => {
+        // Simplify to a single unit when there is no prefix and size <= unit size
-                if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, layout) {
+        if cast.rest.total <= cast.rest.unit.size {
-                    PassMode::ByVal(vector_ty)
+            let clif_ty = match (cast.rest.unit.kind, cast.rest.unit.size.bytes()) {
-                } else {
+                (RegKind::Integer, 1) => types::I8,
-                    PassMode::ByRef {
+                (RegKind::Integer, 2) => types::I16,
-                        size: Some(layout.size),
+                (RegKind::Integer, 3..=4) => types::I32,
-                    }
+                (RegKind::Integer, 5..=8) => types::I64,
-                }
+                (RegKind::Integer, 9..=16) => types::I128,
-            }
+                (RegKind::Float, 4) => types::F32,
-
+                (RegKind::Float, 8) => types::F64,
-            Abi::Aggregate { sized: true } => PassMode::ByRef {
+                (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(),
-                size: Some(layout.size),
+                _ => unreachable!("{:?}", cast.rest.unit),
-            },
+            };
-            Abi::Aggregate { sized: false } => PassMode::ByRef { size: None },
+            return smallvec![AbiParam::new(clif_ty)];
         }
     }
+
+    // Create list of fields in the main structure
+    let mut args = cast
+        .prefix
+        .iter()
+        .flatten()
+        .map(|&kind| {
+            reg_to_abi_param(Reg {
+                kind,
+                size: cast.prefix_chunk_size,
+            })
+        })
+        .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
+        .collect::<SmallVec<_>>();
+
+    // Append final integer
+    if rem_bytes != 0 {
+        // Only integers can be really split further.
+        assert_eq!(cast.rest.unit.kind, RegKind::Integer);
+        args.push(reg_to_abi_param(Reg {
+            kind: RegKind::Integer,
+            size: Size::from_bytes(rem_bytes),
+        }));
+    }
+
+    args
+}
+
+impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
+    fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> {
+        match self.mode {
+            PassMode::Ignore => smallvec![],
+            PassMode::Direct(attrs) => match &self.layout.abi {
+                Abi::Scalar(scalar) => {
+                    smallvec![apply_arg_attrs_to_abi_param(
+                        AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())),
+                        attrs
+                    )]
+                }
+                Abi::Vector { .. } => {
+                    let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
+                    smallvec![AbiParam::new(vector_ty)]
+                }
+                _ => unreachable!("{:?}", self.layout.abi),
+            },
+            PassMode::Pair(attrs_a, attrs_b) => match &self.layout.abi {
+                Abi::ScalarPair(a, b) => {
+                    let a = scalar_to_clif_type(tcx, a.clone());
+                    let b = scalar_to_clif_type(tcx, b.clone());
+                    smallvec![
+                        apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a),
+                        apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b),
+                    ]
+                }
+                _ => unreachable!("{:?}", self.layout.abi),
+            },
+            PassMode::Cast(cast) => cast_target_to_abi_params(cast),
+            PassMode::Indirect {
+                attrs,
+                extra_attrs: None,
+                on_stack,
+            } => {
+                if on_stack {
+                    let size = u32::try_from(self.layout.size.bytes()).unwrap();
+                    smallvec![apply_arg_attrs_to_abi_param(
+                        AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),),
+                        attrs
+                    )]
+                } else {
+                    smallvec![apply_arg_attrs_to_abi_param(
+                        AbiParam::new(pointer_ty(tcx)),
+                        attrs
+                    )]
+                }
+            }
+            PassMode::Indirect {
+                attrs,
+                extra_attrs: Some(extra_attrs),
+                on_stack,
+            } => {
+                assert!(!on_stack);
+                smallvec![
+                    apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
+                    apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), extra_attrs),
+                ]
+            }
+        }
+    }
+
+    fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>) {
+        match self.mode {
+            PassMode::Ignore => (None, vec![]),
+            PassMode::Direct(_) => match &self.layout.abi {
+                Abi::Scalar(scalar) => (
+                    None,
+                    vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))],
+                ),
+                Abi::Vector { .. } => {
+                    let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
+                    (None, vec![AbiParam::new(vector_ty)])
+                }
+                _ => unreachable!("{:?}", self.layout.abi),
+            },
+            PassMode::Pair(_, _) => match &self.layout.abi {
+                Abi::ScalarPair(a, b) => {
+                    let a = scalar_to_clif_type(tcx, a.clone());
+                    let b = scalar_to_clif_type(tcx, b.clone());
+                    (None, vec![AbiParam::new(a), AbiParam::new(b)])
+                }
+                _ => unreachable!("{:?}", self.layout.abi),
+            },
+            PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()),
+            PassMode::Indirect {
+                attrs: _,
+                extra_attrs: None,
+                on_stack,
+            } => {
+                assert!(!on_stack);
+                (
+                    Some(AbiParam::special(
+                        pointer_ty(tcx),
+                        ArgumentPurpose::StructReturn,
+                    )),
+                    vec![],
+                )
+            }
+            PassMode::Indirect {
+                attrs: _,
+                extra_attrs: Some(_),
+                on_stack: _,
+            } => unreachable!("unsized return value"),
+        }
+    }
+}
+
+pub(super) fn to_casted_value<'tcx>(
+    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    arg: CValue<'tcx>,
+    cast: CastTarget,
+) -> SmallVec<[Value; 2]> {
+    let (ptr, meta) = arg.force_stack(fx);
+    assert!(meta.is_none());
+    let mut offset = 0;
+    cast_target_to_abi_params(cast)
+        .into_iter()
+        .map(|param| {
+            let val = ptr
+                .offset_i64(fx, offset)
+                .load(fx, param.value_type, MemFlags::new());
+            offset += i64::from(param.value_type.bytes());
+            val
+        })
+        .collect()
+}
+
+pub(super) fn from_casted_value<'tcx>(
+    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    block_params: &[Value],
+    layout: TyAndLayout<'tcx>,
+    cast: CastTarget,
+) -> CValue<'tcx> {
+    let abi_params = cast_target_to_abi_params(cast);
+    let abi_param_size: u32 = abi_params
+        .iter()
+        .map(|param| param.value_type.bytes())
+        .sum();
+    let layout_size = u32::try_from(layout.size.bytes()).unwrap();
+    let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+        kind: StackSlotKind::ExplicitSlot,
+        // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+        // specify stack slot alignment.
+        // Stack slot size may be bigger for for example `[u8; 3]` which is packed into an `i32`.
+        // It may also be smaller for example when the type is a wrapper around an integer with a
+        // larger alignment than the integer.
+        size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
+        offset: None,
+    });
+    let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
+    let mut offset = 0;
+    let mut block_params_iter = block_params.into_iter().copied();
+    for param in abi_params {
+        let val = ptr.offset_i64(fx, offset).store(
+            fx,
+            block_params_iter.next().unwrap(),
+            MemFlags::new(),
+        );
+        offset += i64::from(param.value_type.bytes());
+        val
+    }
+    assert_eq!(block_params_iter.next(), None, "Leftover block param");
+    CValue::by_ref(ptr, layout)
 }
 
 /// Get a set of values to be passed as function arguments.
 pub(super) fn adjust_arg_for_abi<'tcx>(
     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
     arg: CValue<'tcx>,
-) -> EmptySinglePair<Value> {
+    arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
-    match get_pass_mode(fx.tcx, arg.layout()) {
+) -> SmallVec<[Value; 2]> {
-        PassMode::NoPass => Empty,
+    assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty);
-        PassMode::ByVal(_) => Single(arg.load_scalar(fx)),
+    match arg_abi.mode {
-        PassMode::ByValPair(_, _) => {
+        PassMode::Ignore => smallvec![],
+        PassMode::Direct(_) => smallvec![arg.load_scalar(fx)],
+        PassMode::Pair(_, _) => {
             let (a, b) = arg.load_scalar_pair(fx);
-            Pair(a, b)
+            smallvec![a, b]
         }
-        PassMode::ByRef { size: _ } => match arg.force_stack(fx) {
+        PassMode::Cast(cast) => to_casted_value(fx, arg, cast),
-            (ptr, None) => Single(ptr.get_addr(fx)),
+        PassMode::Indirect { .. } => match arg.force_stack(fx) {
-            (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta),
+            (ptr, None) => smallvec![ptr.get_addr(fx)],
+            (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta],
         },
     }
 }
@@ -143,20 +284,23 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    start_block: Block,
     #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
     #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
-    arg_ty: Ty<'tcx>,
+    arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
+    block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> Option<CValue<'tcx>> {
-    let layout = fx.layout_of(arg_ty);
+    let block_params = arg_abi
-    let pass_mode = get_pass_mode(fx.tcx, layout);
+        .get_abi_param(fx.tcx)
-
+        .into_iter()
-    if let PassMode::NoPass = pass_mode {
+        .map(|abi_param| {
-        return None;
+            let block_param = block_params_iter.next().unwrap();
-    }
+            assert_eq!(
-
+                fx.bcx.func.dfg.value_type(block_param),
-    let clif_types = pass_mode.get_param_ty(fx.tcx);
+                abi_param.value_type
-    let block_params = clif_types.map(|t| fx.bcx.append_block_param(start_block, t));
+            );
+            block_param
+        })
+        .collect::<SmallVec<[_; 2]>>();
 
     #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
@@ -164,25 +308,48 @@ pub(super) fn cvalue_for_param<'tcx>(
         "arg",
         local,
         local_field,
-        block_params,
+        &block_params,
-        pass_mode,
+        arg_abi.mode,
-        arg_ty,
+        arg_abi.layout,
     );
 
-    match pass_mode {
+    match arg_abi.mode {
-        PassMode::NoPass => unreachable!(),
+        PassMode::Ignore => None,
-        PassMode::ByVal(_) => Some(CValue::by_val(block_params.assert_single(), layout)),
+        PassMode::Direct(_) => {
-        PassMode::ByValPair(_, _) => {
+            assert_eq!(block_params.len(), 1, "{:?}", block_params);
-            let (a, b) = block_params.assert_pair();
+            Some(CValue::by_val(block_params[0], arg_abi.layout))
-            Some(CValue::by_val_pair(a, b, layout))
         }
-        PassMode::ByRef { size: Some(_) } => Some(CValue::by_ref(
+        PassMode::Pair(_, _) => {
-            Pointer::new(block_params.assert_single()),
+            assert_eq!(block_params.len(), 2, "{:?}", block_params);
-            layout,
+            Some(CValue::by_val_pair(
-        )),
+                block_params[0],
-        PassMode::ByRef { size: None } => {
+                block_params[1],
-            let (ptr, meta) = block_params.assert_pair();
+                arg_abi.layout,
-            Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout))
+            ))
+        }
+        PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)),
+        PassMode::Indirect {
+            attrs: _,
+            extra_attrs: None,
+            on_stack: _,
+        } => {
+            assert_eq!(block_params.len(), 1, "{:?}", block_params);
+            Some(CValue::by_ref(
+                Pointer::new(block_params[0]),
+                arg_abi.layout,
+            ))
+        }
+        PassMode::Indirect {
+            attrs: _,
+            extra_attrs: Some(_),
+            on_stack: _,
+        } => {
+            assert_eq!(block_params.len(), 2, "{:?}", block_params);
+            Some(CValue::by_ref_unsized(
+                Pointer::new(block_params[0]),
+                block_params[1],
+                arg_abi.layout,
+            ))
         }
     }
 }

compiler/rustc_codegen_cranelift/src/abi/returning.rs

@@ -1,21 +1,57 @@
 //! Return value handling
 
-use crate::abi::pass_mode::*;
 use crate::prelude::*;
 
-fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> {
+use rustc_middle::ty::layout::FnAbiExt;
-    fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty))
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
-}
+use smallvec::{smallvec, SmallVec};
 
 /// Can the given type be returned into an ssa var or does it need to be returned on the stack.
 pub(crate) fn can_return_to_ssa_var<'tcx>(
-    tcx: TyCtxt<'tcx>,
+    fx: &FunctionCx<'_, 'tcx, impl Module>,
-    dest_layout: TyAndLayout<'tcx>,
+    func: &mir::Operand<'tcx>,
+    args: &[mir::Operand<'tcx>],
 ) -> bool {
-    match get_pass_mode(tcx, dest_layout) {
+    let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
-        PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => true,
+    let fn_sig = fx
-        // FIXME Make it possible to return ByRef to an ssa var.
+        .tcx
-        PassMode::ByRef { size: _ } => false,
+        .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
+
+    // Handle special calls like instrinsics and empty drop glue.
+    let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
+        let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs)
+            .unwrap()
+            .unwrap()
+            .polymorphize(fx.tcx);
+
+        match instance.def {
+            InstanceDef::Intrinsic(_) | InstanceDef::DropGlue(_, _) => {
+                return true;
+            }
+            _ => Some(instance),
+        }
+    } else {
+        None
+    };
+
+    let extra_args = &args[fn_sig.inputs().len()..];
+    let extra_args = extra_args
+        .iter()
+        .map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx)))
+        .collect::<Vec<_>>();
+    let fn_abi = if let Some(instance) = instance {
+        FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
+    } else {
+        FnAbi::of_fn_ptr(
+            &RevealAllLayoutCx(fx.tcx),
+            fn_ty.fn_sig(fx.tcx),
+            &extra_args,
+        )
+    };
+    match fn_abi.ret.mode {
+        PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => true,
+        // FIXME Make it possible to return Cast and Indirect to an ssa var.
+        PassMode::Cast(_) | PassMode::Indirect { .. } => false,
     }
 }
 
@@ -24,27 +60,45 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
 pub(super) fn codegen_return_param<'tcx>(
     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
     ssa_analyzed: &rustc_index::vec::IndexVec<Local, crate::analyze::SsaKind>,
-    start_block: Block,
+    block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> CPlace<'tcx> {
-    let ret_layout = return_layout(fx);
+    let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
-    let ret_pass_mode = get_pass_mode(fx.tcx, ret_layout);
+        PassMode::Ignore => (
-    let (ret_place, ret_param) = match ret_pass_mode {
+            CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout),
-        PassMode::NoPass => (CPlace::no_place(ret_layout), Empty),
+            smallvec![],
-        PassMode::ByVal(_) | PassMode::ByValPair(_, _) => {
+        ),
+        PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => {
             let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
             (
-                super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa),
+                super::make_local_place(
-                Empty,
+                    fx,
+                    RETURN_PLACE,
+                    fx.fn_abi.as_ref().unwrap().ret.layout,
+                    is_ssa,
+                ),
+                smallvec![],
             )
         }
-        PassMode::ByRef { size: Some(_) } => {
+        PassMode::Indirect {
-            let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type);
+            attrs: _,
+            extra_attrs: None,
+            on_stack: _,
+        } => {
+            let ret_param = block_params_iter.next().unwrap();
+            assert_eq!(fx.bcx.func.dfg.value_type(ret_param), pointer_ty(fx.tcx));
             (
-                CPlace::for_ptr(Pointer::new(ret_param), ret_layout),
+                CPlace::for_ptr(
-                Single(ret_param),
+                    Pointer::new(ret_param),
+                    fx.fn_abi.as_ref().unwrap().ret.layout,
+                ),
+                smallvec![ret_param],
             )
         }
-        PassMode::ByRef { size: None } => todo!(),
+        PassMode::Indirect {
+            attrs: _,
+            extra_attrs: Some(_),
+            on_stack: _,
+        } => unreachable!("unsized return value"),
     };
 
     #[cfg(not(debug_assertions))]
@@ -56,9 +110,9 @@ pub(super) fn codegen_return_param<'tcx>(
         "ret",
         Some(RETURN_PLACE),
         None,
-        ret_param,
+        &ret_param,
-        ret_pass_mode,
+        fx.fn_abi.as_ref().unwrap().ret.mode,
-        ret_layout.ty,
+        fx.fn_abi.as_ref().unwrap().ret.layout,
     );
 
     ret_place
@@ -68,42 +122,71 @@ pub(super) fn codegen_return_param<'tcx>(
 /// returns the call return value(s) if any are written to the correct place.
 pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
     fx: &mut FunctionCx<'_, 'tcx, M>,
-    fn_sig: FnSig<'tcx>,
+    ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     ret_place: Option<CPlace<'tcx>>,
     f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option<Value>) -> (Inst, T),
 ) -> (Inst, T) {
-    let ret_layout = fx.layout_of(fn_sig.output());
+    let return_ptr = match ret_arg_abi.mode {
-
+        PassMode::Ignore => None,
-    let output_pass_mode = get_pass_mode(fx.tcx, ret_layout);
+        PassMode::Indirect {
-    let return_ptr = match output_pass_mode {
+            attrs: _,
-        PassMode::NoPass => None,
+            extra_attrs: None,
-        PassMode::ByRef { size: Some(_) } => match ret_place {
+            on_stack: _,
+        } => match ret_place {
             Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)),
             None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot
         },
-        PassMode::ByRef { size: None } => todo!(),
+        PassMode::Indirect {
-        PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None,
+            attrs: _,
+            extra_attrs: Some(_),
+            on_stack: _,
+        } => unreachable!("unsized return value"),
+        PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => None,
     };
 
     let (call_inst, meta) = f(fx, return_ptr);
 
-    match output_pass_mode {
+    match ret_arg_abi.mode {
-        PassMode::NoPass => {}
+        PassMode::Ignore => {}
-        PassMode::ByVal(_) => {
+        PassMode::Direct(_) => {
             if let Some(ret_place) = ret_place {
                 let ret_val = fx.bcx.inst_results(call_inst)[0];
-                ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout));
+                ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout));
             }
         }
-        PassMode::ByValPair(_, _) => {
+        PassMode::Pair(_, _) => {
             if let Some(ret_place) = ret_place {
                 let ret_val_a = fx.bcx.inst_results(call_inst)[0];
                 let ret_val_b = fx.bcx.inst_results(call_inst)[1];
-                ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout));
+                ret_place.write_cvalue(
+                    fx,
+                    CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout),
+                );
             }
         }
-        PassMode::ByRef { size: Some(_) } => {}
+        PassMode::Cast(cast) => {
-        PassMode::ByRef { size: None } => todo!(),
+            if let Some(ret_place) = ret_place {
+                let results = fx
+                    .bcx
+                    .inst_results(call_inst)
+                    .into_iter()
+                    .copied()
+                    .collect::<SmallVec<[Value; 2]>>();
+                let result =
+                    super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast);
+                ret_place.write_cvalue(fx, result);
+            }
+        }
+        PassMode::Indirect {
+            attrs: _,
+            extra_attrs: None,
+            on_stack: _,
+        } => {}
+        PassMode::Indirect {
+            attrs: _,
+            extra_attrs: Some(_),
+            on_stack: _,
+        } => unreachable!("unsized return value"),
     }
 
     (call_inst, meta)
@@ -111,20 +194,35 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
 
 /// Codegen a return instruction with the right return value(s) if any.
 pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    match get_pass_mode(fx.tcx, return_layout(fx)) {
+    match fx.fn_abi.as_ref().unwrap().ret.mode {
-        PassMode::NoPass | PassMode::ByRef { size: Some(_) } => {
+        PassMode::Ignore
+        | PassMode::Indirect {
+            attrs: _,
+            extra_attrs: None,
+            on_stack: _,
+        } => {
             fx.bcx.ins().return_(&[]);
         }
-        PassMode::ByRef { size: None } => todo!(),
+        PassMode::Indirect {
-        PassMode::ByVal(_) => {
+            attrs: _,
+            extra_attrs: Some(_),
+            on_stack: _,
+        } => unreachable!("unsized return value"),
+        PassMode::Direct(_) => {
             let place = fx.get_local_place(RETURN_PLACE);
             let ret_val = place.to_cvalue(fx).load_scalar(fx);
             fx.bcx.ins().return_(&[ret_val]);
         }
-        PassMode::ByValPair(_, _) => {
+        PassMode::Pair(_, _) => {
             let place = fx.get_local_place(RETURN_PLACE);
             let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx);
             fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);
         }
+        PassMode::Cast(cast) => {
+            let place = fx.get_local_place(RETURN_PLACE);
+            let ret_val = place.to_cvalue(fx);
+            let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast);
+            fx.bcx.ins().return_(&ret_vals);
+        }
     }
 }

compiler/rustc_codegen_cranelift/src/analyze.rs

@@ -40,11 +40,14 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, S
         }
 
         match &bb.terminator().kind {
-            TerminatorKind::Call { destination, .. } => {
+            TerminatorKind::Call {
+                destination,
+                func,
+                args,
+                ..
+            } => {
                 if let Some((dest_place, _dest_bb)) = destination {
-                    let dest_layout = fx
+                    if !crate::abi::can_return_to_ssa_var(fx, func, args) {
-                        .layout_of(fx.monomorphize(&dest_place.ty(&fx.mir.local_decls, fx.tcx).ty));
-                    if !crate::abi::can_return_to_ssa_var(fx.tcx, dest_layout) {
                         not_ssa(&mut flag_map, dest_place.local)
                     }
                 }

compiler/rustc_codegen_cranelift/src/backend.rs

@@ -162,7 +162,7 @@ impl AddConstructor for ObjectProduct {
 }
 
 pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
-    let triple = crate::build_isa(sess, true).triple().clone();
+    let triple = crate::build_isa(sess).triple().clone();
 
     let binary_format = match triple.binary_format {
         target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
@@ -193,7 +193,7 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
 
 pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule {
     let mut builder = ObjectBuilder::new(
-        crate::build_isa(sess, true),
+        crate::build_isa(sess),
         name + ".o",
         cranelift_module::default_libcall_names(),
     )

compiler/rustc_codegen_cranelift/src/base.rs

@@ -2,6 +2,8 @@
 
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
+use rustc_middle::ty::layout::FnAbiExt;
+use rustc_target::abi::call::FnAbi;
 
 use crate::prelude::*;
 
@@ -19,7 +21,8 @@ pub(crate) fn codegen_fn<'tcx>(
     let mir = tcx.instance_mir(instance.def);
 
     // Declare function
-    let (name, sig) = get_function_name_and_sig(tcx, cx.module.isa().triple(), instance, false);
+    let name = tcx.symbol_name(instance).name.to_string();
+    let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
     let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
 
     cx.cached_context.clear();
@@ -50,6 +53,7 @@ pub(crate) fn codegen_fn<'tcx>(
 
         instance,
         mir,
+        fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])),
 
         bcx,
         block_map,
@@ -117,6 +121,11 @@ pub(crate) fn codegen_fn<'tcx>(
     context.compute_domtree();
     context.eliminate_unreachable_code(cx.module.isa()).unwrap();
     context.dce(cx.module.isa()).unwrap();
+    // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
+    // invalidate it when it would change.
+    context.domtree.clear();
+
+    context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
 
     // Define function
     let module = &mut cx.module;
@@ -140,6 +149,16 @@ pub(crate) fn codegen_fn<'tcx>(
         &clif_comments,
     );
 
+    if let Some(mach_compile_result) = &context.mach_compile_result {
+        if let Some(disasm) = &mach_compile_result.disasm {
+            crate::pretty_clif::write_ir_file(
+                tcx,
+                &format!("{}.vcode", tcx.symbol_name(instance).name),
+                |file| file.write_all(disasm.as_bytes()),
+            )
+        }
+    }
+
     // Define debuginfo for function
     let isa = cx.module.isa();
     let debug_context = &mut cx.debug_context;
@@ -307,7 +326,9 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             } => {
                 let discr = codegen_operand(fx, discr).load_scalar(fx);
 
-                if switch_ty.kind() == fx.tcx.types.bool.kind() {
+                let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
+                    || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0);
+                if use_bool_opt {
                     assert_eq!(targets.iter().count(), 1);
                     let (then_value, then_block) = targets.iter().next().unwrap();
                     let then_block = fx.get_block(then_block);
@@ -325,12 +346,22 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
                     let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr);
                     let discr =
                         crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr);
-                    if test_zero {
+                    if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken(
-                        fx.bcx.ins().brz(discr, then_block, &[]);
+                        &fx.bcx, discr, test_zero,
-                        fx.bcx.ins().jump(else_block, &[]);
+                    ) {
+                        if taken {
+                            fx.bcx.ins().jump(then_block, &[]);
+                        } else {
+                            fx.bcx.ins().jump(else_block, &[]);
+                        }
                     } else {
-                        fx.bcx.ins().brnz(discr, then_block, &[]);
+                        if test_zero {
-                        fx.bcx.ins().jump(else_block, &[]);
+                            fx.bcx.ins().brz(discr, then_block, &[]);
+                            fx.bcx.ins().jump(else_block, &[]);
+                        } else {
+                            fx.bcx.ins().brnz(discr, then_block, &[]);
+                            fx.bcx.ins().jump(else_block, &[]);
+                        }
                     }
                 } else {
                     let mut switch = ::cranelift_frontend::Switch::new();
@@ -1029,7 +1060,11 @@ pub(crate) fn codegen_panic_inner<'tcx>(
 
     fx.lib_call(
         &*symbol_name,
-        vec![fx.pointer_type, fx.pointer_type, fx.pointer_type],
+        vec![
+            AbiParam::new(fx.pointer_type),
+            AbiParam::new(fx.pointer_type),
+            AbiParam::new(fx.pointer_type),
+        ],
         vec![],
         args,
     );

compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs

@@ -6,7 +6,7 @@ extern crate rustc_interface;
 extern crate rustc_session;
 extern crate rustc_target;
 
-use rustc_data_structures::profiling::print_time_passes_entry;
+use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
 use rustc_interface::interface;
 use rustc_session::config::ErrorOutputType;
 use rustc_session::early_error;
@@ -39,14 +39,13 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
 }
 
 fn main() {
-    let start = std::time::Instant::now();
+    let start_time = std::time::Instant::now();
+    let start_rss = get_resident_set_size();
     rustc_driver::init_rustc_env_logger();
     let mut callbacks = CraneliftPassesCallbacks::default();
     rustc_driver::install_ice_hook();
     let exit_code = rustc_driver::catch_with_exit_code(|| {
-        let mut use_jit = false;
+        let args = std::env::args_os()
-
-        let mut args = std::env::args_os()
             .enumerate()
             .map(|(i, arg)| {
                 arg.into_string().unwrap_or_else(|arg| {
@@ -56,27 +55,18 @@ fn main() {
                     )
                 })
             })
-            .filter(|arg| {
-                if arg == "--jit" {
-                    use_jit = true;
-                    false
-                } else {
-                    true
-                }
-            })
             .collect::<Vec<_>>();
-        if use_jit {
-            args.push("-Cprefer-dynamic".to_string());
-        }
         let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
         run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
-            Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend {
+            Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
-                config: rustc_codegen_cranelift::BackendConfig { use_jit },
-            })
         })));
         run_compiler.run()
     });
-    // The extra `\t` is necessary to align this label with the others.
+
-    print_time_passes_entry(callbacks.time_passes, "\ttotal", start.elapsed());
+    if callbacks.time_passes {
+        let end_rss = get_resident_set_size();
+        print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss);
+    }
+
     std::process::exit(exit_code)
 }

compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs

@@ -53,10 +53,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
                 .unwrap()
                 .parent()
                 .unwrap()
-                .parent()
+                .to_owned(),
-                .unwrap()
-                .join("build_sysroot")
-                .join("sysroot"),
         );
     }
 }
@@ -92,9 +89,7 @@ fn main() {
         let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
         if use_clif {
             run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
-                Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend {
+                Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
-                    config: rustc_codegen_cranelift::BackendConfig { use_jit: false },
-                })
             })));
         }
         run_compiler.run()

compiler/rustc_codegen_cranelift/src/codegen_i128.rs

@@ -1,5 +1,7 @@
 //! Replaces 128-bit operators with lang item calls where necessary
 
+use cranelift_codegen::ir::ArgumentPurpose;
+
 use crate::prelude::*;
 
 pub(crate) fn maybe_codegen<'tcx>(
@@ -24,41 +26,41 @@ pub(crate) fn maybe_codegen<'tcx>(
             None
         }
         BinOp::Add | BinOp::Sub if !checked => None,
-        BinOp::Add => {
+        BinOp::Mul if !checked => {
-            let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
+            let val_ty = if is_signed {
-            return Some(if is_signed {
+                fx.tcx.types.i128
-                fx.easy_call("__rust_i128_addo", &[lhs, rhs], out_ty)
             } else {
-                fx.easy_call("__rust_u128_addo", &[lhs, rhs], out_ty)
+                fx.tcx.types.u128
-            });
+            };
+            Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
         }
-        BinOp::Sub => {
+        BinOp::Add | BinOp::Sub | BinOp::Mul => {
+            assert!(checked);
             let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
-            return Some(if is_signed {
+            let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-                fx.easy_call("__rust_i128_subo", &[lhs, rhs], out_ty)
+            let param_types = vec![
-            } else {
+                AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
-                fx.easy_call("__rust_u128_subo", &[lhs, rhs], out_ty)
+                AbiParam::new(types::I128),
-            });
+                AbiParam::new(types::I128),
+            ];
+            let args = [
+                out_place.to_ptr().get_addr(fx),
+                lhs.load_scalar(fx),
+                rhs.load_scalar(fx),
+            ];
+            let name = match (bin_op, is_signed) {
+                (BinOp::Add, false) => "__rust_u128_addo",
+                (BinOp::Add, true) => "__rust_i128_addo",
+                (BinOp::Sub, false) => "__rust_u128_subo",
+                (BinOp::Sub, true) => "__rust_i128_subo",
+                (BinOp::Mul, false) => "__rust_u128_mulo",
+                (BinOp::Mul, true) => "__rust_i128_mulo",
+                _ => unreachable!(),
+            };
+            fx.lib_call(name, param_types, vec![], &args);
+            Some(out_place.to_cvalue(fx))
         }
         BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
-        BinOp::Mul => {
-            let res = if checked {
-                let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
-                if is_signed {
-                    fx.easy_call("__rust_i128_mulo", &[lhs, rhs], out_ty)
-                } else {
-                    fx.easy_call("__rust_u128_mulo", &[lhs, rhs], out_ty)
-                }
-            } else {
-                let val_ty = if is_signed {
-                    fx.tcx.types.i128
-                } else {
-                    fx.tcx.types.u128
-                };
-                fx.easy_call("__multi3", &[lhs, rhs], val_ty)
-            };
-            Some(res)
-        }
         BinOp::Div => {
             assert!(!checked);
             if is_signed {

compiler/rustc_codegen_cranelift/src/common.rs

@@ -1,4 +1,5 @@
 use rustc_index::vec::IndexVec;
+use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{Integer, Primitive};
 use rustc_target::spec::{HasTargetSpec, Target};
 
@@ -294,6 +295,7 @@ pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
 
     pub(crate) instance: Instance<'tcx>,
     pub(crate) mir: &'tcx Body<'tcx>,
+    pub(crate) fn_abi: Option<FnAbi<'tcx, Ty<'tcx>>>,
 
     pub(crate) bcx: FunctionBuilder<'clif>,
     pub(crate) block_map: IndexVec<BasicBlock, Block>,
@@ -319,16 +321,7 @@ impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
     type TyAndLayout = TyAndLayout<'tcx>;
 
     fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> {
-        assert!(!ty.still_further_specializable());
+        RevealAllLayoutCx(self.tcx).layout_of(ty)
-        self.tcx
-            .layout_of(ParamEnv::reveal_all().and(&ty))
-            .unwrap_or_else(|e| {
-                if let layout::LayoutError::SizeOverflow(_) = e {
-                    self.tcx.sess.fatal(&e.to_string())
-                } else {
-                    bug!("failed to get layout for `{}`: {}", ty, e)
-                }
-            })
     }
 }
 
@@ -442,3 +435,47 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         self.bcx.ins().global_value(self.pointer_type, local_msg_id)
     }
 }
+
+pub(crate) struct RevealAllLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>);
+
+impl<'tcx> LayoutOf for RevealAllLayoutCx<'tcx> {
+    type Ty = Ty<'tcx>;
+    type TyAndLayout = TyAndLayout<'tcx>;
+
+    fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> {
+        assert!(!ty.still_further_specializable());
+        self.0
+            .layout_of(ParamEnv::reveal_all().and(&ty))
+            .unwrap_or_else(|e| {
+                if let layout::LayoutError::SizeOverflow(_) = e {
+                    self.0.sess.fatal(&e.to_string())
+                } else {
+                    bug!("failed to get layout for `{}`: {}", ty, e)
+                }
+            })
+    }
+}
+
+impl<'tcx> layout::HasTyCtxt<'tcx> for RevealAllLayoutCx<'tcx> {
+    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+        self.0
+    }
+}
+
+impl<'tcx> rustc_target::abi::HasDataLayout for RevealAllLayoutCx<'tcx> {
+    fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout {
+        &self.0.data_layout
+    }
+}
+
+impl<'tcx> layout::HasParamEnv<'tcx> for RevealAllLayoutCx<'tcx> {
+    fn param_env(&self) -> ParamEnv<'tcx> {
+        ParamEnv::reveal_all()
+    }
+}
+
+impl<'tcx> HasTargetSpec for RevealAllLayoutCx<'tcx> {
+    fn target_spec(&self) -> &Target {
+        &self.0.sess.target
+    }
+}

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -100,7 +100,10 @@ fn codegen_static_ref<'tcx>(
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     assert!(!layout.is_unsized(), "unsized statics aren't supported");
     assert!(
-        matches!(fx.bcx.func.global_values[local_data_id], GlobalValueData::Symbol { tls: false, ..}),
+        matches!(
+            fx.bcx.func.global_values[local_data_id],
+            GlobalValueData::Symbol { tls: false, .. }
+        ),
         "tls static referenced without Rvalue::ThreadLocalRef"
     );
     CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout)
@@ -131,11 +134,9 @@ pub(crate) fn codegen_constant<'tcx>(
             {
                 Ok(const_val) => const_val,
                 Err(_) => {
-                    if promoted.is_none() {
+                    fx.tcx
-                        fx.tcx
+                        .sess
-                            .sess
+                        .span_err(constant.span, "erroneous constant encountered");
-                            .span_err(constant.span, "erroneous constant encountered");
-                    }
                     return crate::trap::trap_unreachable_ret_value(
                         fx,
                         fx.layout_of(const_.ty),
@@ -447,7 +448,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
             data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64);
         }
 
-        module.define_data(data_id, &data_ctx).unwrap();
+        // FIXME don't duplicate definitions in lazy jit mode
+        let _ = module.define_data(data_id, &data_ctx);
         cx.done.insert(data_id);
     }
 

compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs

@@ -74,10 +74,7 @@ impl WriterRelocate {
 
     /// Perform the collected relocations to be usable for JIT usage.
     #[cfg(feature = "jit")]
-    pub(super) fn relocate_for_jit(
+    pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
-        mut self,
-        jit_module: &cranelift_simplejit::SimpleJITModule,
-    ) -> Vec<u8> {
         use std::convert::TryInto;
 
         for reloc in self.relocs.drain(..) {

compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs

@@ -15,11 +15,11 @@ pub(crate) struct UnwindContext<'tcx> {
 }
 
 impl<'tcx> UnwindContext<'tcx> {
-    pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self {
+    pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
         let mut frame_table = FrameTable::default();
 
         let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
-            if isa.flags().is_pic() {
+            if pic_eh_frame {
                 cie.fde_address_encoding =
                     gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0);
             }
@@ -80,7 +80,7 @@ impl<'tcx> UnwindContext<'tcx> {
     #[cfg(feature = "jit")]
     pub(crate) unsafe fn register_jit(
         self,
-        jit_module: &cranelift_simplejit::SimpleJITModule,
+        jit_module: &cranelift_jit::JITModule,
     ) -> Option<UnwindRegistry> {
         let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
             self.tcx,

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::middle::cstore::EncodedMetadata;
-use rustc_middle::mir::mono::CodegenUnit;
+use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{DebugInfo, OutputType};
 
@@ -146,11 +146,34 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
         }
     }
 
-    let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None);
+    let mut cx = crate::CodegenCx::new(
+        tcx,
+        module,
+        tcx.sess.opts.debuginfo != DebugInfo::None,
+        true,
+    );
     super::predefine_mono_items(&mut cx, &mono_items);
     for (mono_item, (linkage, visibility)) in mono_items {
         let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
-        super::codegen_mono_item(&mut cx, mono_item, linkage);
+        match mono_item {
+            MonoItem::Fn(inst) => {
+                cx.tcx.sess.time("codegen fn", || {
+                    crate::base::codegen_fn(&mut cx, inst, linkage)
+                });
+            }
+            MonoItem::Static(def_id) => {
+                crate::constant::codegen_static(&mut cx.constants_cx, def_id)
+            }
+            MonoItem::GlobalAsm(hir_id) => {
+                let item = cx.tcx.hir().expect_item(hir_id);
+                if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
+                    cx.global_asm.push_str(&*asm.as_str());
+                    cx.global_asm.push_str("\n\n");
+                } else {
+                    bug!("Expected GlobalAsm found {:?}", item);
+                }
+            }
+        }
     }
     let (mut module, global_asm, debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
@@ -236,7 +259,7 @@ pub(super) fn run_aot(
     tcx.sess.abort_if_errors();
 
     let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
-    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa());
+    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
     let created_alloc_shim =
         crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
 
@@ -258,9 +281,6 @@ pub(super) fn run_aot(
         None
     };
 
-    rustc_incremental::assert_dep_graph(tcx);
-    rustc_incremental::save_dep_graph(tcx);
-
     let metadata_module = if need_metadata_module {
         let _timer = tcx.prof.generic_activity("codegen crate metadata");
         let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || {
@@ -299,10 +319,6 @@ pub(super) fn run_aot(
         None
     };
 
-    if tcx.sess.opts.output_types.should_codegen() {
-        rustc_incremental::assert_module_sources::assert_module_sources(tcx);
-    }
-
     Box::new((
         CodegenResults {
             crate_name: tcx.crate_name(LOCAL_CRATE),

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -1,16 +1,23 @@
 //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
 //! files.
 
+use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
 use rustc_codegen_ssa::CrateInfo;
+use rustc_middle::mir::mono::MonoItem;
 
-use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule};
+use cranelift_jit::{JITBuilder, JITModule};
 
 use crate::prelude::*;
+use crate::{CodegenCx, CodegenMode};
 
-pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
+thread_local! {
+    pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
+}
+
+pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
     if !tcx.sess.opts.output_types.should_codegen() {
         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
     }
@@ -35,12 +42,13 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
 
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
-    let mut jit_builder = SimpleJITBuilder::with_isa(
+    let mut jit_builder = JITBuilder::with_isa(
-        crate::build_isa(tcx.sess, false),
+        crate::build_isa(tcx.sess),
         cranelift_module::default_libcall_names(),
     );
+    jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
     jit_builder.symbols(imported_symbols);
-    let mut jit_module = SimpleJITModule::new(jit_builder);
+    let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
     let sig = Signature {
@@ -66,20 +74,42 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    let mut cx = crate::CodegenCx::new(tcx, jit_module, false);
+    let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+
+    super::time(tcx, "codegen mono items", || {
+        super::predefine_mono_items(&mut cx, &mono_items);
+        for (mono_item, (linkage, visibility)) in mono_items {
+            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+            match mono_item {
+                MonoItem::Fn(inst) => match codegen_mode {
+                    CodegenMode::Aot => unreachable!(),
+                    CodegenMode::Jit => {
+                        cx.tcx.sess.time("codegen fn", || {
+                            crate::base::codegen_fn(&mut cx, inst, linkage)
+                        });
+                    }
+                    CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
+                },
+                MonoItem::Static(def_id) => {
+                    crate::constant::codegen_static(&mut cx.constants_cx, def_id);
+                }
+                MonoItem::GlobalAsm(hir_id) => {
+                    let item = cx.tcx.hir().expect_item(hir_id);
+                    tcx.sess
+                        .span_fatal(item.span, "Global asm is not supported in JIT mode");
+                }
+            }
+        }
+    });
 
     let (mut jit_module, global_asm, _debug, mut unwind_context) =
-        super::time(tcx, "codegen mono items", || {
+        tcx.sess.time("finalize CodegenCx", || cx.finalize());
-            super::predefine_mono_items(&mut cx, &mono_items);
+    jit_module.finalize_definitions();
-            for (mono_item, (linkage, visibility)) in mono_items {
+
-                let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
-                super::codegen_mono_item(&mut cx, mono_item, linkage);
-            }
-            tcx.sess.time("finalize CodegenCx", || cx.finalize())
-        });
     if !global_asm.is_empty() {
-        tcx.sess.fatal("Global asm is not supported in JIT mode");
+        tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
+
     crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
@@ -91,7 +121,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
 
     let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
 
-    println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed");
+    println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
 
     let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
         unsafe { ::std::mem::transmute(finalized_main) };
@@ -107,11 +137,46 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
     // useful as some dynamic linkers use it as a marker to jump over.
     argv.push(std::ptr::null());
 
+    CURRENT_MODULE
+        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
+
     let ret = f(args.len() as c_int, argv.as_ptr());
 
     std::process::exit(ret);
 }
 
+#[no_mangle]
+extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
+    rustc_middle::ty::tls::with(|tcx| {
+        // lift is used to ensure the correct lifetime for instance.
+        let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
+
+        CURRENT_MODULE.with(|jit_module| {
+            let mut jit_module = jit_module.borrow_mut();
+            let jit_module = jit_module.as_mut().unwrap();
+            let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+
+            let name = tcx.symbol_name(instance).name.to_string();
+            let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance);
+            let func_id = cx
+                .module
+                .declare_function(&name, Linkage::Export, &sig)
+                .unwrap();
+            cx.module.prepare_for_function_redefine(func_id).unwrap();
+
+            tcx.sess.time("codegen fn", || {
+                crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
+            });
+
+            let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
+            assert!(global_asm.is_empty());
+            jit_module.finalize_definitions();
+            std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
+            jit_module.get_finalized_function(func_id)
+        })
+    })
+}
+
 fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     use rustc_middle::middle::dependency_format::Linkage;
 
@@ -171,3 +236,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
 
     imported_symbols
 }
+
+pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
+    let tcx = cx.tcx;
+
+    let pointer_type = cx.module.target_config().pointer_type();
+
+    let name = tcx.symbol_name(inst).name.to_string();
+    let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
+    let func_id = cx
+        .module
+        .declare_function(&name, Linkage::Export, &sig)
+        .unwrap();
+
+    let instance_ptr = Box::into_raw(Box::new(inst));
+
+    let jit_fn = cx
+        .module
+        .declare_function(
+            "__clif_jit_fn",
+            Linkage::Import,
+            &Signature {
+                call_conv: cx.module.target_config().default_call_conv,
+                params: vec![AbiParam::new(pointer_type)],
+                returns: vec![AbiParam::new(pointer_type)],
+            },
+        )
+        .unwrap();
+
+    let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
+    let mut builder_ctx = FunctionBuilderContext::new();
+    let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
+
+    let jit_fn = cx
+        .module
+        .declare_func_in_func(jit_fn, trampoline_builder.func);
+    let sig_ref = trampoline_builder.func.import_signature(sig);
+
+    let entry_block = trampoline_builder.create_block();
+    trampoline_builder.append_block_params_for_function_params(entry_block);
+    let fn_args = trampoline_builder
+        .func
+        .dfg
+        .block_params(entry_block)
+        .to_vec();
+
+    trampoline_builder.switch_to_block(entry_block);
+    let instance_ptr = trampoline_builder
+        .ins()
+        .iconst(pointer_type, instance_ptr as u64 as i64);
+    let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
+    let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
+    let call_inst = trampoline_builder
+        .ins()
+        .call_indirect(sig_ref, jitted_fn, &fn_args);
+    let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
+    trampoline_builder.ins().return_(&ret_vals);
+
+    cx.module
+        .define_function(
+            func_id,
+            &mut Context::for_function(trampoline),
+            &mut cranelift_codegen::binemit::NullTrapSink {},
+        )
+        .unwrap();
+}

compiler/rustc_codegen_cranelift/src/driver/mod.rs

@@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility};
 
 use crate::prelude::*;
+use crate::CodegenMode;
 
 mod aot;
 #[cfg(feature = "jit")]
@@ -20,24 +21,25 @@ pub(crate) fn codegen_crate(
 ) -> Box<dyn Any> {
     tcx.sess.abort_if_errors();
 
-    if config.use_jit {
+    match config.codegen_mode {
-        let is_executable = tcx
+        CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module),
-            .sess
+        CodegenMode::Jit | CodegenMode::JitLazy => {
-            .crate_types()
+            let is_executable = tcx
-            .contains(&rustc_session::config::CrateType::Executable);
+                .sess
-        if !is_executable {
+                .crate_types()
-            tcx.sess.fatal("can't jit non-executable crate");
+                .contains(&rustc_session::config::CrateType::Executable);
+            if !is_executable {
+                tcx.sess.fatal("can't jit non-executable crate");
+            }
+
+            #[cfg(feature = "jit")]
+            let _: ! = jit::run_jit(tcx, config.codegen_mode);
+
+            #[cfg(not(feature = "jit"))]
+            tcx.sess
+                .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         }
-
-        #[cfg(feature = "jit")]
-        let _: ! = jit::run_jit(tcx);
-
-        #[cfg(not(feature = "jit"))]
-        tcx.sess
-            .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
     }
-
-    aot::run_aot(tcx, metadata, need_metadata_module)
 }
 
 fn predefine_mono_items<'tcx>(
@@ -48,12 +50,9 @@ fn predefine_mono_items<'tcx>(
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
                 MonoItem::Fn(instance) => {
-                    let (name, sig) = get_function_name_and_sig(
+                    let name = cx.tcx.symbol_name(instance).name.to_string();
-                        cx.tcx,
+                    let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
-                        cx.module.isa().triple(),
+                    let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
-                        instance,
-                        false,
-                    );
                     let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
                     cx.module.declare_function(&name, linkage, &sig).unwrap();
                 }
@@ -63,30 +62,6 @@ fn predefine_mono_items<'tcx>(
     });
 }
 
-fn codegen_mono_item<'tcx, M: Module>(
-    cx: &mut crate::CodegenCx<'tcx, M>,
-    mono_item: MonoItem<'tcx>,
-    linkage: Linkage,
-) {
-    match mono_item {
-        MonoItem::Fn(inst) => {
-            cx.tcx
-                .sess
-                .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage));
-        }
-        MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id),
-        MonoItem::GlobalAsm(hir_id) => {
-            let item = cx.tcx.hir().expect_item(hir_id);
-            if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
-                cx.global_asm.push_str(&*asm.as_str());
-                cx.global_asm.push_str("\n\n");
-            } else {
-                bug!("Expected GlobalAsm found {:?}", item);
-            }
-        }
-    }
-}
-
 fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
     if std::env::var("CG_CLIF_DISPLAY_CG_TIME")
         .as_ref()

compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

@@ -23,8 +23,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
 
         // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
         llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) {
-            let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout());
+            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
-            let lane_ty = fx.clif_type(lane_layout.ty).unwrap();
+            let lane_ty = fx.clif_type(lane_ty).unwrap();
             assert!(lane_count <= 32);
 
             let mut res = fx.bcx.ins().iconst(types::I32, 0);

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -171,27 +171,6 @@ macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     }
 }
 
-fn lane_type_and_count<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    layout: TyAndLayout<'tcx>,
-) -> (TyAndLayout<'tcx>, u16) {
-    assert!(layout.ty.is_simd());
-    let lane_count = match layout.fields {
-        rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(),
-        _ => unreachable!("lane_type_and_count({:?})", layout),
-    };
-    let lane_layout = layout
-        .field(
-            &ty::layout::LayoutCx {
-                tcx,
-                param_env: ParamEnv::reveal_all(),
-            },
-            0,
-        )
-        .unwrap();
-    (lane_layout, lane_count)
-}
-
 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
     let (element, count) = match &layout.abi {
         Abi::Vector { element, count } => (element.clone(), *count),
@@ -218,8 +197,10 @@ fn simd_for_each_lane<'tcx, M: Module>(
 ) {
     let layout = val.layout();
 
-    let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
+    let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-    let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+    let lane_layout = fx.layout_of(lane_ty);
+    let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+    let ret_lane_layout = fx.layout_of(ret_lane_ty);
     assert_eq!(lane_count, ret_lane_count);
 
     for lane_idx in 0..lane_count {
@@ -248,8 +229,10 @@ fn simd_pair_for_each_lane<'tcx, M: Module>(
     assert_eq!(x.layout(), y.layout());
     let layout = x.layout();
 
-    let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
+    let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-    let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+    let lane_layout = fx.layout_of(lane_ty);
+    let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+    let ret_lane_layout = fx.layout_of(ret_lane_ty);
     assert_eq!(lane_count, ret_lane_count);
 
     for lane in 0..lane_count {
@@ -269,13 +252,14 @@ fn simd_reduce<'tcx, M: Module>(
     ret: CPlace<'tcx>,
     f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value,
 ) {
-    let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout());
+    let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+    let lane_layout = fx.layout_of(lane_ty);
     assert_eq!(lane_layout, ret.layout());
 
     let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     for lane_idx in 1..lane_count {
         let lane = val
-            .value_field(fx, mir::Field::new(lane_idx.into()))
+            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
             .load_scalar(fx);
         res_val = f(fx, lane_layout, res_val, lane);
     }
@@ -289,14 +273,14 @@ fn simd_reduce_bool<'tcx, M: Module>(
     ret: CPlace<'tcx>,
     f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value,
 ) {
-    let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout());
+    let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     assert!(ret.layout().ty.is_bool());
 
     let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
     for lane_idx in 1..lane_count {
         let lane = val
-            .value_field(fx, mir::Field::new(lane_idx.into()))
+            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
             .load_scalar(fx);
         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
         res_val = f(fx, res_val, lane);
@@ -460,9 +444,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 "abort" => {
                     trap_abort(fx, "Called intrinsic::abort.");
                 }
-                "unreachable" => {
-                    trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
-                }
                 "transmute" => {
                     crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
                 }
@@ -575,12 +556,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
             }
         };
-        discriminant_value, (c ptr) {
-            let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
-            let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
-            let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
-            ret.write_cvalue(fx, discr);
-        };
         size_of_val, <T> (c ptr) {
             let layout = fx.layout_of(T);
             let size = if layout.is_unsized() {
@@ -641,22 +616,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             );
             ret.write_cvalue(fx, res);
         };
-        _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
-            assert_eq!(x.layout().ty, y.layout().ty);
-            let bin_op = match intrinsic {
-                "wrapping_add" => BinOp::Add,
-                "wrapping_sub" => BinOp::Sub,
-                "wrapping_mul" => BinOp::Mul,
-                _ => unreachable!("intrinsic {}", intrinsic),
-            };
-            let res = crate::num::codegen_int_binop(
-                fx,
-                bin_op,
-                x,
-                y,
-            );
-            ret.write_cvalue(fx, res);
-        };
         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
             assert_eq!(lhs.layout().ty, rhs.layout().ty);
             let bin_op = match intrinsic {
@@ -865,7 +824,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                     }
                     ty => unreachable!("bswap {}", ty),
                 }
-            };
+            }
             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
             ret.write_cvalue(fx, res);
         };
@@ -916,7 +875,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             dest.write_cvalue(fx, val);
         };
 
-        size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
+        pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
             let const_val =
                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
             let val = crate::constant::codegen_const_value(

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -73,11 +73,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             assert_eq!(x.layout(), y.layout());
             let layout = x.layout();
 
-            let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout);
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-            let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 
-            assert_eq!(lane_type, ret_lane_type);
+            assert_eq!(lane_ty, ret_lane_ty);
-            assert_eq!(n, ret_lane_count);
+            assert_eq!(u64::from(n), ret_lane_count);
 
             let total_len = lane_count * 2;
 
@@ -105,14 +105,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             for &idx in &indexes {
-                assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len);
+                assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
             }
 
             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
-                let in_lane = if in_idx < lane_count {
+                let in_lane = if u64::from(in_idx) < lane_count {
                     x.value_field(fx, mir::Field::new(in_idx.into()))
                 } else {
-                    y.value_field(fx, mir::Field::new((in_idx - lane_count).into()))
+                    y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap()))
                 };
                 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
                 out_lane.write_cvalue(fx, in_lane);
@@ -131,7 +131,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
-            let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout());
+            let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
             }
@@ -160,7 +160,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
-            let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout());
+            let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
             }
@@ -212,12 +212,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             assert_eq!(a.layout(), c.layout());
             let layout = a.layout();
 
-            let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
+            let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-            let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
             assert_eq!(lane_count, ret_lane_count);
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
 
             for lane in 0..lane_count {
-                let lane = mir::Field::new(lane.into());
+                let lane = mir::Field::new(lane.try_into().unwrap());
                 let a_lane = a.value_field(fx, lane).load_scalar(fx);
                 let b_lane = b.value_field(fx, lane).load_scalar(fx);
                 let c_lane = c.value_field(fx, lane).load_scalar(fx);

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -5,7 +5,8 @@
     associated_type_bounds,
     never_type,
     try_blocks,
-    hash_drain_filter
+    hash_drain_filter,
+    str_split_once
 )]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
@@ -26,7 +27,6 @@ extern crate rustc_incremental;
 extern crate rustc_index;
 extern crate rustc_session;
 extern crate rustc_span;
-extern crate rustc_symbol_mangling;
 extern crate rustc_target;
 
 // This prevents duplicating functions and statics that are already part of the host rustc process.
@@ -34,6 +34,7 @@ extern crate rustc_target;
 extern crate rustc_driver;
 
 use std::any::Any;
+use std::str::FromStr;
 
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
@@ -81,7 +82,6 @@ mod vtable;
 mod prelude {
     pub(crate) use std::convert::{TryFrom, TryInto};
 
-    pub(crate) use rustc_ast::ast::{FloatTy, IntTy, UintTy};
     pub(crate) use rustc_span::Span;
 
     pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
@@ -89,7 +89,8 @@ mod prelude {
     pub(crate) use rustc_middle::mir::{self, *};
     pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout};
     pub(crate) use rustc_middle::ty::{
-        self, FnSig, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable,
+        self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
+        TypeFoldable, UintTy,
     };
     pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx};
 
@@ -141,8 +142,8 @@ struct CodegenCx<'tcx, M: Module> {
 }
 
 impl<'tcx, M: Module> CodegenCx<'tcx, M> {
-    fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool) -> Self {
+    fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self {
-        let unwind_context = UnwindContext::new(tcx, module.isa());
+        let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame);
         let debug_context = if debug_info {
             Some(DebugContext::new(tcx, module.isa()))
         } else {
@@ -172,12 +173,55 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> {
 }
 
 #[derive(Copy, Clone, Debug)]
+pub enum CodegenMode {
+    Aot,
+    Jit,
+    JitLazy,
+}
+
+impl Default for CodegenMode {
+    fn default() -> Self {
+        CodegenMode::Aot
+    }
+}
+
+impl FromStr for CodegenMode {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "aot" => Ok(CodegenMode::Aot),
+            "jit" => Ok(CodegenMode::Jit),
+            "jit-lazy" => Ok(CodegenMode::JitLazy),
+            _ => Err(format!("Unknown codegen mode `{}`", s)),
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, Default)]
 pub struct BackendConfig {
-    pub use_jit: bool,
+    pub codegen_mode: CodegenMode,
+}
+
+impl BackendConfig {
+    fn from_opts(opts: &[String]) -> Result<Self, String> {
+        let mut config = BackendConfig::default();
+        for opt in opts {
+            if let Some((name, value)) = opt.split_once('=') {
+                match name {
+                    "mode" => config.codegen_mode = value.parse()?,
+                    _ => return Err(format!("Unknown option `{}`", name)),
+                }
+            } else {
+                return Err(format!("Invalid option `{}`", opt));
+            }
+        }
+        Ok(config)
+    }
 }
 
 pub struct CraneliftCodegenBackend {
-    pub config: BackendConfig,
+    pub config: Option<BackendConfig>,
 }
 
 impl CodegenBackend for CraneliftCodegenBackend {
@@ -204,9 +248,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, self.config);
+        let config = if let Some(config) = self.config {
-
+            config
-        rustc_symbol_mangling::test::report_symbol_names(tcx);
+        } else {
+            BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
+                .unwrap_or_else(|err| tcx.sess.fatal(&err))
+        };
+        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
 
         res
     }
@@ -229,18 +277,14 @@ impl CodegenBackend for CraneliftCodegenBackend {
     ) -> Result<(), ErrorReported> {
         use rustc_codegen_ssa::back::link::link_binary;
 
-        let _timer = sess.prof.generic_activity("link_crate");
+        let target_cpu = crate::target_triple(sess).to_string();
-
+        link_binary::<crate::archive::ArArchiveBuilder<'_>>(
-        sess.time("linking", || {
+            sess,
-            let target_cpu = crate::target_triple(sess).to_string();
+            &codegen_results,
-            link_binary::<crate::archive::ArArchiveBuilder<'_>>(
+            outputs,
-                sess,
+            &codegen_results.crate_name.as_str(),
-                &codegen_results,
+            &target_cpu,
-                outputs,
+        );
-                &codegen_results.crate_name.as_str(),
-                &target_cpu,
-            );
-        });
 
         Ok(())
     }
@@ -250,17 +294,13 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
     sess.target.llvm_target.parse().unwrap()
 }
 
-fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'static> {
+fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     use target_lexicon::BinaryFormat;
 
     let target_triple = crate::target_triple(sess);
 
     let mut flags_builder = settings::builder();
-    if enable_pic {
+    flags_builder.enable("is_pic").unwrap();
-        flags_builder.enable("is_pic").unwrap();
-    } else {
-        flags_builder.set("is_pic", "false").unwrap();
-    }
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
     flags_builder
         .set(
@@ -283,8 +323,6 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'stat
 
     flags_builder.set("enable_simd", "true").unwrap();
 
-    // FIXME(CraneStation/cranelift#732) fix LICM in presence of jump tables
-    /*
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
@@ -297,11 +335,16 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'stat
         OptLevel::Size | OptLevel::SizeMin => {
             sess.warn("Optimizing for size is not supported. Just ignoring the request");
         }
-    }*/
+    }
 
     let flags = settings::Flags::new(flags_builder);
 
-    let mut isa_builder = cranelift_codegen::isa::lookup(target_triple).unwrap();
+    let variant = if cfg!(feature = "oldbe") {
+        cranelift_codegen::isa::BackendVariant::Legacy
+    } else {
+        cranelift_codegen::isa::BackendVariant::MachInst
+    };
+    let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
     // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
     // is interpreted as `bsr`.
     isa_builder.enable("nehalem").unwrap();
@@ -311,7 +354,5 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'stat
 /// This is the entrypoint for a hot plugged rustc_codegen_cranelift
 #[no_mangle]
 pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
-    Box::new(CraneliftCodegenBackend {
+    Box::new(CraneliftCodegenBackend { config: None })
-        config: BackendConfig { use_jit: false },
-    })
 }

compiler/rustc_codegen_cranelift/src/main_shim.rs

@@ -69,8 +69,8 @@ pub(crate) fn maybe_create_entry_wrapper(
 
         let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
 
-        let (main_name, main_sig) =
+        let main_name = tcx.symbol_name(instance).name.to_string();
-            get_function_name_and_sig(tcx, m.isa().triple(), instance, false);
+        let main_sig = get_function_sig(tcx, m.isa().triple(), instance);
         let main_func_id = m
             .declare_function(&main_name, Linkage::Import, &main_sig)
             .unwrap();

compiler/rustc_codegen_cranelift/src/num.rs

@@ -280,7 +280,6 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
                     (val, fx.bcx.ins().bor(has_underflow, has_overflow))
                 }
                 types::I64 => {
-                    //let val = fx.easy_call("__mulodi4", &[lhs, rhs, overflow_ptr], types::I64);
                     let val = fx.bcx.ins().imul(lhs, rhs);
                     let has_overflow = if !signed {
                         let val_hi = fx.bcx.ins().umulhi(lhs, rhs);

compiler/rustc_codegen_cranelift/src/optimize/peephole.rs

@@ -73,7 +73,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
     })()
     .unwrap_or_else(|| {
         match bcx.func.dfg.value_type(arg) {
-            types::I8 | types::I32 => {
+            types::I8 | types::I16 => {
                 // WORKAROUND for brz.i8 and brnz.i8 not yet being implemented
                 bcx.ins().uextend(types::I32, arg)
             }
@@ -81,3 +81,40 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
         }
     })
 }
+
+/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known.
+pub(crate) fn maybe_known_branch_taken(
+    bcx: &FunctionBuilder<'_>,
+    arg: Value,
+    test_zero: bool,
+) -> Option<bool> {
+    let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
+        arg_inst
+    } else {
+        return None;
+    };
+
+    match bcx.func.dfg[arg_inst] {
+        InstructionData::UnaryBool {
+            opcode: Opcode::Bconst,
+            imm,
+        } => {
+            if test_zero {
+                Some(!imm)
+            } else {
+                Some(imm)
+            }
+        }
+        InstructionData::UnaryImm {
+            opcode: Opcode::Iconst,
+            imm,
+        } => {
+            if test_zero {
+                Some(imm.bits() == 0)
+            } else {
+                Some(imm.bits() != 0)
+            }
+        }
+        _ => None,
+    }
+}

compiler/rustc_codegen_cranelift/src/pretty_clif.rs

@@ -53,6 +53,7 @@
 //! ```
 
 use std::fmt;
+use std::io::Write;
 
 use cranelift_codegen::{
     entity::SecondaryMap,
@@ -60,7 +61,9 @@ use cranelift_codegen::{
     write::{FuncWriter, PlainWriter},
 };
 
+use rustc_middle::ty::layout::FnAbiExt;
 use rustc_session::config::OutputType;
+use rustc_target::abi::call::FnAbi;
 
 use crate::prelude::*;
 
@@ -77,11 +80,8 @@ impl CommentWriter {
                 format!("symbol {}", tcx.symbol_name(instance).name),
                 format!("instance {:?}", instance),
                 format!(
-                    "sig {:?}",
+                    "abi {:?}",
-                    tcx.normalize_erasing_late_bound_regions(
+                    FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])
-                        ParamEnv::reveal_all(),
-                        crate::abi::fn_sig_for_fn_abi(tcx, instance)
-                    )
                 ),
                 String::new(),
             ]
@@ -200,32 +200,24 @@ impl<M: Module> FunctionCx<'_, '_, M> {
     }
 }
 
-pub(crate) fn write_clif_file<'tcx>(
+pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
-    tcx: TyCtxt<'tcx>,
+    cfg!(debug_assertions)
-    postfix: &str,
+        || tcx
-    isa: Option<&dyn cranelift_codegen::isa::TargetIsa>,
-    instance: Instance<'tcx>,
-    context: &cranelift_codegen::Context,
-    mut clif_comments: &CommentWriter,
-) {
-    use std::io::Write;
-
-    if !cfg!(debug_assertions)
-        && !tcx
             .sess
             .opts
             .output_types
             .contains_key(&OutputType::LlvmAssembly)
-    {
+}
+
+pub(crate) fn write_ir_file<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    name: &str,
+    write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
+) {
+    if !should_write_ir(tcx) {
         return;
     }
 
-    let value_ranges = isa.map(|isa| {
-        context
-            .build_value_labels_ranges(isa)
-            .expect("value location ranges")
-    });
-
     let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif");
 
     match std::fs::create_dir(&clif_output_dir) {
@@ -234,41 +226,58 @@ pub(crate) fn write_clif_file<'tcx>(
         res @ Err(_) => res.unwrap(),
     }
 
-    let clif_file_name = clif_output_dir.join(format!(
+    let clif_file_name = clif_output_dir.join(name);
-        "{}.{}.clif",
-        tcx.symbol_name(instance).name,
-        postfix
-    ));
-
-    let mut clif = String::new();
-    cranelift_codegen::write::decorate_function(
-        &mut clif_comments,
-        &mut clif,
-        &context.func,
-        &DisplayFunctionAnnotations {
-            isa: Some(&*crate::build_isa(
-                tcx.sess, true, /* PIC doesn't matter here */
-            )),
-            value_ranges: value_ranges.as_ref(),
-        },
-    )
-    .unwrap();
 
     let res: std::io::Result<()> = try {
         let mut file = std::fs::File::create(clif_file_name)?;
-        let target_triple = crate::target_triple(tcx.sess);
+        write(&mut file)?;
-        writeln!(file, "test compile")?;
-        writeln!(file, "set is_pic")?;
-        writeln!(file, "set enable_simd")?;
-        writeln!(file, "target {} haswell", target_triple)?;
-        writeln!(file)?;
-        file.write_all(clif.as_bytes())?;
     };
     if let Err(err) = res {
-        tcx.sess.warn(&format!("err writing clif file: {}", err));
+        tcx.sess.warn(&format!("error writing ir file: {}", err));
     }
 }
 
+pub(crate) fn write_clif_file<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    postfix: &str,
+    isa: Option<&dyn cranelift_codegen::isa::TargetIsa>,
+    instance: Instance<'tcx>,
+    context: &cranelift_codegen::Context,
+    mut clif_comments: &CommentWriter,
+) {
+    write_ir_file(
+        tcx,
+        &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
+        |file| {
+            let value_ranges = isa.map(|isa| {
+                context
+                    .build_value_labels_ranges(isa)
+                    .expect("value location ranges")
+            });
+
+            let mut clif = String::new();
+            cranelift_codegen::write::decorate_function(
+                &mut clif_comments,
+                &mut clif,
+                &context.func,
+                &DisplayFunctionAnnotations {
+                    isa: Some(&*crate::build_isa(tcx.sess)),
+                    value_ranges: value_ranges.as_ref(),
+                },
+            )
+            .unwrap();
+
+            writeln!(file, "test compile")?;
+            writeln!(file, "set is_pic")?;
+            writeln!(file, "set enable_simd")?;
+            writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
+            writeln!(file)?;
+            file.write_all(clif.as_bytes())?;
+            Ok(())
+        },
+    );
+}
+
 impl<M: Module> fmt::Debug for FunctionCx<'_, '_, M> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "{:?}", self.instance.substs)?;

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -334,7 +334,9 @@ impl<'tcx> CPlace<'tcx> {
 
         let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
             kind: StackSlotKind::ExplicitSlot,
-            size: u32::try_from(layout.size.bytes()).unwrap(),
+            // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+            // specify stack slot alignment.
+            size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
             offset: None,
         });
         CPlace {
@@ -450,64 +452,6 @@ impl<'tcx> CPlace<'tcx> {
         fx: &mut FunctionCx<'_, 'tcx, impl Module>,
         from: CValue<'tcx>,
     ) {
-        fn assert_assignable<'tcx>(
-            fx: &FunctionCx<'_, 'tcx, impl Module>,
-            from_ty: Ty<'tcx>,
-            to_ty: Ty<'tcx>,
-        ) {
-            match (from_ty.kind(), to_ty.kind()) {
-                (ty::Ref(_, a, _), ty::Ref(_, b, _))
-                | (
-                    ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }),
-                    ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }),
-                ) => {
-                    assert_assignable(fx, a, b);
-                }
-                (ty::FnPtr(_), ty::FnPtr(_)) => {
-                    let from_sig = fx.tcx.normalize_erasing_late_bound_regions(
-                        ParamEnv::reveal_all(),
-                        from_ty.fn_sig(fx.tcx),
-                    );
-                    let to_sig = fx.tcx.normalize_erasing_late_bound_regions(
-                        ParamEnv::reveal_all(),
-                        to_ty.fn_sig(fx.tcx),
-                    );
-                    assert_eq!(
-                        from_sig, to_sig,
-                        "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
-                        from_sig, to_sig, fx,
-                    );
-                    // fn(&T) -> for<'l> fn(&'l T) is allowed
-                }
-                (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
-                    for (from, to) in from_traits.iter().zip(to_traits) {
-                        let from = fx
-                            .tcx
-                            .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
-                        let to = fx
-                            .tcx
-                            .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
-                        assert_eq!(
-                            from, to,
-                            "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}",
-                            from_traits, to_traits, fx,
-                        );
-                    }
-                    // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
-                }
-                _ => {
-                    assert_eq!(
-                        from_ty,
-                        to_ty,
-                        "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}",
-                        from_ty,
-                        to_ty,
-                        fx,
-                    );
-                }
-            }
-        }
-
         assert_assignable(fx, from.layout().ty, self.layout().ty);
 
         self.write_cvalue_maybe_transmute(fx, from, "write_cvalue");
@@ -556,7 +500,9 @@ impl<'tcx> CPlace<'tcx> {
                     // FIXME do something more efficient for transmutes between vectors and integers.
                     let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
                         kind: StackSlotKind::ExplicitSlot,
-                        size: src_ty.bytes(),
+                        // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+                        // specify stack slot alignment.
+                        size: (src_ty.bytes() + 15) / 16 * 16,
                         offset: None,
                     });
                     let ptr = Pointer::stack_slot(stack_slot);
@@ -794,3 +740,62 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 }
+
+#[track_caller]
+pub(crate) fn assert_assignable<'tcx>(
+    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    from_ty: Ty<'tcx>,
+    to_ty: Ty<'tcx>,
+) {
+    match (from_ty.kind(), to_ty.kind()) {
+        (ty::Ref(_, a, _), ty::Ref(_, b, _))
+        | (
+            ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }),
+            ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }),
+        ) => {
+            assert_assignable(fx, a, b);
+        }
+        (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }))
+        | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => {
+            assert_assignable(fx, a, b);
+        }
+        (ty::FnPtr(_), ty::FnPtr(_)) => {
+            let from_sig = fx.tcx.normalize_erasing_late_bound_regions(
+                ParamEnv::reveal_all(),
+                from_ty.fn_sig(fx.tcx),
+            );
+            let to_sig = fx
+                .tcx
+                .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx));
+            assert_eq!(
+                from_sig, to_sig,
+                "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
+                from_sig, to_sig, fx,
+            );
+            // fn(&T) -> for<'l> fn(&'l T) is allowed
+        }
+        (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
+            for (from, to) in from_traits.iter().zip(to_traits) {
+                let from = fx
+                    .tcx
+                    .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
+                let to = fx
+                    .tcx
+                    .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
+                assert_eq!(
+                    from, to,
+                    "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}",
+                    from_traits, to_traits, fx,
+                );
+            }
+            // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
+        }
+        _ => {
+            assert_eq!(
+                from_ty, to_ty,
+                "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}",
+                from_ty, to_ty, fx,
+            );
+        }
+    }
+}

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -158,7 +158,8 @@ fn build_vtable<'tcx>(
         )
         .unwrap();
 
-    fx.cx.module.define_data(data_id, &data_ctx).unwrap();
+    // FIXME don't duplicate definitions in lazy jit mode
+    let _ = fx.cx.module.define_data(data_id, &data_ctx);
 
     data_id
 }

compiler/rustc_codegen_cranelift/test.sh

@@ -1,9 +1,7 @@
 #!/bin/bash
 set -e
 
-export RUSTFLAGS="-Zrun_dsymutil=no"
+./build.sh --sysroot none "$@"
-
-./build.sh --without-sysroot "$@"
 
 rm -r target/out || true
 

compiler/rustc_codegen_llvm/src/abi.rs

@@ -389,7 +389,7 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 
     fn llvm_cconv(&self) -> llvm::CallConv {
         match self.conv {
-            Conv::C | Conv::Rust => llvm::CCallConv,
+            Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv,
             Conv::AmdGpuKernel => llvm::AmdGpuKernel,
             Conv::AvrInterrupt => llvm::AvrInterrupt,
             Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt,
@@ -546,6 +546,18 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         if cconv != llvm::CCallConv {
             llvm::SetInstructionCallConv(callsite, cconv);
         }
+
+        if self.conv == Conv::CCmseNonSecureCall {
+            // This will probably get ignored on all targets but those supporting the TrustZone-M
+            // extension (thumbv8m targets).
+            unsafe {
+                llvm::AddCallSiteAttrString(
+                    callsite,
+                    llvm::AttributePlace::Function,
+                    rustc_data_structures::const_cstr!("cmse_nonsecure_call"),
+                );
+            }
+        }
     }
 }
 

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -13,6 +13,7 @@ use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::{OptLevel, SanitizerSet};
 use rustc_session::Session;
+use rustc_target::spec::StackProbeType;
 
 use crate::attributes;
 use crate::llvm::AttributePlace::Function;
@@ -98,12 +99,6 @@ fn set_instrument_function(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
 }
 
 fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
-    // Only use stack probes if the target specification indicates that we
-    // should be using stack probes
-    if !cx.sess().target.stack_probes {
-        return;
-    }
-
     // Currently stack probes seem somewhat incompatible with the address
     // sanitizer and thread sanitizer. With asan we're already protected from
     // stack overflow anyway so we don't really need stack probes regardless.
@@ -127,14 +122,31 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
         return;
     }
 
-    // Flag our internal `__rust_probestack` function as the stack probe symbol.
+    let attr_value = match cx.sess().target.stack_probes {
-    // This is defined in the `compiler-builtins` crate for each architecture.
+        StackProbeType::None => None,
-    llvm::AddFunctionAttrStringValue(
+        // Request LLVM to generate the probes inline. If the given LLVM version does not support
-        llfn,
+        // this, no probe is generated at all (even if the attribute is specified).
-        llvm::AttributePlace::Function,
+        StackProbeType::Inline => Some(const_cstr!("inline-asm")),
-        const_cstr!("probe-stack"),
+        // Flag our internal `__rust_probestack` function as the stack probe symbol.
-        const_cstr!("__rust_probestack"),
+        // This is defined in the `compiler-builtins` crate for each architecture.
-    );
+        StackProbeType::Call => Some(const_cstr!("__rust_probestack")),
+        // Pick from the two above based on the LLVM version.
+        StackProbeType::InlineOrCall { min_llvm_version_for_inline } => {
+            if llvm_util::get_version() < min_llvm_version_for_inline {
+                Some(const_cstr!("__rust_probestack"))
+            } else {
+                Some(const_cstr!("inline-asm"))
+            }
+        }
+    };
+    if let Some(attr_value) = attr_value {
+        llvm::AddFunctionAttrStringValue(
+            llfn,
+            llvm::AttributePlace::Function,
+            const_cstr!("probe-stack"),
+            attr_value,
+        );
+    }
 }
 
 pub fn llvm_target_features(sess: &Session) -> impl Iterator<Item = &str> {

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -732,10 +732,7 @@ pub unsafe fn optimize_thin_module(
     let diag_handler = cgcx.create_diag_handler();
 
     let module_name = &thin_module.shared.module_names[thin_module.idx];
-    let split_dwarf_file = cgcx
+    let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap());
-        .output_filenames
-        .split_dwarf_filename(cgcx.split_dwarf_kind, Some(module_name.to_str().unwrap()));
-    let tm_factory_config = TargetMachineFactoryConfig { split_dwarf_file };
     let tm =
         (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?;