New upstream version 1.48.0~beta.8+dfsg1

Cargo.toml

@@ -1,8 +1,7 @@
 [workspace]
 members = [
   "src/bootstrap",
-  "src/rustc",
+  "compiler/rustc",
-  "src/librustc_codegen_llvm",
   "library/std",
   "library/test",
   "src/tools/cargotest",
@@ -10,6 +9,7 @@ members = [
   "src/tools/compiletest",
   "src/tools/error_index_generator",
   "src/tools/linkchecker",
+  "src/tools/lint-docs",
   "src/tools/rustbook",
   "src/tools/unstable-book-gen",
   "src/tools/tidy",

README.md

@@ -7,7 +7,10 @@ standard library, and documentation.
 
 [Rust]: https://www.rust-lang.org
 
-**Note: this README is for _users_ rather than _contributors_.**
+**Note: this README is for _users_ rather than _contributors_.
+If you wish to _contribute_ to the compiler, you should read the
+[Getting Started][gettingstarted] of the rustc-dev-guide instead of this
+section.**
 
 ## Quick Start
 
@@ -18,10 +21,6 @@ Read ["Installation"] from [The Book].
 
 ## Installing from Source
 
-**Note: If you wish to _contribute_ to the compiler, you should read the
-[Getting Started][gettingstarted] of the rustc-dev-guide instead of this
-section.**
-
 The Rust build system uses a Python script called `x.py` to build the compiler,
 which manages the bootstrapping process. More information about it can be found
 by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
@@ -36,6 +35,7 @@ by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
    * `python` 3 or 2.7
    * GNU `make` 3.81 or later
    * `cmake` 3.4.3 or later
+   * `ninja`
    * `curl`
    * `git`
    * `ssl` which comes in `libssl-dev` or `openssl-devel`
@@ -44,8 +44,8 @@ by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
 2. Clone the [source] with `git`:
 
    ```sh
-   $ git clone https://github.com/rust-lang/rust.git
+   git clone https://github.com/rust-lang/rust.git
-   $ cd rust
+   cd rust
    ```
 
 [source]: https://github.com/rust-lang/rust
@@ -57,7 +57,7 @@ by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
     Copy the default `config.toml.example` to `config.toml` to get started.
 
     ```sh
-    $ cp config.toml.example config.toml
+    cp config.toml.example config.toml
     ```
 
     If you plan to use `x.py install` to create an installation, it is recommended
@@ -68,7 +68,7 @@ by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
 4. Build and install:
 
     ```sh
-    $ ./x.py build && ./x.py install
+    ./x.py build && ./x.py install
     ```
 
     When complete, `./x.py install` will place several programs into
@@ -106,27 +106,28 @@ build.
 
    ```sh
    # Update package mirrors (may be needed if you have a fresh install of MSYS2)
-   $ pacman -Sy pacman-mirrors
+   pacman -Sy pacman-mirrors
 
    # Install build tools needed for Rust. If you're building a 32-bit compiler,
    # then replace "x86_64" below with "i686". If you've already got git, python,
    # or CMake installed and in PATH you can remove them from this list. Note
-   # that it is important that you do **not** use the 'python2' and 'cmake'
+   # that it is important that you do **not** use the 'python2', 'cmake' and 'ninja'
    # packages from the 'msys2' subsystem. The build has historically been known
    # to fail with these packages.
-   $ pacman -S git \
+   pacman -S git \
                make \
                diffutils \
                tar \
                mingw-w64-x86_64-python \
                mingw-w64-x86_64-cmake \
-               mingw-w64-x86_64-gcc
+               mingw-w64-x86_64-gcc \
+               mingw-w64-x86_64-ninja
    ```
 
 4. Navigate to Rust's source code (or clone it), then build it:
 
    ```sh
-   $ ./x.py build && ./x.py install
+   ./x.py build && ./x.py install
    ```
 
 #### MSVC
@@ -144,7 +145,7 @@ With these dependencies installed, you can build the compiler in a `cmd.exe`
 shell with:
 
 ```sh
-> python x.py build
+python x.py build
 ```
 
 Currently, building Rust only works with some known versions of Visual Studio. If
@@ -153,8 +154,8 @@ you may need to force rustbuild to use an older version. This can be done
 by manually calling the appropriate vcvars file before running the bootstrap.
 
 ```batch
-> CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
+CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
-> python x.py build
+python x.py build
 ```
 
 #### Specifying an ABI
@@ -180,8 +181,8 @@ While it's not the recommended build system, this project also provides a
 configure script and makefile (the latter of which just invokes `x.py`).
 
 ```sh
-$ ./configure
+./configure
-$ make && sudo make install
+make && sudo make install
 ```
 
 When using the configure script, the generated `config.mk` file may override the
@@ -193,7 +194,7 @@ When using the configure script, the generated `config.mk` file may override the
 If you’d like to build the documentation, it’s almost the same:
 
 ```sh
-$ ./x.py doc
+./x.py doc
 ```
 
 The generated documentation will appear under `doc` in the `build` directory for
@@ -209,11 +210,17 @@ fetch snapshots, and an OS that can execute the available snapshot binaries.
 
 Snapshot binaries are currently built and tested on several platforms:
 
-| Platform / Architecture    | x86 | x86_64 |
+| Platform / Architecture                     | x86 | x86_64 |
-|----------------------------|-----|--------|
+|---------------------------------------------|-----|--------|
-| Windows (7, 8, 10, ...)    | ✓   | ✓      |
+| Windows (7, 8, 10, ...)                     | ✓   | ✓      |
-| Linux (2.6.18 or later)    | ✓   | ✓      |
+| Linux (kernel 2.6.32, glibc 2.11 or later)  | ✓   | ✓      |
-| macOS (10.7 Lion or later) | ✓   | ✓      |
+| macOS (10.7 Lion or later)                  | (\*) | ✓      |
+
+(\*): Apple dropped support for running 32-bit binaries starting from macOS 10.15 and iOS 11.
+Due to this decision from Apple, the targets are no longer useful to our users.
+Please read [our blog post][macx32] for more info.
+
+[macx32]: https://blog.rust-lang.org/2020/01/03/reducing-support-for-32-bit-apple-targets.html
 
 You may find that other platforms work, but these are our officially
 supported build environments that are most likely to work.
@@ -235,6 +242,8 @@ The Rust community congregates in a few places:
 If you are interested in contributing to the Rust project, please take a look
 at the [Getting Started][gettingstarted] guide in the [rustc-dev-guide].
 
+[rustc-dev-guide]: https://rustc-dev-guide.rust-lang.org
+
 ## License
 
 Rust is primarily distributed under the terms of both the MIT license

RELEASES.md

@@ -84,16 +84,15 @@ Compatibility Notes
   `Delimiter::None`.
 - [Moved support for the CloudABI target to tier 3.][75568]
 - [`linux-gnu` targets now require minimum kernel 2.6.32 and glibc 2.11.][74163]
-
-Internal Only
---------
-- [Improved default settings for bootstrapping in `x.py`.][73964] You can read details about this change in the ["Changes to `x.py` defaults"](https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html) post on the Inside Rust blog.
 - [Added the `rustc-docs` component.][75560] This allows you to install
   and read the documentation for the compiler internal APIs. (Currently only
   available for `x86_64-unknown-linux-gnu`.)
 
+Internal Only
+--------
+- [Improved default settings for bootstrapping in `x.py`.][73964] You can read details about this change in the ["Changes To `x.py` Defaults"](https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html) post on the Inside Rust blog.
+
 [1.47.0-cfg]: https://docs.microsoft.com/en-us/windows/win32/secbp/control-flow-guard
-[76980]: https://github.com/rust-lang/rust/issues/76980
 [75048]: https://github.com/rust-lang/rust/pull/75048/
 [74163]: https://github.com/rust-lang/rust/pull/74163/
 [71237]: https://github.com/rust-lang/rust/pull/71237/

compiler/rustc/Cargo.toml

@@ -4,16 +4,12 @@ name = "rustc-main"
 version = "0.0.0"
 edition = '2018'
 
-[[bin]]
-name = "rustc_binary"
-path = "rustc.rs"
-
 [dependencies]
-rustc_driver = { path = "../librustc_driver" }
+rustc_driver = { path = "../rustc_driver" }
 
 # Make sure rustc_codegen_ssa ends up in the sysroot, because this
 # crate is intended to be used by codegen backends, which may not be in-tree.
-rustc_codegen_ssa = { path = "../librustc_codegen_ssa" }
+rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 
 [dependencies.jemalloc-sys]
 version = '0.3.0'
@@ -23,3 +19,4 @@ features = ['unprefixed_malloc_on_supported_platforms']
 [features]
 jemalloc = ['jemalloc-sys']
 llvm = ['rustc_driver/llvm']
+max_level_info = ['rustc_driver/max_level_info']

compiler/rustc_apfloat/Cargo.toml

@@ -4,10 +4,6 @@ name = "rustc_apfloat"
 version = "0.0.0"
 edition = "2018"
 
-[lib]
-name = "rustc_apfloat"
-path = "lib.rs"
-
 [dependencies]
 bitflags = "1.2.1"
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }

compiler/rustc_apfloat/src/lib.rs

@@ -30,7 +30,7 @@
 //!
 //! This API is completely unstable and subject to change.
 
-#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![no_std]
 #![forbid(unsafe_code)]
 #![feature(nll)]

compiler/rustc_arena/Cargo.toml

@@ -4,10 +4,6 @@ name = "rustc_arena"
 version = "0.0.0"
 edition = "2018"
 
-[lib]
-name = "rustc_arena"
-path = "lib.rs"
-
 [dependencies]
-rustc_data_structures = { path = "../librustc_data_structures" }
+rustc_data_structures = { path = "../rustc_data_structures" }
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }

compiler/rustc_arena/src/lib.rs

@@ -8,16 +8,13 @@
 //! This crate implements several kinds of arena.
 
 #![doc(
-    html_root_url = "https://doc.rust-lang.org/nightly/",
+    html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(no_crate_inject, attr(deny(warnings)))
 )]
-#![feature(core_intrinsics)]
 #![feature(dropck_eyepatch)]
-#![feature(raw_vec_internals)]
+#![feature(new_uninit)]
+#![feature(maybe_uninit_slice)]
 #![cfg_attr(test, feature(test))]
-#![allow(deprecated)]
-
-extern crate alloc;
 
 use rustc_data_structures::cold_path;
 use smallvec::SmallVec;
@@ -25,14 +22,11 @@ use smallvec::SmallVec;
 use std::alloc::Layout;
 use std::cell::{Cell, RefCell};
 use std::cmp;
-use std::intrinsics;
 use std::marker::{PhantomData, Send};
-use std::mem;
+use std::mem::{self, MaybeUninit};
 use std::ptr;
 use std::slice;
 
-use alloc::raw_vec::RawVec;
-
 /// An arena that can hold objects of only one type.
 pub struct TypedArena<T> {
     /// A pointer to the next object to be allocated.
@@ -52,7 +46,7 @@ pub struct TypedArena<T> {
 
 struct TypedArenaChunk<T> {
     /// The raw storage for the arena chunk.
-    storage: RawVec<T>,
+    storage: Box<[MaybeUninit<T>]>,
     /// The number of valid entries in the chunk.
     entries: usize,
 }
@@ -60,7 +54,7 @@ struct TypedArenaChunk<T> {
 impl<T> TypedArenaChunk<T> {
     #[inline]
     unsafe fn new(capacity: usize) -> TypedArenaChunk<T> {
-        TypedArenaChunk { storage: RawVec::with_capacity(capacity), entries: 0 }
+        TypedArenaChunk { storage: Box::new_uninit_slice(capacity), entries: 0 }
     }
 
     /// Destroys this arena chunk.
@@ -69,30 +63,25 @@ impl<T> TypedArenaChunk<T> {
         // The branch on needs_drop() is an -O1 performance optimization.
         // Without the branch, dropping TypedArena<u8> takes linear time.
         if mem::needs_drop::<T>() {
-            let mut start = self.start();
+            ptr::drop_in_place(MaybeUninit::slice_assume_init_mut(&mut self.storage[..len]));
-            // Destroy all allocated objects.
-            for _ in 0..len {
-                ptr::drop_in_place(start);
-                start = start.offset(1);
-            }
         }
     }
 
     // Returns a pointer to the first allocated object.
     #[inline]
-    fn start(&self) -> *mut T {
+    fn start(&mut self) -> *mut T {
-        self.storage.ptr()
+        MaybeUninit::slice_as_mut_ptr(&mut self.storage)
     }
 
     // Returns a pointer to the end of the allocated space.
     #[inline]
-    fn end(&self) -> *mut T {
+    fn end(&mut self) -> *mut T {
         unsafe {
             if mem::size_of::<T>() == 0 {
                 // A pointer as large as possible for zero-sized elements.
                 !0 as *mut T
             } else {
-                self.start().add(self.storage.capacity())
+                self.start().add(self.storage.len())
             }
         }
     }
@@ -130,7 +119,7 @@ impl<T> TypedArena<T> {
 
         unsafe {
             if mem::size_of::<T>() == 0 {
-                self.ptr.set(intrinsics::arith_offset(self.ptr.get() as *mut u8, 1) as *mut T);
+                self.ptr.set((self.ptr.get() as *mut u8).wrapping_offset(1) as *mut T);
                 let ptr = mem::align_of::<T>() as *mut T;
                 // Don't drop the object. This `write` is equivalent to `forget`.
                 ptr::write(ptr, object);
@@ -226,10 +215,10 @@ impl<T> TypedArena<T> {
                 let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize;
                 last_chunk.entries = used_bytes / mem::size_of::<T>();
 
-                // If the previous chunk's capacity is less than HUGE_PAGE
+                // If the previous chunk's len is less than HUGE_PAGE
                 // bytes, then this chunk will be least double the previous
                 // chunk's size.
-                new_cap = last_chunk.storage.capacity();
+                new_cap = last_chunk.storage.len();
                 if new_cap < HUGE_PAGE / elem_size {
                     new_cap = new_cap.checked_mul(2).unwrap();
                 }
@@ -239,7 +228,7 @@ impl<T> TypedArena<T> {
             // Also ensure that this chunk can fit `additional`.
             new_cap = cmp::max(additional, new_cap);
 
-            let chunk = TypedArenaChunk::<T>::new(new_cap);
+            let mut chunk = TypedArenaChunk::<T>::new(new_cap);
             self.ptr.set(chunk.start());
             self.end.set(chunk.end());
             chunks.push(chunk);
@@ -301,7 +290,7 @@ unsafe impl<#[may_dangle] T> Drop for TypedArena<T> {
                     chunk.destroy(chunk.entries);
                 }
             }
-            // RawVec handles deallocation of `last_chunk` and `self.chunks`.
+            // Box handles deallocation of `last_chunk` and `self.chunks`.
         }
     }
 }
@@ -309,11 +298,13 @@ unsafe impl<#[may_dangle] T> Drop for TypedArena<T> {
 unsafe impl<T: Send> Send for TypedArena<T> {}
 
 pub struct DroplessArena {
-    /// A pointer to the next object to be allocated.
+    /// A pointer to the start of the free space.
-    ptr: Cell<*mut u8>,
+    start: Cell<*mut u8>,
 
-    /// A pointer to the end of the allocated area. When this pointer is
+    /// A pointer to the end of free space.
-    /// reached, a new chunk is allocated.
+    ///
+    /// The allocation proceeds from the end of the chunk towards the start.
+    /// When this pointer crosses the start pointer, a new chunk is allocated.
     end: Cell<*mut u8>,
 
     /// A vector of arena chunks.
@@ -326,7 +317,7 @@ impl Default for DroplessArena {
     #[inline]
     fn default() -> DroplessArena {
         DroplessArena {
-            ptr: Cell::new(ptr::null_mut()),
+            start: Cell::new(ptr::null_mut()),
             end: Cell::new(ptr::null_mut()),
             chunks: Default::default(),
         }
@@ -344,10 +335,10 @@ impl DroplessArena {
                 // There is no need to update `last_chunk.entries` because that
                 // field isn't used by `DroplessArena`.
 
-                // If the previous chunk's capacity is less than HUGE_PAGE
+                // If the previous chunk's len is less than HUGE_PAGE
                 // bytes, then this chunk will be least double the previous
                 // chunk's size.
-                new_cap = last_chunk.storage.capacity();
+                new_cap = last_chunk.storage.len();
                 if new_cap < HUGE_PAGE {
                     new_cap = new_cap.checked_mul(2).unwrap();
                 }
@@ -357,8 +348,8 @@ impl DroplessArena {
             // Also ensure that this chunk can fit `additional`.
             new_cap = cmp::max(additional, new_cap);
 
-            let chunk = TypedArenaChunk::<u8>::new(new_cap);
+            let mut chunk = TypedArenaChunk::<u8>::new(new_cap);
-            self.ptr.set(chunk.start());
+            self.start.set(chunk.start());
             self.end.set(chunk.end());
             chunks.push(chunk);
         }
@@ -369,24 +360,17 @@ impl DroplessArena {
     /// request.
     #[inline]
     fn alloc_raw_without_grow(&self, layout: Layout) -> Option<*mut u8> {
-        let ptr = self.ptr.get() as usize;
+        let start = self.start.get() as usize;
         let end = self.end.get() as usize;
+
         let align = layout.align();
         let bytes = layout.size();
-        // The allocation request fits into the current chunk iff:
+
-        //
+        let new_end = end.checked_sub(bytes)? & !(align - 1);
-        // let aligned = align_to(ptr, align);
+        if start <= new_end {
-        // ptr <= aligned && aligned + bytes <= end
+            let new_end = new_end as *mut u8;
-        //
+            self.end.set(new_end);
-        // Except that we work with fixed width integers and need to be careful
+            Some(new_end)
-        // about potential overflow in the calcuation. If the overflow does
-        // happen, then we definitely don't have enough free and need to grow
-        // the arena.
-        let aligned = ptr.checked_add(align - 1)? & !(align - 1);
-        let new_ptr = aligned.checked_add(bytes)?;
-        if new_ptr <= end {
-            self.ptr.set(new_ptr as *mut u8);
-            Some(aligned as *mut u8)
         } else {
             None
         }

compiler/rustc_arena/src/tests.rs

@@ -121,6 +121,17 @@ pub fn bench_typed_arena_clear(b: &mut Bencher) {
     })
 }
 
+#[bench]
+pub fn bench_typed_arena_clear_100(b: &mut Bencher) {
+    let mut arena = TypedArena::default();
+    b.iter(|| {
+        for _ in 0..100 {
+            arena.alloc(Point { x: 1, y: 2, z: 3 });
+        }
+        arena.clear();
+    })
+}
+
 // Drop tests
 
 struct DropCounter<'a> {

compiler/rustc_ast/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_ast"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+rustc_serialize = { path = "../rustc_serialize" }
+tracing = "0.1"
+rustc_span = { path = "../rustc_span" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_index = { path = "../rustc_index" }
+rustc_lexer = { path = "../rustc_lexer" }
+rustc_macros = { path = "../rustc_macros" }
+smallvec = { version = "1.0", features = ["union", "may_dangle"] }
+bitflags = "1.2.1"

compiler/rustc_ast/src/ast.rs

@@ -96,6 +96,7 @@ pub struct Path {
     /// The segments in the path: the things separated by `::`.
     /// Global paths begin with `kw::PathRoot`.
     pub segments: Vec<PathSegment>,
+    pub tokens: Option<TokenStream>,
 }
 
 impl PartialEq<Symbol> for Path {
@@ -117,7 +118,7 @@ impl Path {
     // Convert a span and an identifier to the corresponding
     // one-segment path.
     pub fn from_ident(ident: Ident) -> Path {
-        Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span }
+        Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span, tokens: None }
     }
 
     pub fn is_global(&self) -> bool {
@@ -540,6 +541,7 @@ pub struct Block {
     /// Distinguishes between `unsafe { ... }` and `{ ... }`.
     pub rules: BlockCheckMode,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 /// A match pattern.
@@ -586,7 +588,7 @@ impl Pat {
             _ => return None,
         };
 
-        Some(P(Ty { kind, id: self.id, span: self.span }))
+        Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
     }
 
     /// Walk top-down and call `it` in each place where a pattern occurs
@@ -916,15 +918,20 @@ pub struct Stmt {
     pub id: NodeId,
     pub kind: StmtKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 impl Stmt {
     pub fn add_trailing_semicolon(mut self) -> Self {
         self.kind = match self.kind {
             StmtKind::Expr(expr) => StmtKind::Semi(expr),
-            StmtKind::MacCall(mac) => StmtKind::MacCall(
+            StmtKind::MacCall(mac) => {
-                mac.map(|(mac, _style, attrs)| (mac, MacStmtStyle::Semicolon, attrs)),
+                StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs }| MacCallStmt {
-            ),
+                    mac,
+                    style: MacStmtStyle::Semicolon,
+                    attrs,
+                }))
+            }
             kind => kind,
         };
         self
@@ -958,7 +965,14 @@ pub enum StmtKind {
     /// Just a trailing semi-colon.
     Empty,
     /// Macro.
-    MacCall(P<(MacCall, MacStmtStyle, AttrVec)>),
+    MacCall(P<MacCallStmt>),
+}
+
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct MacCallStmt {
+    pub mac: MacCall,
+    pub style: MacStmtStyle,
+    pub attrs: AttrVec,
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
@@ -1057,7 +1071,7 @@ pub struct Expr {
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Expr, 104);
+rustc_data_structures::static_assert_size!(Expr, 112);
 
 impl Expr {
     /// Returns `true` if this expression would be valid somewhere that expects a value;
@@ -1157,7 +1171,7 @@ impl Expr {
             _ => return None,
         };
 
-        Some(P(Ty { kind, id: self.id, span: self.span }))
+        Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
     }
 
     pub fn precedence(&self) -> ExprPrecedence {
@@ -1855,6 +1869,7 @@ pub struct Ty {
     pub id: NodeId,
     pub kind: TyKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -1916,7 +1931,7 @@ pub enum TyKind {
 
 impl TyKind {
     pub fn is_implicit_self(&self) -> bool {
-        if let TyKind::ImplicitSelf = *self { true } else { false }
+        matches!(self, TyKind::ImplicitSelf)
     }
 
     pub fn is_unit(&self) -> bool {
@@ -2133,7 +2148,7 @@ impl Param {
     /// Builds a `Param` object from `ExplicitSelf`.
     pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param {
         let span = eself.span.to(eself_ident.span);
-        let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span });
+        let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None });
         let param = |mutbl, ty| Param {
             attrs,
             pat: P(Pat {
@@ -2156,6 +2171,7 @@ impl Param {
                     id: DUMMY_NODE_ID,
                     kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }),
                     span,
+                    tokens: None,
                 }),
             ),
         }
@@ -2211,7 +2227,7 @@ pub enum Async {
 
 impl Async {
     pub fn is_async(self) -> bool {
-        if let Async::Yes { .. } = self { true } else { false }
+        matches!(self, Async::Yes { .. })
     }
 
     /// In this case this is an `async` return, the `NodeId` for the generated `impl Trait` item.
@@ -2278,12 +2294,15 @@ impl FnRetTy {
 /// Module declaration.
 ///
 /// E.g., `mod foo;` or `mod foo { .. }`.
-#[derive(Clone, Encodable, Decodable, Debug, Default)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub struct Mod {
     /// A span from the first token past `{` to the last token until `}`.
     /// For `mod foo;`, the inner span ranges from the first token
     /// to the last token in the external file.
     pub inner: Span,
+    /// `unsafe` keyword accepted syntactically for macro DSLs, but not
+    /// semantically by Rust.
+    pub unsafety: Unsafe,
     pub items: Vec<P<Item>>,
     /// `true` for `mod foo { .. }`; `false` for `mod foo;`.
     pub inline: bool,
@@ -2291,9 +2310,12 @@ pub struct Mod {
 
 /// Foreign module declaration.
 ///
-/// E.g., `extern { .. }` or `extern C { .. }`.
+/// E.g., `extern { .. }` or `extern "C" { .. }`.
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct ForeignMod {
+    /// `unsafe` keyword accepted syntactically for macro DSLs, but not
+    /// semantically by Rust.
+    pub unsafety: Unsafe,
     pub abi: Option<StrLit>,
     pub items: Vec<P<ForeignItem>>,
 }
@@ -2399,6 +2421,7 @@ impl<D: Decoder> rustc_serialize::Decodable<D> for AttrId {
 pub struct AttrItem {
     pub path: Path,
     pub args: MacArgs,
+    pub tokens: Option<TokenStream>,
 }
 
 /// A list of attributes.
@@ -2468,7 +2491,12 @@ pub enum CrateSugar {
     JustCrate,
 }
 
-pub type Visibility = Spanned<VisibilityKind>;
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct Visibility {
+    pub kind: VisibilityKind,
+    pub span: Span,
+    pub tokens: Option<TokenStream>,
+}
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub enum VisibilityKind {
@@ -2480,7 +2508,7 @@ pub enum VisibilityKind {
 
 impl VisibilityKind {
     pub fn is_pub(&self) -> bool {
-        if let VisibilityKind::Public = *self { true } else { false }
+        matches!(self, VisibilityKind::Public)
     }
 }
 

compiler/rustc_ast/src/attr/mod.rs

@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
 use crate::mut_visit::visit_clobber;
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Token};
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
+use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
 
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::{BytePos, Spanned};
@@ -16,7 +16,6 @@ use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
 
 use std::iter;
-use std::ops::DerefMut;
 
 pub struct MarkedAttrs(GrowableBitSet<AttrId>);
 
@@ -331,7 +330,7 @@ crate fn mk_attr_id() -> AttrId {
 }
 
 pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute {
-    mk_attr_from_item(style, AttrItem { path, args }, span)
+    mk_attr_from_item(style, AttrItem { path, args, tokens: None }, span)
 }
 
 pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {
@@ -362,7 +361,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
 }
 
 impl MetaItem {
-    fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
+    fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
         let mut idents = vec![];
         let mut last_pos = BytePos(0 as u32);
         for (i, segment) in self.path.segments.iter().enumerate() {
@@ -375,7 +374,7 @@ impl MetaItem {
             idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
             last_pos = segment.ident.span.hi();
         }
-        idents.extend(self.kind.token_trees_and_joints(self.span));
+        idents.extend(self.kind.token_trees_and_spacings(self.span));
         idents
     }
 
@@ -416,7 +415,7 @@ impl MetaItem {
                     }
                 }
                 let span = span.with_hi(segments.last().unwrap().ident.span.hi());
-                Path { span, segments }
+                Path { span, segments, tokens: None }
             }
             Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
                 token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
@@ -448,7 +447,7 @@ impl MetaItemKind {
                     if i > 0 {
                         tts.push(TokenTree::token(token::Comma, span).into());
                     }
-                    tts.extend(item.token_trees_and_joints())
+                    tts.extend(item.token_trees_and_spacings())
                 }
                 MacArgs::Delimited(
                     DelimSpan::from_single(span),
@@ -459,7 +458,7 @@ impl MetaItemKind {
         }
     }
 
-    fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
+    fn token_trees_and_spacings(&self, span: Span) -> Vec<TreeAndSpacing> {
         match *self {
             MetaItemKind::Word => vec![],
             MetaItemKind::NameValue(ref lit) => {
@@ -471,7 +470,7 @@ impl MetaItemKind {
                     if i > 0 {
                         tokens.push(TokenTree::token(token::Comma, span).into());
                     }
-                    tokens.extend(item.token_trees_and_joints())
+                    tokens.extend(item.token_trees_and_spacings())
                 }
                 vec![
                     TokenTree::Delimited(
@@ -554,9 +553,9 @@ impl NestedMetaItem {
         }
     }
 
-    fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
+    fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
         match *self {
-            NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(),
+            NestedMetaItem::MetaItem(ref item) => item.token_trees_and_spacings(),
             NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()],
         }
     }
@@ -634,10 +633,7 @@ impl HasAttrs for StmtKind {
             StmtKind::Local(ref local) => local.attrs(),
             StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(),
             StmtKind::Empty | StmtKind::Item(..) => &[],
-            StmtKind::MacCall(ref mac) => {
+            StmtKind::MacCall(ref mac) => mac.attrs.attrs(),
-                let (_, _, ref attrs) = **mac;
-                attrs.attrs()
-            }
         }
     }
 
@@ -647,8 +643,7 @@ impl HasAttrs for StmtKind {
             StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.visit_attrs(f),
             StmtKind::Empty | StmtKind::Item(..) => {}
             StmtKind::MacCall(mac) => {
-                let (_mac, _style, attrs) = mac.deref_mut();
+                mac.attrs.visit_attrs(f);
-                attrs.visit_attrs(f);
             }
         }
     }

compiler/rustc_ast/src/lib.rs

@@ -4,18 +4,18 @@
 //!
 //! This API is completely unstable and subject to change.
 
-#![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
+#![doc(
-#![feature(bool_to_option)]
+    html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
+    test(attr(deny(warnings)))
+)]
 #![feature(box_syntax)]
 #![feature(const_fn)] // For the `transmute` in `P::new`
-#![feature(const_panic)]
 #![feature(const_fn_transmute)]
+#![feature(const_panic)]
 #![feature(crate_visibility_modifier)]
 #![feature(label_break_value)]
 #![feature(nll)]
 #![feature(or_patterns)]
-#![feature(try_trait)]
-#![feature(unicode_internals)]
 #![recursion_limit = "256"]
 
 #[macro_use]

compiler/rustc_ast/src/mut_visit.rs

@@ -14,7 +14,7 @@ use crate::tokenstream::*;
 
 use rustc_data_structures::map_in_place::MapInPlace;
 use rustc_data_structures::sync::Lrc;
-use rustc_span::source_map::{respan, Spanned};
+use rustc_span::source_map::Spanned;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -451,7 +451,7 @@ pub fn noop_visit_ty_constraint<T: MutVisitor>(
 }
 
 pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
-    let Ty { id, kind, span } = ty.deref_mut();
+    let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
     vis.visit_id(id);
     match kind {
         TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {}
@@ -490,7 +490,7 @@ pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
 }
 
 pub fn noop_visit_foreign_mod<T: MutVisitor>(foreign_mod: &mut ForeignMod, vis: &mut T) {
-    let ForeignMod { abi: _, items } = foreign_mod;
+    let ForeignMod { unsafety: _, abi: _, items } = foreign_mod;
     items.flat_map_in_place(|item| vis.flat_map_foreign_item(item));
 }
 
@@ -513,7 +513,7 @@ pub fn noop_visit_ident<T: MutVisitor>(Ident { name: _, span }: &mut Ident, vis:
     vis.visit_span(span);
 }
 
-pub fn noop_visit_path<T: MutVisitor>(Path { segments, span }: &mut Path, vis: &mut T) {
+pub fn noop_visit_path<T: MutVisitor>(Path { segments, span, tokens: _ }: &mut Path, vis: &mut T) {
     vis.visit_span(span);
     for PathSegment { ident, id, args } in segments {
         vis.visit_ident(ident);
@@ -579,7 +579,7 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
     let Attribute { kind, id: _, style: _, span } = attr;
     match kind {
-        AttrKind::Normal(AttrItem { path, args }) => {
+        AttrKind::Normal(AttrItem { path, args, tokens: _ }) => {
             vis.visit_path(path);
             visit_mac_args(args, vis);
         }
@@ -709,7 +709,7 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis:
         token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {
-            let AttrItem { path, args } = item.deref_mut();
+            let AttrItem { path, args, tokens: _ } = item.deref_mut();
             vis.visit_path(path);
             visit_mac_args(args, vis);
         }
@@ -871,7 +871,7 @@ pub fn noop_visit_mt<T: MutVisitor>(MutTy { ty, mutbl: _ }: &mut MutTy, vis: &mu
 }
 
 pub fn noop_visit_block<T: MutVisitor>(block: &mut P<Block>, vis: &mut T) {
-    let Block { id, stmts, rules: _, span } = block.deref_mut();
+    let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut();
     vis.visit_id(id);
     stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt));
     vis.visit_span(span);
@@ -970,18 +970,21 @@ pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
     vis.visit_asyncness(asyncness);
 }
 
-pub fn noop_visit_mod<T: MutVisitor>(Mod { inner, items, inline: _ }: &mut Mod, vis: &mut T) {
+pub fn noop_visit_mod<T: MutVisitor>(module: &mut Mod, vis: &mut T) {
+    let Mod { inner, unsafety: _, items, inline: _ } = module;
     vis.visit_span(inner);
     items.flat_map_in_place(|item| vis.flat_map_item(item));
 }
 
 pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) {
     visit_clobber(krate, |Crate { module, attrs, span, proc_macros }| {
+        let item_vis =
+            Visibility { kind: VisibilityKind::Public, span: span.shrink_to_lo(), tokens: None };
         let item = P(Item {
             ident: Ident::invalid(),
             attrs,
             id: DUMMY_NODE_ID,
-            vis: respan(span.shrink_to_lo(), VisibilityKind::Public),
+            vis: item_vis,
             span,
             kind: ItemKind::Mod(module),
             tokens: None,
@@ -990,7 +993,7 @@ pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) {
 
         let len = items.len();
         if len == 0 {
-            let module = Mod { inner: span, items: vec![], inline: true };
+            let module = Mod { inner: span, unsafety: Unsafe::No, items: vec![], inline: true };
             Crate { module, attrs: vec![], span, proc_macros }
         } else if len == 1 {
             let Item { attrs, span, kind, .. } = items.into_iter().next().unwrap().into_inner();
@@ -1283,12 +1286,15 @@ pub fn noop_filter_map_expr<T: MutVisitor>(mut e: P<Expr>, vis: &mut T) -> Optio
 }
 
 pub fn noop_flat_map_stmt<T: MutVisitor>(
-    Stmt { kind, mut span, mut id }: Stmt,
+    Stmt { kind, mut span, mut id, tokens }: Stmt,
     vis: &mut T,
 ) -> SmallVec<[Stmt; 1]> {
     vis.visit_id(&mut id);
     vis.visit_span(&mut span);
-    noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect()
+    noop_flat_map_stmt_kind(kind, vis)
+        .into_iter()
+        .map(|kind| Stmt { id, kind, span, tokens: tokens.clone() })
+        .collect()
 }
 
 pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
@@ -1305,7 +1311,7 @@ pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
         StmtKind::Semi(expr) => vis.filter_map_expr(expr).into_iter().map(StmtKind::Semi).collect(),
         StmtKind::Empty => smallvec![StmtKind::Empty],
         StmtKind::MacCall(mut mac) => {
-            let (mac_, _semi, attrs) = mac.deref_mut();
+            let MacCallStmt { mac: mac_, style: _, attrs } = mac.deref_mut();
             vis.visit_mac(mac_);
             visit_thin_attrs(attrs, vis);
             smallvec![StmtKind::MacCall(mac)]
@@ -1313,13 +1319,13 @@ pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
     }
 }
 
-pub fn noop_visit_vis<T: MutVisitor>(Spanned { node, span }: &mut Visibility, vis: &mut T) {
+pub fn noop_visit_vis<T: MutVisitor>(visibility: &mut Visibility, vis: &mut T) {
-    match node {
+    match &mut visibility.kind {
         VisibilityKind::Public | VisibilityKind::Crate(_) | VisibilityKind::Inherited => {}
         VisibilityKind::Restricted { path, id } => {
             vis.visit_path(path);
             vis.visit_id(id);
         }
     }
-    vis.visit_span(span);
+    vis.visit_span(&mut visibility.span);
 }

compiler/rustc_ast/src/token.rs

@@ -173,6 +173,7 @@ pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
             kw::Move,
             kw::Return,
             kw::True,
+            kw::Try,
             kw::Unsafe,
             kw::While,
             kw::Yield,
@@ -251,17 +252,6 @@ pub enum TokenKind {
     /// similarly to symbols in string literal tokens.
     DocComment(CommentKind, ast::AttrStyle, Symbol),
 
-    // Junk. These carry no data because we don't really care about the data
-    // they *would* carry, and don't really want to allocate a new ident for
-    // them. Instead, users could extract that from the associated span.
-    /// Whitespace.
-    Whitespace,
-    /// A comment.
-    Comment,
-    Shebang(Symbol),
-    /// A completely invalid token which should be skipped.
-    Unknown(Symbol),
-
     Eof,
 }
 
@@ -331,7 +321,7 @@ impl Token {
 
     /// Some token that will be thrown away later.
     pub fn dummy() -> Self {
-        Token::new(TokenKind::Whitespace, DUMMY_SP)
+        Token::new(TokenKind::Question, DUMMY_SP)
     }
 
     /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
@@ -360,7 +350,7 @@ impl Token {
     pub fn is_op(&self) -> bool {
         match self.kind {
             OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
-            | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false,
+            | Lifetime(..) | Interpolated(..) | Eof => false,
             _ => true,
         }
     }
@@ -676,8 +666,7 @@ impl Token {
             Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
             | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
             | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
-            | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment
+            | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
-            | Shebang(..) | Unknown(..) | Eof => return None,
         };
 
         Some(Token::new(kind, self.span.to(joint.span)))
@@ -711,7 +700,7 @@ pub enum Nonterminal {
 
 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Nonterminal, 40);
+rustc_data_structures::static_assert_size!(Nonterminal, 48);
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
 pub enum NonterminalKind {

compiler/rustc_ast/src/tokenstream.rs

@@ -83,7 +83,7 @@ impl TokenTree {
     }
 
     pub fn joint(self) -> TokenStream {
-        TokenStream::new(vec![(self, Joint)])
+        TokenStream::new(vec![(self, Spacing::Joint)])
     }
 
     pub fn token(kind: TokenKind, span: Span) -> TokenTree {
@@ -125,22 +125,20 @@ where
 /// instead of a representation of the abstract syntax tree.
 /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
-pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
+pub struct TokenStream(pub Lrc<Vec<TreeAndSpacing>>);
 
-pub type TreeAndJoint = (TokenTree, IsJoint);
+pub type TreeAndSpacing = (TokenTree, Spacing);
 
 // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
 rustc_data_structures::static_assert_size!(TokenStream, 8);
 
 #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)]
-pub enum IsJoint {
+pub enum Spacing {
+    Alone,
     Joint,
-    NonJoint,
 }
 
-use IsJoint::*;
-
 impl TokenStream {
     /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
     /// separating the two arguments with a comma for diagnostic suggestions.
@@ -153,7 +151,7 @@ impl TokenStream {
                 let sp = match (&ts, &next) {
                     (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue,
                     (
-                        (TokenTree::Token(token_left), NonJoint),
+                        (TokenTree::Token(token_left), Spacing::Alone),
                         (TokenTree::Token(token_right), _),
                     ) if ((token_left.is_ident() && !token_left.is_reserved_ident())
                         || token_left.is_lit())
@@ -162,11 +160,11 @@ impl TokenStream {
                     {
                         token_left.span
                     }
-                    ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(),
+                    ((TokenTree::Delimited(sp, ..), Spacing::Alone), _) => sp.entire(),
                     _ => continue,
                 };
                 let sp = sp.shrink_to_hi();
-                let comma = (TokenTree::token(token::Comma, sp), NonJoint);
+                let comma = (TokenTree::token(token::Comma, sp), Spacing::Alone);
                 suggestion = Some((pos, comma, sp));
             }
         }
@@ -184,19 +182,19 @@ impl TokenStream {
 
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
-        TokenStream::new(vec![(tree, NonJoint)])
+        TokenStream::new(vec![(tree, Spacing::Alone)])
     }
 }
 
-impl From<TokenTree> for TreeAndJoint {
+impl From<TokenTree> for TreeAndSpacing {
-    fn from(tree: TokenTree) -> TreeAndJoint {
+    fn from(tree: TokenTree) -> TreeAndSpacing {
-        (tree, NonJoint)
+        (tree, Spacing::Alone)
     }
 }
 
 impl iter::FromIterator<TokenTree> for TokenStream {
     fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
-        TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndJoint>>())
+        TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndSpacing>>())
     }
 }
 
@@ -209,7 +207,7 @@ impl PartialEq<TokenStream> for TokenStream {
 }
 
 impl TokenStream {
-    pub fn new(streams: Vec<TreeAndJoint>) -> TokenStream {
+    pub fn new(streams: Vec<TreeAndSpacing>) -> TokenStream {
         TokenStream(Lrc::new(streams))
     }
 
@@ -320,11 +318,11 @@ impl TokenStreamBuilder {
         // If `self` is not empty and the last tree within the last stream is a
         // token tree marked with `Joint`...
         if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() {
-            if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() {
+            if let Some((TokenTree::Token(last_token), Spacing::Joint)) = last_stream_lrc.last() {
                 // ...and `stream` is not empty and the first tree within it is
                 // a token tree...
                 let TokenStream(ref mut stream_lrc) = stream;
-                if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() {
+                if let Some((TokenTree::Token(token), spacing)) = stream_lrc.first() {
                     // ...and the two tokens can be glued together...
                     if let Some(glued_tok) = last_token.glue(&token) {
                         // ...then do so, by overwriting the last token
@@ -337,8 +335,7 @@ impl TokenStreamBuilder {
                         // Overwrite the last token tree with the merged
                         // token.
                         let last_vec_mut = Lrc::make_mut(last_stream_lrc);
-                        *last_vec_mut.last_mut().unwrap() =
+                        *last_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing);
-                            (TokenTree::Token(glued_tok), *is_joint);
 
                         // Remove the first token tree from `stream`. (This
                         // is almost always the only tree in `stream`.)
@@ -375,7 +372,7 @@ impl Iterator for Cursor {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
-        self.next_with_joint().map(|(tree, _)| tree)
+        self.next_with_spacing().map(|(tree, _)| tree)
     }
 }
 
@@ -384,7 +381,7 @@ impl Cursor {
         Cursor { stream, index: 0 }
     }
 
-    pub fn next_with_joint(&mut self) -> Option<TreeAndJoint> {
+    pub fn next_with_spacing(&mut self) -> Option<TreeAndSpacing> {
         if self.index < self.stream.len() {
             self.index += 1;
             Some(self.stream.0[self.index - 1].clone())

compiler/rustc_ast/src/util/lev_distance.rs

@@ -103,6 +103,7 @@ fn find_match_by_sorted_words<'a>(iter_names: Vec<&'a Symbol>, lookup: &str) ->
 
 fn sort_by_words(name: &str) -> String {
     let mut split_words: Vec<&str> = name.split('_').collect();
-    split_words.sort();
+    // We are sorting primitive &strs and can use unstable sort here
+    split_words.sort_unstable();
     split_words.join("_")
 }

compiler/rustc_ast/src/visit.rs

@@ -692,7 +692,7 @@ pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) {
         StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => visitor.visit_expr(expr),
         StmtKind::Empty => {}
         StmtKind::MacCall(ref mac) => {
-            let (ref mac, _, ref attrs) = **mac;
+            let MacCallStmt { ref mac, style: _, ref attrs } = **mac;
             visitor.visit_mac(mac);
             for attr in attrs.iter() {
                 visitor.visit_attribute(attr);
@@ -879,7 +879,7 @@ pub fn walk_arm<'a, V: Visitor<'a>>(visitor: &mut V, arm: &'a Arm) {
 }
 
 pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) {
-    if let VisibilityKind::Restricted { ref path, id } = vis.node {
+    if let VisibilityKind::Restricted { ref path, id } = vis.kind {
         visitor.visit_path(path, id);
     }
 }

compiler/rustc_ast_lowering/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_ast_lowering"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+rustc_arena = { path = "../rustc_arena" }
+tracing = "0.1"
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_target = { path = "../rustc_target" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_index = { path = "../rustc_index" }
+rustc_span = { path = "../rustc_span" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_session = { path = "../rustc_session" }
+rustc_ast = { path = "../rustc_ast" }
+smallvec = { version = "1.0", features = ["union", "may_dangle"] }

compiler/rustc_ast_lowering/src/expr.rs

@@ -1121,7 +1121,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // features. We check that at least one type is available for
                 // the current target.
                 let reg_class = reg.reg_class();
-                let mut required_features = vec![];
+                let mut required_features: Vec<&str> = vec![];
                 for &(_, feature) in reg_class.supported_types(asm_arch) {
                     if let Some(feature) = feature {
                         if self.sess.target_features.contains(&Symbol::intern(feature)) {
@@ -1135,7 +1135,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         break;
                     }
                 }
-                required_features.sort();
+                // We are sorting primitive strs here and can use unstable sort here
+                required_features.sort_unstable();
                 required_features.dedup();
                 match &required_features[..] {
                     [] => {}

compiler/rustc_ast_lowering/src/item.rs

@@ -27,7 +27,7 @@ pub(super) struct ItemLowerer<'a, 'lowering, 'hir> {
 impl ItemLowerer<'_, '_, '_> {
     fn with_trait_impl_ref(&mut self, impl_ref: &Option<TraitRef>, f: impl FnOnce(&mut Self)) {
         let old = self.lctx.is_in_trait_impl;
-        self.lctx.is_in_trait_impl = if let &None = impl_ref { false } else { true };
+        self.lctx.is_in_trait_impl = impl_ref.is_some();
         f(self);
         self.lctx.is_in_trait_impl = old;
     }
@@ -251,7 +251,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             ItemKind::ExternCrate(orig_name) => hir::ItemKind::ExternCrate(orig_name),
             ItemKind::Use(ref use_tree) => {
                 // Start with an empty prefix.
-                let prefix = Path { segments: vec![], span: use_tree.span };
+                let prefix = Path { segments: vec![], span: use_tree.span, tokens: None };
 
                 self.lower_use_tree(use_tree, &prefix, id, vis, ident, attrs)
             }
@@ -488,7 +488,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 *ident = tree.ident();
 
                 // First, apply the prefix to the path.
-                let mut path = Path { segments, span: path.span };
+                let mut path = Path { segments, span: path.span, tokens: None };
 
                 // Correctly resolve `self` imports.
                 if path.segments.len() > 1
@@ -540,8 +540,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 hir::ItemKind::Use(path, hir::UseKind::Single)
             }
             UseTreeKind::Glob => {
-                let path =
+                let path = self.lower_path(
-                    self.lower_path(id, &Path { segments, span: path.span }, ParamMode::Explicit);
+                    id,
+                    &Path { segments, span: path.span, tokens: None },
+                    ParamMode::Explicit,
+                );
                 hir::ItemKind::Use(path, hir::UseKind::Glob)
             }
             UseTreeKind::Nested(ref trees) => {
@@ -569,7 +572,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // for that we return the `{}` import (called the
                 // `ListStem`).
 
-                let prefix = Path { segments, span: prefix.span.to(path.span) };
+                let prefix = Path { segments, span: prefix.span.to(path.span), tokens: None };
 
                 // Add all the nested `PathListItem`s to the HIR.
                 for &(ref use_tree, id) in trees {
@@ -927,7 +930,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         v: &Visibility,
         explicit_owner: Option<NodeId>,
     ) -> hir::Visibility<'hir> {
-        let node = match v.node {
+        let node = match v.kind {
             VisibilityKind::Public => hir::VisibilityKind::Public,
             VisibilityKind::Crate(sugar) => hir::VisibilityKind::Crate(sugar),
             VisibilityKind::Restricted { ref path, id } => {

compiler/rustc_ast_lowering/src/lib.rs

@@ -967,6 +967,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             AttrKind::Normal(ref item) => AttrKind::Normal(AttrItem {
                 path: item.path.clone(),
                 args: self.lower_mac_args(&item.args),
+                tokens: None,
             }),
             AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data),
         };
@@ -1106,6 +1107,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                 id: node_id,
                                 kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
                                 span: constraint.span,
+                                tokens: None,
                             },
                             itctx,
                         );

compiler/rustc_ast_passes/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_ast_passes"
+version = "0.0.0"
+edition = "2018"
+
+[dependencies]
+itertools = "0.9"
+tracing = "0.1"
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_attr = { path = "../rustc_attr" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_feature = { path = "../rustc_feature" }
+rustc_parse = { path = "../rustc_parse" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_ast = { path = "../rustc_ast" }

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -198,13 +198,13 @@ impl<'a> AstValidator<'a> {
     }
 
     fn invalid_visibility(&self, vis: &Visibility, note: Option<&str>) {
-        if let VisibilityKind::Inherited = vis.node {
+        if let VisibilityKind::Inherited = vis.kind {
             return;
         }
 
         let mut err =
             struct_span_err!(self.session, vis.span, E0449, "unnecessary visibility qualifier");
-        if vis.node.is_pub() {
+        if vis.kind.is_pub() {
             err.span_label(vis.span, "`pub` not permitted here because it's implied");
         }
         if let Some(note) = note {
@@ -868,10 +868,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     .emit();
                 }
 
-                if !bounds
+                if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) {
-                    .iter()
-                    .any(|b| if let GenericBound::Trait(..) = *b { true } else { false })
-                {
                     self.err_handler().span_err(ty.span, "at least one trait must be specified");
                 }
 
@@ -990,12 +987,15 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     self.error_item_without_body(item.span, "function", msg, " { <body> }");
                 }
             }
-            ItemKind::ForeignMod(_) => {
+            ItemKind::ForeignMod(ForeignMod { unsafety, .. }) => {
                 let old_item = mem::replace(&mut self.extern_mod, Some(item));
                 self.invalid_visibility(
                     &item.vis,
                     Some("place qualifiers on individual foreign items instead"),
                 );
+                if let Unsafe::Yes(span) = unsafety {
+                    self.err_handler().span_err(span, "extern block cannot be declared unsafe");
+                }
                 visit::walk_item(self, item);
                 self.extern_mod = old_item;
                 return; // Avoid visiting again.
@@ -1029,7 +1029,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 walk_list!(self, visit_attribute, &item.attrs);
                 return;
             }
-            ItemKind::Mod(Mod { inline, .. }) => {
+            ItemKind::Mod(Mod { inline, unsafety, .. }) => {
+                if let Unsafe::Yes(span) = unsafety {
+                    self.err_handler().span_err(span, "module cannot be declared unsafe");
+                }
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
                 if !inline && !self.session.contains_name(&item.attrs, sym::path) {
                     self.check_mod_file_item_asciionly(item.ident);

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -260,7 +260,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     cfg => doc_cfg
                     masked => doc_masked
                     spotlight => doc_spotlight
-                    alias => doc_alias
                     keyword => doc_keyword
                 );
             }
@@ -594,7 +593,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
     }
 
     fn visit_vis(&mut self, vis: &'a ast::Visibility) {
-        if let ast::VisibilityKind::Crate(ast::CrateSugar::JustCrate) = vis.node {
+        if let ast::VisibilityKind::Crate(ast::CrateSugar::JustCrate) = vis.kind {
             gate_feature_post!(
                 &self,
                 crate_visibility_modifier,
@@ -608,6 +607,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
 pub fn check_crate(krate: &ast::Crate, sess: &Session) {
     maybe_stage_features(sess, krate);
+    check_incompatible_features(sess);
     let mut visitor = PostExpansionVisitor { sess, features: &sess.features_untracked() };
 
     let spans = sess.parse_sess.gated_spans.spans.borrow();
@@ -677,3 +677,36 @@ fn maybe_stage_features(sess: &Session, krate: &ast::Crate) {
         }
     }
 }
+
+fn check_incompatible_features(sess: &Session) {
+    let features = sess.features_untracked();
+
+    let declared_features = features
+        .declared_lang_features
+        .iter()
+        .copied()
+        .map(|(name, span, _)| (name, span))
+        .chain(features.declared_lib_features.iter().copied());
+
+    for (f1, f2) in rustc_feature::INCOMPATIBLE_FEATURES
+        .iter()
+        .filter(|&&(f1, f2)| features.enabled(f1) && features.enabled(f2))
+    {
+        if let Some((f1_name, f1_span)) = declared_features.clone().find(|(name, _)| name == f1) {
+            if let Some((f2_name, f2_span)) = declared_features.clone().find(|(name, _)| name == f2)
+            {
+                let spans = vec![f1_span, f2_span];
+                sess.struct_span_err(
+                    spans.clone(),
+                    &format!(
+                        "features `{}` and `{}` are incompatible, using them at the same time \
+                        is not allowed",
+                        f1_name, f2_name
+                    ),
+                )
+                .help("remove one of these features")
+                .emit();
+            }
+        }
+    }
+}

compiler/rustc_ast_pretty/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_ast_pretty"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+tracing = "0.1"
+rustc_span = { path = "../rustc_span" }
+rustc_ast = { path = "../rustc_ast" }
+rustc_target = { path = "../rustc_target" }

compiler/rustc_ast_pretty/src/pprust.rs

@@ -289,10 +289,6 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option<Span>)
             doc_comment_to_string(comment_kind, attr_style, data)
         }
         token::Eof => "<eof>".to_string(),
-        token::Whitespace => " ".to_string(),
-        token::Comment => "/* */".to_string(),
-        token::Shebang(s) => format!("/* shebang: {}*/", s),
-        token::Unknown(s) => s.to_string(),
 
         token::Interpolated(ref nt) => nonterminal_to_string(nt),
     }
@@ -1143,7 +1139,11 @@ impl<'a> State<'a> {
                 self.print_fn_full(sig, item.ident, gen, &item.vis, def, body, &item.attrs);
             }
             ast::ItemKind::Mod(ref _mod) => {
-                self.head(visibility_qualified(&item.vis, "mod"));
+                self.head(to_string(|s| {
+                    s.print_visibility(&item.vis);
+                    s.print_unsafety(_mod.unsafety);
+                    s.word("mod");
+                }));
                 self.print_ident(item.ident);
 
                 if _mod.inline || self.is_expanded {
@@ -1158,7 +1158,10 @@ impl<'a> State<'a> {
                 }
             }
             ast::ItemKind::ForeignMod(ref nmod) => {
-                self.head("extern");
+                self.head(to_string(|s| {
+                    s.print_unsafety(nmod.unsafety);
+                    s.word("extern");
+                }));
                 if let Some(abi) = nmod.abi {
                     self.print_literal(&abi.as_lit());
                     self.nbsp();
@@ -1356,7 +1359,7 @@ impl<'a> State<'a> {
     }
 
     crate fn print_visibility(&mut self, vis: &ast::Visibility) {
-        match vis.node {
+        match vis.kind {
             ast::VisibilityKind::Public => self.word_nbsp("pub"),
             ast::VisibilityKind::Crate(sugar) => match sugar {
                 ast::CrateSugar::PubCrate => self.word_nbsp("pub(crate)"),
@@ -1507,11 +1510,10 @@ impl<'a> State<'a> {
                 self.s.word(";");
             }
             ast::StmtKind::MacCall(ref mac) => {
-                let (ref mac, style, ref attrs) = **mac;
                 self.space_if_not_bol();
-                self.print_outer_attributes(attrs);
+                self.print_outer_attributes(&mac.attrs);
-                self.print_mac(mac);
+                self.print_mac(&mac.mac);
-                if style == ast::MacStmtStyle::Semicolon {
+                if mac.style == ast::MacStmtStyle::Semicolon {
                     self.s.word(";");
                 }
             }

compiler/rustc_ast_pretty/src/pprust/tests.rs

@@ -1,7 +1,6 @@
 use super::*;
 
 use rustc_ast as ast;
-use rustc_span::source_map::respan;
 use rustc_span::symbol::Ident;
 use rustc_span::with_default_session_globals;
 
@@ -45,7 +44,11 @@ fn test_variant_to_string() {
 
         let var = ast::Variant {
             ident,
-            vis: respan(rustc_span::DUMMY_SP, ast::VisibilityKind::Inherited),
+            vis: ast::Visibility {
+                span: rustc_span::DUMMY_SP,
+                kind: ast::VisibilityKind::Inherited,
+                tokens: None,
+            },
             attrs: Vec::new(),
             id: ast::DUMMY_NODE_ID,
             data: ast::VariantData::Unit(ast::DUMMY_NODE_ID),

compiler/rustc_attr/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_attr"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_span = { path = "../rustc_span" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_feature = { path = "../rustc_feature" }
+rustc_lexer = { path = "../rustc_lexer" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_session = { path = "../rustc_session" }
+rustc_ast = { path = "../rustc_ast" }
+version_check = "0.9"

compiler/rustc_attr/src/builtin.rs

@@ -145,8 +145,6 @@ pub struct ConstStability {
     pub feature: Symbol,
     /// whether the function has a `#[rustc_promotable]` attribute
     pub promotable: bool,
-    /// whether the function has a `#[rustc_allow_const_fn_ptr]` attribute
-    pub allow_const_fn_ptr: bool,
 }
 
 /// The available stability levels.
@@ -160,10 +158,10 @@ pub enum StabilityLevel {
 
 impl StabilityLevel {
     pub fn is_unstable(&self) -> bool {
-        if let StabilityLevel::Unstable { .. } = *self { true } else { false }
+        matches!(self, StabilityLevel::Unstable { .. })
     }
     pub fn is_stable(&self) -> bool {
-        if let StabilityLevel::Stable { .. } = *self { true } else { false }
+        matches!(self, StabilityLevel::Stable { .. })
     }
 }
 
@@ -190,7 +188,6 @@ where
     let mut stab: Option<Stability> = None;
     let mut const_stab: Option<ConstStability> = None;
     let mut promotable = false;
-    let mut allow_const_fn_ptr = false;
     let diagnostic = &sess.parse_sess.span_diagnostic;
 
     'outer: for attr in attrs_iter {
@@ -200,7 +197,6 @@ where
             sym::unstable,
             sym::stable,
             sym::rustc_promotable,
-            sym::rustc_allow_const_fn_ptr,
         ]
         .iter()
         .any(|&s| attr.has_name(s))
@@ -215,9 +211,6 @@ where
         if attr.has_name(sym::rustc_promotable) {
             promotable = true;
         }
-        if attr.has_name(sym::rustc_allow_const_fn_ptr) {
-            allow_const_fn_ptr = true;
-        }
         // attributes with data
         else if let Some(MetaItem { kind: MetaItemKind::List(ref metas), .. }) = meta {
             let meta = meta.as_ref().unwrap();
@@ -301,7 +294,7 @@ where
                                                 .emit();
                                             };
                                             match issue.parse() {
-                                                Ok(num) if num == 0 => {
+                                                Ok(0) => {
                                                     emit_diag(
                                                         "`issue` must not be \"0\", \
                                                         use \"none\" instead",
@@ -360,12 +353,8 @@ where
                             if sym::unstable == meta_name {
                                 stab = Some(Stability { level, feature });
                             } else {
-                                const_stab = Some(ConstStability {
+                                const_stab =
-                                    level,
+                                    Some(ConstStability { level, feature, promotable: false });
-                                    feature,
-                                    promotable: false,
-                                    allow_const_fn_ptr: false,
-                                });
                             }
                         }
                         (None, _, _) => {
@@ -440,12 +429,8 @@ where
                             if sym::stable == meta_name {
                                 stab = Some(Stability { level, feature });
                             } else {
-                                const_stab = Some(ConstStability {
+                                const_stab =
-                                    level,
+                                    Some(ConstStability { level, feature, promotable: false });
-                                    feature,
-                                    promotable: false,
-                                    allow_const_fn_ptr: false,
-                                });
                             }
                         }
                         (None, _) => {
@@ -464,18 +449,16 @@ where
     }
 
     // Merge the const-unstable info into the stability info
-    if promotable || allow_const_fn_ptr {
+    if promotable {
         if let Some(ref mut stab) = const_stab {
             stab.promotable = promotable;
-            stab.allow_const_fn_ptr = allow_const_fn_ptr;
         } else {
             struct_span_err!(
                 diagnostic,
                 item_sp,
                 E0717,
-                "rustc_promotable and rustc_allow_const_fn_ptr attributes \
+                "`rustc_promotable` attribute must be paired with either a `rustc_const_unstable` \
-                      must be paired with either a rustc_const_unstable or a rustc_const_stable \
+                or a `rustc_const_stable` attribute"
-                      attribute"
             )
             .emit();
         }
@@ -1022,14 +1005,21 @@ pub fn find_transparency(
 
 pub fn allow_internal_unstable<'a>(
     sess: &'a Session,
-    attrs: &[Attribute],
+    attrs: &'a [Attribute],
 ) -> Option<impl Iterator<Item = Symbol> + 'a> {
-    let attr = sess.find_by_name(attrs, sym::allow_internal_unstable)?;
+    let attrs = sess.filter_by_name(attrs, sym::allow_internal_unstable);
-    let list = attr.meta_item_list().or_else(|| {
+    let list = attrs
-        sess.diagnostic()
+        .filter_map(move |attr| {
-            .span_err(attr.span, "allow_internal_unstable expects list of feature names");
+            attr.meta_item_list().or_else(|| {
-        None
+                sess.diagnostic().span_err(
-    })?;
+                    attr.span,
+                    "`allow_internal_unstable` expects a list of feature names",
+                );
+                None
+            })
+        })
+        .flatten();
+
     Some(list.into_iter().filter_map(move |it| {
         let name = it.ident().map(|ident| ident.name);
         if name.is_none() {

compiler/rustc_builtin_macros/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_builtin_macros"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+rustc_parse_format = { path = "../rustc_parse_format" }
+tracing = "0.1"
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_attr = { path = "../rustc_attr" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_feature = { path = "../rustc_feature" }
+rustc_parse = { path = "../rustc_parse" }
+rustc_target = { path = "../rustc_target" }
+rustc_session = { path = "../rustc_session" }
+smallvec = { version = "1.0", features = ["union", "may_dangle"] }
+rustc_ast = { path = "../rustc_ast" }
+rustc_expand = { path = "../rustc_expand" }
+rustc_span = { path = "../rustc_span" }

compiler/rustc_builtin_macros/src/asm.rs

@@ -368,7 +368,7 @@ fn parse_reg<'a>(
     explicit_reg: &mut bool,
 ) -> Result<ast::InlineAsmRegOrRegClass, DiagnosticBuilder<'a>> {
     p.expect(&token::OpenDelim(token::DelimToken::Paren))?;
-    let result = match p.token.kind {
+    let result = match p.token.uninterpolate().kind {
         token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name),
         token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
             *explicit_reg = true;

compiler/rustc_builtin_macros/src/cmdline_attrs.rs

@@ -15,7 +15,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.token.span;
-        let AttrItem { path, args } = match parser.parse_attr_item() {
+        let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item() {
             Ok(ai) => ai,
             Err(mut err) => {
                 err.emit();

compiler/rustc_builtin_macros/src/concat_idents.rs

@@ -27,15 +27,15 @@ pub fn expand_concat_idents<'cx>(
                 }
             }
         } else {
-            match e {
+            if let TokenTree::Token(token) = e {
-                TokenTree::Token(Token { kind: token::Ident(name, _), .. }) => {
+                if let Some((ident, _)) = token.ident() {
-                    res_str.push_str(&name.as_str())
+                    res_str.push_str(&ident.name.as_str());
-                }
+                    continue;
-                _ => {
-                    cx.span_err(sp, "concat_idents! requires ident args.");
-                    return DummyResult::any(sp);
                 }
             }
+
+            cx.span_err(sp, "concat_idents! requires ident args.");
+            return DummyResult::any(sp);
         }
     }
 
@@ -61,6 +61,7 @@ pub fn expand_concat_idents<'cx>(
                 id: ast::DUMMY_NODE_ID,
                 kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)),
                 span: self.ident.span,
+                tokens: None,
             }))
         }
     }

compiler/rustc_builtin_macros/src/deriving/debug.rs

@@ -133,5 +133,5 @@ fn stmt_let_underscore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<ast::Expr>) -> as
         span: sp,
         attrs: ast::AttrVec::new(),
     });
-    ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
+    ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None }
 }

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -187,7 +187,6 @@ use rustc_ast::{GenericArg, GenericParamKind, VariantData};
 use rustc_attr as attr;
 use rustc_data_structures::map_in_place::MapInPlace;
 use rustc_expand::base::{Annotatable, ExtCtxt};
-use rustc_span::source_map::respan;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
 
@@ -532,7 +531,11 @@ impl<'a> TraitDef<'a> {
                 id: ast::DUMMY_NODE_ID,
                 span: self.span,
                 ident,
-                vis: respan(self.span.shrink_to_lo(), ast::VisibilityKind::Inherited),
+                vis: ast::Visibility {
+                    span: self.span.shrink_to_lo(),
+                    kind: ast::VisibilityKind::Inherited,
+                    tokens: None,
+                },
                 attrs: Vec::new(),
                 kind: ast::AssocItemKind::TyAlias(
                     ast::Defaultness::Final,
@@ -933,7 +936,11 @@ impl<'a> MethodDef<'a> {
             id: ast::DUMMY_NODE_ID,
             attrs: self.attributes.clone(),
             span: trait_.span,
-            vis: respan(trait_lo_sp, ast::VisibilityKind::Inherited),
+            vis: ast::Visibility {
+                span: trait_lo_sp,
+                kind: ast::VisibilityKind::Inherited,
+                tokens: None,
+            },
             ident: method_ident,
             kind: ast::AssocItemKind::Fn(def, sig, fn_generics, Some(body_block)),
             tokens: None,
@@ -1522,7 +1529,7 @@ impl<'a> TraitDef<'a> {
             }
         }
 
-        let is_tuple = if let ast::VariantData::Tuple(..) = struct_def { true } else { false };
+        let is_tuple = matches!(struct_def, ast::VariantData::Tuple(..));
         match (just_spans.is_empty(), named_idents.is_empty()) {
             (false, false) => cx.span_bug(
                 self.span,

compiler/rustc_builtin_macros/src/deriving/mod.rs

@@ -75,6 +75,7 @@ fn call_intrinsic(
         id: ast::DUMMY_NODE_ID,
         rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated),
         span,
+        tokens: None,
     }))
 }
 

compiler/rustc_builtin_macros/src/format.rs

@@ -135,21 +135,52 @@ fn parse_args<'a>(
         return Err(ecx.struct_span_err(sp, "requires at least a format string argument"));
     }
 
-    let fmtstr = p.parse_expr()?;
+    let first_token = &p.token;
+    let fmtstr = match first_token.kind {
+        token::TokenKind::Literal(token::Lit {
+            kind: token::LitKind::Str | token::LitKind::StrRaw(_),
+            ..
+        }) => {
+            // If the first token is a string literal, then a format expression
+            // is constructed from it.
+            //
+            // This allows us to properly handle cases when the first comma
+            // after the format string is mistakenly replaced with any operator,
+            // which cause the expression parser to eat too much tokens.
+            p.parse_literal_maybe_minus()?
+        }
+        _ => {
+            // Otherwise, we fall back to the expression parser.
+            p.parse_expr()?
+        }
+    };
+
     let mut first = true;
     let mut named = false;
 
     while p.token != token::Eof {
         if !p.eat(&token::Comma) {
             if first {
-                // After `format!(""` we always expect *only* a comma...
+                p.clear_expected_tokens();
-                let mut err = ecx.struct_span_err(p.token.span, "expected token: `,`");
+            }
-                err.span_label(p.token.span, "expected `,`");
+
-                p.maybe_annotate_with_ascription(&mut err, false);
+            // `Parser::expect` tries to recover using the
-                return Err(err);
+            // `Parser::unexpected_try_recover` function. This function is able
-            } else {
+            // to recover if the expected token is a closing delimiter.
-                // ...after that delegate to `expect` to also include the other expected tokens.
+            //
-                let _ = p.expect(&token::Comma)?;
+            // As `,` is not a closing delimiter, it will always return an `Err`
+            // variant.
+            let mut err = p.expect(&token::Comma).unwrap_err();
+
+            match token::TokenKind::Comma.similar_tokens() {
+                Some(tks) if tks.contains(&p.token.kind) => {
+                    // If a similar token is found, then it may be a typo. We
+                    // consider it as a comma, and continue parsing.
+                    err.emit();
+                    p.bump();
+                }
+                // Otherwise stop the parsing and return the error.
+                _ => return Err(err),
             }
         }
         first = false;
@@ -512,9 +543,12 @@ impl<'a, 'b> Context<'a, 'b> {
                             let idx = self.args.len();
                             self.arg_types.push(Vec::new());
                             self.arg_unique_types.push(Vec::new());
-                            self.args.push(
+                            let span = if self.is_literal {
-                                self.ecx.expr_ident(self.fmtsp, Ident::new(name, self.fmtsp)),
+                                *self.arg_spans.get(self.curpiece).unwrap_or(&self.fmtsp)
-                            );
+                            } else {
+                                self.fmtsp
+                            };
+                            self.args.push(self.ecx.expr_ident(span, Ident::new(name, span)));
                             self.names.insert(name, idx);
                             self.verify_arg_type(Exact(idx), ty)
                         } else {

compiler/rustc_builtin_macros/src/format_foreign.rs

@@ -166,14 +166,14 @@ pub mod printf {
             let cap = self.span.len() + if has_options { 2 } else { 0 };
             let mut s = String::with_capacity(cap);
 
-            s.push_str("{");
+            s.push('{');
 
             if let Some(arg) = self.parameter {
                 write!(s, "{}", arg.checked_sub(1)?).ok()?;
             }
 
             if has_options {
-                s.push_str(":");
+                s.push(':');
 
                 let align = if let Some(fill) = fill {
                     s.push_str(fill);
@@ -191,11 +191,11 @@ pub mod printf {
                 }
 
                 if alt {
-                    s.push_str("#");
+                    s.push('#');
                 }
 
                 if zero_fill {
-                    s.push_str("0");
+                    s.push('0');
                 }
 
                 if let Some(width) = width {
@@ -203,7 +203,7 @@ pub mod printf {
                 }
 
                 if let Some(precision) = precision {
-                    s.push_str(".");
+                    s.push('.');
                     precision.translate(&mut s).ok()?;
                 }
 
@@ -212,7 +212,7 @@ pub mod printf {
                 }
             }
 
-            s.push_str("}");
+            s.push('}');
             Some(s)
         }
     }
@@ -518,8 +518,7 @@ pub mod printf {
                         .and_then(|end| end.at_next_cp())
                         .map(|end| (next.slice_between(end).unwrap(), end));
                     let end = match end {
-                        Some(("32", end)) => end,
+                        Some(("32" | "64", end)) => end,
-                        Some(("64", end)) => end,
                         _ => next,
                     };
                     state = Type;

compiler/rustc_builtin_macros/src/global_asm.rs

@@ -14,7 +14,6 @@ use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::DiagnosticBuilder;
 use rustc_expand::base::{self, *};
-use rustc_span::source_map::respan;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 use smallvec::smallvec;
@@ -30,7 +29,11 @@ pub fn expand_global_asm<'cx>(
             attrs: Vec::new(),
             id: ast::DUMMY_NODE_ID,
             kind: ast::ItemKind::GlobalAsm(P(global_asm)),
-            vis: respan(sp.shrink_to_lo(), ast::VisibilityKind::Inherited),
+            vis: ast::Visibility {
+                span: sp.shrink_to_lo(),
+                kind: ast::VisibilityKind::Inherited,
+                tokens: None,
+            },
             span: cx.with_def_site_ctxt(sp),
             tokens: None,
         })]),

compiler/rustc_builtin_macros/src/lib.rs

@@ -1,7 +1,7 @@
 //! This crate contains implementations of built-in macros and other code generating facilities
 //! injecting code into the crate before it is lowered to HIR.
 
-#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]

compiler/rustc_builtin_macros/src/proc_macro_harness.rs

@@ -98,7 +98,7 @@ pub fn inject(
 
 impl<'a> CollectProcMacros<'a> {
     fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) {
-        if self.is_proc_macro_crate && self.in_root && vis.node.is_pub() {
+        if self.is_proc_macro_crate && self.in_root && vis.kind.is_pub() {
             self.handler.span_err(
                 sp,
                 "`proc-macro` crate types currently cannot export any items other \
@@ -184,7 +184,7 @@ impl<'a> CollectProcMacros<'a> {
             Vec::new()
         };
 
-        if self.in_root && item.vis.node.is_pub() {
+        if self.in_root && item.vis.kind.is_pub() {
             self.macros.push(ProcMacro::Derive(ProcMacroDerive {
                 id: item.id,
                 span: item.span,
@@ -204,7 +204,7 @@ impl<'a> CollectProcMacros<'a> {
     }
 
     fn collect_attr_proc_macro(&mut self, item: &'a ast::Item) {
-        if self.in_root && item.vis.node.is_pub() {
+        if self.in_root && item.vis.kind.is_pub() {
             self.macros.push(ProcMacro::Def(ProcMacroDef {
                 id: item.id,
                 span: item.span,
@@ -223,7 +223,7 @@ impl<'a> CollectProcMacros<'a> {
     }
 
     fn collect_bang_proc_macro(&mut self, item: &'a ast::Item) {
-        if self.in_root && item.vis.node.is_pub() {
+        if self.in_root && item.vis.kind.is_pub() {
             self.macros.push(ProcMacro::Def(ProcMacroDef {
                 id: item.id,
                 span: item.span,

compiler/rustc_builtin_macros/src/test.rs

@@ -7,7 +7,6 @@ use rustc_ast::attr;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::*;
 use rustc_session::Session;
-use rustc_span::source_map::respan;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
 
@@ -35,7 +34,11 @@ pub fn expand_test_case(
     let sp = ecx.with_def_site_ctxt(attr_sp);
     let mut item = anno_item.expect_item();
     item = item.map(|mut item| {
-        item.vis = respan(item.vis.span, ast::VisibilityKind::Public);
+        item.vis = ast::Visibility {
+            span: item.vis.span,
+            kind: ast::VisibilityKind::Public,
+            tokens: None,
+        };
         item.ident.span = item.ident.span.with_ctxt(sp.ctxt());
         item.attrs.push(ecx.attribute(ecx.meta_word(sp, sym::rustc_test_marker)));
         item
@@ -292,7 +295,7 @@ pub fn expand_test_or_bench(
         ),
     );
     test_const = test_const.map(|mut tc| {
-        tc.vis.node = ast::VisibilityKind::Public;
+        tc.vis.kind = ast::VisibilityKind::Public;
         tc
     });
 

compiler/rustc_builtin_macros/src/test_harness.rs

@@ -10,7 +10,6 @@ use rustc_expand::expand::{AstFragment, ExpansionConfig};
 use rustc_feature::Features;
 use rustc_session::Session;
 use rustc_span::hygiene::{AstPass, SyntaxContext, Transparency};
-use rustc_span::source_map::respan;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::spec::PanicStrategy;
@@ -333,7 +332,7 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
         attrs: vec![main_attr],
         id: ast::DUMMY_NODE_ID,
         kind: main,
-        vis: respan(sp, ast::VisibilityKind::Public),
+        vis: ast::Visibility { span: sp, kind: ast::VisibilityKind::Public, tokens: None },
         span: sp,
         tokens: None,
     });

compiler/rustc_codegen_llvm/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_codegen_llvm"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+test = false
+doctest = false
+
+[dependencies]
+bitflags = "1.0"
+libc = "0.2"
+measureme = "0.7.1"
+snap = "1"
+tracing = "0.1"
+rustc_middle = { path = "../rustc_middle" }
+rustc-demangle = "0.1"
+rustc_attr = { path = "../rustc_attr" }
+rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_feature = { path = "../rustc_feature" }
+rustc_fs_util = { path = "../rustc_fs_util" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
+rustc_llvm = { path = "../rustc_llvm" }
+rustc_session = { path = "../rustc_session" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_target = { path = "../rustc_target" }
+smallvec = { version = "1.0", features = ["union", "may_dangle"] }
+rustc_ast = { path = "../rustc_ast" }
+rustc_span = { path = "../rustc_span" }