New upstream version 1.54.0+dfsg1

Cargo.lock

@@ -55,7 +55,7 @@ dependencies = [
  "markup5ever_rcdom",
  "matches",
  "tendril",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
@@ -252,7 +252,7 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da"
 
 [[package]]
 name = "cargo"
-version = "0.54.0"
+version = "0.55.0"
 dependencies = [
  "anyhow",
  "atty",
@@ -278,6 +278,7 @@ dependencies = [
  "humantime 2.0.1",
  "ignore",
  "im-rc",
+ "itertools 0.10.0",
  "jobserver",
  "lazy_static",
  "lazycell",
@@ -293,7 +294,7 @@ dependencies = [
  "rand 0.8.3",
  "rustc-workspace-hack",
  "rustfix",
- "semver 0.10.0",
+ "semver 1.0.3",
  "serde",
  "serde_ignored",
  "serde_json",
@@ -305,7 +306,7 @@ dependencies = [
  "toml",
  "unicode-width",
  "unicode-xid",
- "url 2.1.1",
+ "url 2.2.2",
  "walkdir",
  "winapi 0.3.9",
 ]
@@ -378,7 +379,7 @@ dependencies = [
  "serde_json",
  "tar",
  "toml",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
@@ -441,9 +442,9 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.60"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c"
+checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
 dependencies = [
  "jobserver",
 ]
@@ -548,13 +549,13 @@ dependencies = [
 
 [[package]]
 name = "clippy"
-version = "0.1.53"
+version = "0.1.54"
 dependencies = [
  "cargo_metadata 0.12.0",
- "clippy-mini-macro-test",
  "clippy_lints",
  "compiletest_rs",
  "derive-new",
+ "filetime",
  "quote",
  "regex",
  "rustc-workspace-hack",
@@ -566,10 +567,6 @@ dependencies = [
  "tester",
 ]
 
-[[package]]
-name = "clippy-mini-macro-test"
-version = "0.2.0"
-
 [[package]]
 name = "clippy_dev"
 version = "0.0.1"
@@ -585,7 +582,7 @@ dependencies = [
 
 [[package]]
 name = "clippy_lints"
-version = "0.1.53"
+version = "0.1.54"
 dependencies = [
  "cargo_metadata 0.12.0",
  "clippy_utils",
@@ -597,14 +594,15 @@ dependencies = [
  "rustc-semver",
  "semver 0.11.0",
  "serde",
+ "serde_json",
  "toml",
  "unicode-normalization",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
 name = "clippy_utils"
-version = "0.1.53"
+version = "0.1.54"
 dependencies = [
  "if_chain",
  "itertools 0.9.0",
@@ -654,9 +652,9 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.39"
+version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b"
+checksum = "787187ae221adfcda34b03006f1617099e4ae26b50e5a4db282496014ab75837"
 dependencies = [
  "cc",
  "rustc-std-workspace-core",
@@ -749,7 +747,7 @@ dependencies = [
  "percent-encoding 2.1.0",
  "serde",
  "serde_json",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
@@ -864,24 +862,24 @@ dependencies = [
 
 [[package]]
 name = "curl"
-version = "0.4.34"
+version = "0.4.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e268162af1a5fe89917ae25ba3b0a77c8da752bdc58e7dbb4f15b91fbd33756e"
+checksum = "d0bac9f84ca0977c4d9b8db998689de55b9e976656a6bc87fada2ca710d504c7"
 dependencies = [
  "curl-sys",
  "libc",
  "openssl-probe",
  "openssl-sys",
  "schannel",
- "socket2",
+ "socket2 0.4.0",
  "winapi 0.3.9",
 ]
 
 [[package]]
 name = "curl-sys"
-version = "0.4.39+curl-7.74.0"
+version = "0.4.42+curl-7.76.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07a8ce861e7b68a0b394e814d7ee9f1b2750ff8bd10372c6ad3bacc10e86f874"
+checksum = "4636d8d6109c842707018a104051436bffb8991ea20b2d1293db70b6e0ee4c7c"
 dependencies = [
  "cc",
  "libc",
@@ -1136,13 +1134,13 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
 [[package]]
 name = "filetime"
-version = "0.2.12"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e"
+checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "libc",
- "redox_syscall 0.1.57",
+ "redox_syscall",
  "winapi 0.3.9",
 ]
 
@@ -1186,6 +1184,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
+[[package]]
+name = "form_urlencoded"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
+dependencies = [
+ "matches",
+ "percent-encoding 2.1.0",
+]
+
 [[package]]
 name = "fortanix-sgx-abi"
 version = "0.3.3"
@@ -1440,7 +1448,7 @@ dependencies = [
  "log",
  "openssl-probe",
  "openssl-sys",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
@@ -1452,7 +1460,7 @@ dependencies = [
  "curl",
  "git2",
  "log",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
@@ -1704,6 +1712,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "0.4.6"
@@ -1732,11 +1749,10 @@ dependencies = [
  "fs-err",
  "getopts",
  "jsonpath_lib",
- "lazy_static",
+ "once_cell",
  "regex",
- "serde",
  "serde_json",
- "shlex 0.1.1",
+ "shlex",
 ]
 
 [[package]]
@@ -1974,11 +1990,11 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.11"
+version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -2002,7 +2018,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_repr",
- "url 2.1.1",
+ "url 2.2.2",
 ]
 
 [[package]]
@@ -2117,19 +2133,19 @@ dependencies = [
  "serde",
  "serde_derive",
  "serde_json",
- "shlex 1.0.0",
+ "shlex",
  "tempfile",
  "toml",
 ]
 
 [[package]]
 name = "measureme"
-version = "9.1.0"
+version = "9.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a98e07fe802486895addb2b5467f33f205e82c426bfaf350f5d8109b137767c"
+checksum = "78f7a41bc6f856a2cf0e95094ad5121f82500e2d9a0f3c0171d98f6566d8117d"
 dependencies = [
  "log",
- "memmap",
+ "memmap2",
  "parking_lot",
  "perf-event-open-sys",
  "rustc-hash",
@@ -2138,19 +2154,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.3.3"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
+checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
-
-[[package]]
-name = "memmap"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
-dependencies = [
- "libc",
- "winapi 0.3.9",
-]
 
 [[package]]
 name = "memmap2"
@@ -2194,9 +2200,9 @@ dependencies = [
 
 [[package]]
 name = "minifier"
-version = "0.0.39"
+version = "0.0.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cdf618de5c9c98d4a7b2e0d1f1e44f82a19196cfd94040bb203621c25d28d98"
+checksum = "5594542d20834f2b974f5e5fb8e0cf1c67a2119dcadc29ef5d93a081fb30cc08"
 dependencies = [
  "macro-utils",
 ]
@@ -2273,7 +2279,7 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
 dependencies = [
- "socket2",
+ "socket2 0.3.19",
  "winapi 0.3.9",
 ]
 
@@ -2288,6 +2294,7 @@ dependencies = [
  "hex 0.4.2",
  "libc",
  "log",
+ "measureme",
  "rand 0.8.3",
  "rustc-workspace-hack",
  "rustc_version",
@@ -2352,6 +2359,17 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "object"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8bc1d42047cf336f0f939c99e97183cf31551bf0f2865a2ec9c8d91fd4ffb5e"
+dependencies = [
+ "crc32fast",
+ "indexmap",
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.7.2"
@@ -2390,15 +2408,15 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.30"
+version = "0.10.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4"
+checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577"
 dependencies = [
  "bitflags",
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "foreign-types",
- "lazy_static",
  "libc",
+ "once_cell",
  "openssl-sys",
 ]
 
@@ -2410,18 +2428,18 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
 
 [[package]]
 name = "openssl-src"
-version = "111.12.0+1.1.1h"
+version = "111.15.0+1.1.1k"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "858a4132194f8570a7ee9eb8629e85b23cbc4565f2d4a162e87556e5956abf61"
+checksum = "b1a5f6ae2ac04393b217ea9f700cd04fa9bf3d93fae2872069f3d15d908af70a"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.58"
+version = "0.9.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
+checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f"
 dependencies = [
  "autocfg",
  "cc",
@@ -2515,7 +2533,7 @@ dependencies = [
  "cfg-if 1.0.0",
  "instant",
  "libc",
- "redox_syscall 0.2.5",
+ "redox_syscall",
  "smallvec",
  "winapi 0.3.9",
 ]
@@ -2844,9 +2862,9 @@ dependencies = [
 
 [[package]]
 name = "racer"
-version = "2.1.46"
+version = "2.1.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7cbda48a9124ed2e83766d2c15e3725710d344abca35fad8cf52341a55883b1"
+checksum = "7fec2e85e7a30f8fd31b7cf288ad363b5e51fd2cb6f53b416b0cfaabd84e1ccb"
 dependencies = [
  "bitflags",
  "clap",
@@ -2999,12 +3017,6 @@ dependencies = [
  "num_cpus",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.1.57"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
-
 [[package]]
 name = "redox_syscall"
 version = "0.2.5"
@@ -3021,7 +3033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
  "getrandom 0.2.0",
- "redox_syscall 0.2.5",
+ "redox_syscall",
 ]
 
 [[package]]
@@ -3076,7 +3088,7 @@ dependencies = [
  "anyhow",
  "cargo",
  "cargo-util",
- "cargo_metadata 0.8.2",
+ "cargo_metadata 0.12.0",
  "clippy_lints",
  "crossbeam-channel",
  "difference",
@@ -3113,7 +3125,7 @@ dependencies = [
  "tokio",
  "tokio-util",
  "toml",
- "url 2.1.1",
+ "url 2.2.2",
  "walkdir",
 ]
 
@@ -3209,9 +3221,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_arena"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "526610f47139efa440178239553b59ea805ff57a532b4e295c71d2a9b18fd676"
+checksum = "550ca1a0925d31a0af089b18c89f5adf3b286e319e3e1f1a5204c21bd2f17371"
 dependencies = [
  "rustc-ap-rustc_data_structures",
  "smallvec",
@@ -3219,9 +3231,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_ast"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf6a9dda0804a7243b0282e3b75a8cf4654c7a61f033e587751941e1fe39391b"
+checksum = "4aa53b68080df17994a54747f7c37b0686288a670efb9ba3b382ce62e744aed2"
 dependencies = [
  "bitflags",
  "rustc-ap-rustc_data_structures",
@@ -3236,9 +3248,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_ast_pretty"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82f5019be8b41a58664169fd2f4b1a37fe82705681db394b76419e4e87d40ab1"
+checksum = "0ae71e68fada466a4b2c39c79ca6aee3226587abe6787170d2f6c92237569565"
 dependencies = [
  "rustc-ap-rustc_ast",
  "rustc-ap-rustc_span",
@@ -3247,9 +3259,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_data_structures"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a701717fb14549331085756b9741ae3b4bf35808489f1887d72c1d0e0fe52b77"
+checksum = "faa484d6e0ca32d1d82303647275c696f745599b3d97e686f396ceef5b99d7ae"
 dependencies = [
  "arrayvec",
  "bitflags",
@@ -3279,9 +3291,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_errors"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3182ce85e8bfc96443475547f2f5aa2b5e67655d9b88721795f36f0ba9e265a"
+checksum = "5f85ba19cca320ad797e3a29c35cab9bddfff0e7adbde336a436249e54cee7b1"
 dependencies = [
  "annotate-snippets",
  "atty",
@@ -3299,9 +3311,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_feature"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed033b93270126ef60963c3ebbd0e026bf53b985172b6366c7b0e7881c9d507"
+checksum = "97d538adab96b8b2b1ca9fcd4c8c47d4e23e862a23d1a38b6c15cd8fd52b34b1"
 dependencies = [
  "rustc-ap-rustc_data_structures",
  "rustc-ap-rustc_span",
@@ -3309,21 +3321,21 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_fs_util"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28ee6531986a205101e09fd143d7bf31897388f33b1814d4bcc45fd62211dca6"
+checksum = "8ad6f13d240944fa8f360d2f3b849a7cadaec75e477829e7dde61e838deda83d"
 
 [[package]]
 name = "rustc-ap-rustc_graphviz"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3398fddc0e23d2db89c036f8952ddf78cadc597f7059752116e69483e164a5b6"
+checksum = "08b3451153cc5828c02cc4f1a0df146d25ac4b3382a112e25fd9d3f5bff15cdc"
 
 [[package]]
 name = "rustc-ap-rustc_index"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dca4e27eb5b701f6bbd47d8fc9d242378fca3e4107a519a28415c2989c4a3bd3"
+checksum = "cd39a9f01b442c629bdff5778cb3dd29b7c2ea4afe62d5ab61d216bd1b556692"
 dependencies = [
  "arrayvec",
  "rustc-ap-rustc_macros",
@@ -3332,18 +3344,18 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_lexer"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "786bbfe9d4d5264294c1819dbf1497a2480b583d5eda1ca9ae22e12d6661f5df"
+checksum = "a5de290c44a90e671d2cd730062b9ef73d11155da7e44e7741d633e1e51e616e"
 dependencies = [
  "unicode-xid",
 ]
 
 [[package]]
 name = "rustc-ap-rustc_lint_defs"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be2f045e2b999c154ec505d5fea69c994b742f3ebd2f552d11a6c81723921e47"
+checksum = "69570b4beb61088926b131579865bbe70d124d30778c46307a62ec8b310ae462"
 dependencies = [
  "rustc-ap-rustc_ast",
  "rustc-ap-rustc_data_structures",
@@ -3356,9 +3368,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_macros"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27789cd26d6b9e2fdfa68a262a20664d79ca67d31a3886d40fb88ebf6935869c"
+checksum = "86bd877df37f15c5a44d9679d1b5207ebc95f3943fbc336eeac670195ac58610"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3368,9 +3380,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_parse"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dc331f4958350679679e619d63a891e8d5d34ef99087068c89aa9e657d52caa"
+checksum = "02502d8522ba31d0bcad28a78822b68c1b6ba947a2b4aa6a2341b30594379b80"
 dependencies = [
  "bitflags",
  "rustc-ap-rustc_ast",
@@ -3388,9 +3400,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_serialize"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9a6824a462c4c1a379e911b0faf86d303a54bcf8673d4cc445195085966a4a4"
+checksum = "5f741f8e9aee6323fbe127329490608a5a250cc0072ac91e684ef62518cdb1ff"
 dependencies = [
  "indexmap",
  "smallvec",
@@ -3398,9 +3410,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_session"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a782a5f6ada0dbe089c6416ad0104f0b8a8bdb4bd26ea95e5fefaec67aed5e8a"
+checksum = "dba61eca749f4fced4427ad1cc7f23342cfc6527c3bcc624e3aa56abc1f81298"
 dependencies = [
  "bitflags",
  "getopts",
@@ -3420,9 +3432,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_span"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a257546cb264b250c7abdb81239bb02f18a274a966211755a3ea89411b122214"
+checksum = "a642e8d6fc883f34e0778e079f8242ac40c6614a6b7a0ef61681333e847f5e62"
 dependencies = [
  "cfg-if 0.1.10",
  "md-5",
@@ -3440,9 +3452,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_target"
-version = "718.0.0"
+version = "722.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5a72dd689421bcb5750f3ed0dedf367076e714ef0ba56c02ed391b9a8582862"
+checksum = "80feebd8c323b80dd73a395fa7fabba9e2098b6277670ff89c473f618ffa07de"
 dependencies = [
  "bitflags",
  "rustc-ap-rustc_data_structures",
@@ -3545,7 +3557,7 @@ dependencies = [
  "serde_json",
  "smallvec",
  "syn",
- "url 2.1.1",
+ "url 2.2.2",
  "winapi 0.3.9",
 ]
 
@@ -3681,6 +3693,7 @@ dependencies = [
  "rustc_incremental",
  "rustc_index",
  "rustc_llvm",
+ "rustc_metadata",
  "rustc_middle",
  "rustc_serialize",
  "rustc_session",
@@ -3700,6 +3713,7 @@ dependencies = [
  "itertools 0.9.0",
  "jobserver",
  "libc",
+ "object 0.25.2",
  "pathdiff",
  "rustc_apfloat",
  "rustc_ast",
@@ -3880,6 +3894,7 @@ dependencies = [
  "rand 0.7.3",
  "rustc_ast",
  "rustc_data_structures",
+ "rustc_errors",
  "rustc_fs_util",
  "rustc_graphviz",
  "rustc_hir",
@@ -4676,6 +4691,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "semver-parser"
 version = "0.7.0"
@@ -4796,12 +4820,6 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
 
-[[package]]
-name = "shlex"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
-
 [[package]]
 name = "shlex"
 version = "1.0.0"
@@ -4853,13 +4871,22 @@ checksum = "da73c8f77aebc0e40c300b93f0a5f1bece7a248a36eee287d4e095f35c7b7d6e"
 
 [[package]]
 name = "socket2"
-version = "0.3.16"
+version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d"
+checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "socket2"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
 dependencies = [
- "cfg-if 0.1.10",
  "libc",
- "redox_syscall 0.1.57",
  "winapi 0.3.9",
 ]
 
@@ -4897,7 +4924,7 @@ dependencies = [
  "hermit-abi",
  "libc",
  "miniz_oxide",
- "object",
+ "object 0.22.0",
  "panic_abort",
  "panic_unwind",
  "profiler_builtins",
@@ -5026,13 +5053,12 @@ dependencies = [
 
 [[package]]
 name = "tar"
-version = "0.4.29"
+version = "0.4.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8a4c1d0bee3230179544336c15eefb563cf0302955d962e456542323e8c2e8a"
+checksum = "7d779dc6aeff029314570f666ec83f19df7280bb36ef338442cfa8c604021b80"
 dependencies = [
  "filetime",
  "libc",
- "redox_syscall 0.1.57",
  "xattr",
 ]
 
@@ -5045,7 +5071,7 @@ dependencies = [
  "cfg-if 1.0.0",
  "libc",
  "rand 0.8.3",
- "redox_syscall 0.2.5",
+ "redox_syscall",
  "remove_dir_all",
  "winapi 0.3.9",
 ]
@@ -5522,10 +5548,11 @@ dependencies = [
 
 [[package]]
 name = "url"
-version = "2.1.1"
+version = "2.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb"
+checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
 dependencies = [
+ "form_urlencoded",
  "idna 0.2.0",
  "matches",
  "percent-encoding 2.1.0",
@@ -5573,9 +5600,9 @@ dependencies = [
 
 [[package]]
 name = "version_check"
-version = "0.9.1"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
+checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
 
 [[package]]
 name = "vte"

README.md

@@ -19,8 +19,28 @@ Read ["Installation"] from [The Book].
 ## Installing from Source
 
 The Rust build system uses a Python script called `x.py` to build the compiler,
-which manages the bootstrapping process. More information about it can be found
+which manages the bootstrapping process. It lives in the root of the project.
-by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild].
+
+The `x.py` command can be run directly on most systems in the following format: 
+
+```sh
+./x.py <subcommand> [flags]
+```
+
+This is how the documentation and examples assume you are running `x.py`.
+
+Systems such as Ubuntu 20.04 LTS do not create the necessary `python` command by default when Python is installed that allows `x.py` to be run directly. In that case you can either create a symlink for `python` (Ubuntu provides the `python-is-python3` package for this), or run `x.py` using Python itself:
+
+```sh
+# Python 3
+python3 x.py <subcommand> [flags]
+
+# Python 2.7
+python2.7 x.py <subcommand> [flags]
+```
+
+More information about `x.py` can be found
+by running it with the `--help` flag or reading the [rustc dev guide][rustcguidebuild].
 
 [gettingstarted]: https://rustc-dev-guide.rust-lang.org/getting-started.html
 [rustcguidebuild]: https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html

RELEASES.md

@@ -1,3 +1,127 @@
+Version 1.54.0 (2021-07-29)
+============================
+
+Language
+-----------------------
+
+- [You can now use macros for values in built-in attribute macros.][83366]
+  While a seemingly minor addition on its own, this enables a lot of
+  powerful functionality when combined correctly. Most notably you can
+  now include external documentation in your crate by writing the following.
+  ```rust
+  #![doc = include_str!("README.md")]
+  ```
+  You can also use this to include auto-generated modules:
+  ```rust
+  #[path = concat!(env!("OUT_DIR"), "/generated.rs")]
+  mod generated;
+  ```
+
+- [You can now cast between unsized slice types (and types which contain
+  unsized slices) in `const fn`.][85078]
+- [You can now use multiple generic lifetimes with `impl Trait` where the
+   lifetimes don't explicitly outlive another.][84701] In code this means
+   that you can now have `impl Trait<'a, 'b>` where as before you could
+   only have `impl Trait<'a, 'b> where 'b: 'a`.
+
+Compiler
+-----------------------
+
+- [Rustc will now search for custom JSON targets in
+  `/lib/rustlib/<target-triple>/target.json` where `/` is the "sysroot"
+  directory.][83800] You can find your sysroot directory by running
+  `rustc --print sysroot`.
+- [Added `wasm` as a `target_family` for WebAssembly platforms.][84072]
+- [You can now use `#[target_feature]` on safe functions when targeting
+  WebAssembly platforms.][84988]
+- [Improved debugger output for enums on Windows MSVC platforms.][85292]
+- [Added tier 3\* support for `bpfel-unknown-none`
+   and `bpfeb-unknown-none`.][79608]
+
+\* Refer to Rust's [platform support page][platform-support-doc] for more
+   information on Rust's tiered platform support.
+
+Libraries
+-----------------------
+
+- [`panic::panic_any` will now `#[track_caller]`.][85745]
+- [Added `OutOfMemory` as a variant of `io::ErrorKind`.][84744]
+- [ `proc_macro::Literal` now implements `FromStr`.][84717]
+- [The implementations of vendor intrinsics in core::arch have been
+   significantly refactored.][83278] The main user-visible changes are
+   a 50% reduction in the size of libcore.rlib and stricter validation
+   of constant operands passed to intrinsics. The latter is technically
+   a breaking change, but allows Rust to more closely match the C vendor
+   intrinsics API.
+
+Stabilized APIs
+---------------
+
+- [`BTreeMap::into_keys`]
+- [`BTreeMap::into_values`]
+- [`HashMap::into_keys`]
+- [`HashMap::into_values`]
+- [`arch::wasm32`]
+- [`VecDeque::binary_search`]
+- [`VecDeque::binary_search_by`]
+- [`VecDeque::binary_search_by_key`]
+- [`VecDeque::partition_point`]
+
+Cargo
+-----
+
+- [Added the `--prune <spec>` option to `cargo-tree` to remove a package from
+  the dependency graph.][cargo/9520]
+- [Added the `--depth` option to `cargo-tree` to print only to a certain depth
+  in the tree ][cargo/9499]
+- [Added the `no-proc-macro` value to `cargo-tree --edges` to hide procedural
+  macro dependencies.][cargo/9488]
+- [A new environment variable named `CARGO_TARGET_TMPDIR` is available.][cargo/9375]
+  This variable points to a directory that integration tests and benches
+  can use as a "scratchpad" for testing filesystem operations.
+
+Compatibility Notes
+-------------------
+- [Mixing Option and Result via `?` is no longer permitted in closures for inferred types.][86831]
+- [Previously unsound code is no longer permitted where different constructors in branches
+  could require different lifetimes.][85574]
+- As previously mentioned the [`std::arch` instrinsics now uses stricter const checking][83278]
+  than before and may reject some previously accepted code.
+- [`i128` multiplication on Cortex M0+ platforms currently unconditionally causes overflow
+   when compiled with `codegen-units = 1`.][86063]
+
+[85574]: https://github.com/rust-lang/rust/issues/85574
+[86831]: https://github.com/rust-lang/rust/issues/86831
+[86063]: https://github.com/rust-lang/rust/issues/86063
+[86831]: https://github.com/rust-lang/rust/issues/86831
+[79608]: https://github.com/rust-lang/rust/pull/79608
+[84988]: https://github.com/rust-lang/rust/pull/84988
+[84701]: https://github.com/rust-lang/rust/pull/84701
+[84072]: https://github.com/rust-lang/rust/pull/84072
+[85745]: https://github.com/rust-lang/rust/pull/85745
+[84744]: https://github.com/rust-lang/rust/pull/84744
+[85078]: https://github.com/rust-lang/rust/pull/85078
+[84717]: https://github.com/rust-lang/rust/pull/84717
+[83800]: https://github.com/rust-lang/rust/pull/83800
+[83366]: https://github.com/rust-lang/rust/pull/83366
+[83278]: https://github.com/rust-lang/rust/pull/83278
+[85292]: https://github.com/rust-lang/rust/pull/85292
+[cargo/9520]: https://github.com/rust-lang/cargo/pull/9520
+[cargo/9499]: https://github.com/rust-lang/cargo/pull/9499
+[cargo/9488]: https://github.com/rust-lang/cargo/pull/9488
+[cargo/9375]: https://github.com/rust-lang/cargo/pull/9375
+[`BTreeMap::into_keys`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.into_keys
+[`BTreeMap::into_values`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.into_values
+[`HashMap::into_keys`]: https://doc.rust-lang.org/std/collections/struct.HashMap.html#method.into_keys
+[`HashMap::into_values`]: https://doc.rust-lang.org/std/collections/struct.HashMap.html#method.into_values
+[`arch::wasm32`]: https://doc.rust-lang.org/core/arch/wasm32/index.html
+[`VecDeque::binary_search`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.binary_search
+[`VecDeque::binary_search_by`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.binary_search_by
+
+[`VecDeque::binary_search_by_key`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.binary_search_by_key
+
+[`VecDeque::partition_point`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.partition_point
+
 Version 1.53.0 (2021-06-17)
 ============================
 
@@ -81,13 +205,6 @@ Stabilised APIs
 - [`Vec::extend_from_within`]
 - [`array::from_mut`]
 - [`array::from_ref`]
-- [`char::MAX`]
-- [`char::REPLACEMENT_CHARACTER`]
-- [`char::UNICODE_VERSION`]
-- [`char::decode_utf16`]
-- [`char::from_digit`]
-- [`char::from_u32_unchecked`]
-- [`char::from_u32`]
 - [`cmp::max_by_key`]
 - [`cmp::max_by`]
 - [`cmp::min_by_key`]
@@ -116,7 +233,11 @@ Compatibility Notes
   to rejecting hexadecimal IP addresses.][83652] The octal format can lead
   to confusion and potential security vulnerabilities and [is no
   longer recommended][ietf6943].
-
+- [The added `BITS` constant may conflict with external definitions.][85667]
+  In particular, this was known to be a problem in the `lexical-core` crate,
+  but they have published fixes for semantic versions 0.4 through 0.7. To
+  update this dependency alone, use `cargo update -p lexical-core`.
+- Incremental compilation remains off by default, unless one uses the `RUSTC_FORCE_INCREMENTAL=1` environment variable added in 1.52.1.
 
 Internal Only
 -------------
@@ -129,6 +250,7 @@ related tools.
 - [rustdoc: Only look at blanket impls in `get_blanket_impls`][83681]
 - [Rework rustdoc const type][82873]
 
+[85667]: https://github.com/rust-lang/rust/pull/85667
 [83386]: https://github.com/rust-lang/rust/pull/83386
 [82771]: https://github.com/rust-lang/rust/pull/82771
 [84147]: https://github.com/rust-lang/rust/pull/84147
@@ -152,13 +274,6 @@ related tools.
 [cargo/9298]: https://github.com/rust-lang/cargo/pull/9298
 [cargo/9282]: https://github.com/rust-lang/cargo/pull/9282
 [cargo/9392]: https://github.com/rust-lang/cargo/pull/9392
-[`char::MAX`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.MAX
-[`char::REPLACEMENT_CHARACTER`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.REPLACEMENT_CHARACTER
-[`char::UNICODE_VERSION`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.UNICODE_VERSION
-[`char::decode_utf16`]: https://doc.rust-lang.org/std/primitive.char.html#method.decode_utf16
-[`char::from_u32`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_u32
-[`char::from_u32_unchecked`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_u32_unchecked
-[`char::from_digit`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_digit
 [`AtomicBool::fetch_update`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicBool.html#method.fetch_update
 [`AtomicPtr::fetch_update`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicPtr.html#method.fetch_update
 [`BTreeMap::retain`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.retain
@@ -320,6 +435,7 @@ Compatibility Notes
 - [Rustc now catches more cases of `pub_use_of_private_extern_crate`][80763]
 - [Changes in how proc macros handle whitespace may lead to panics when used
   with older `proc-macro-hack` versions. A `cargo update` should be sufficient to fix this in all cases.][84136]
+- [Turn `#[derive]` into a regular macro attribute][79078]
 
 [84136]: https://github.com/rust-lang/rust/issues/84136
 [80763]: https://github.com/rust-lang/rust/pull/80763
@@ -346,6 +462,7 @@ Compatibility Notes
 [78429]: https://github.com/rust-lang/rust/pull/78429
 [82733]: https://github.com/rust-lang/rust/pull/82733
 [82594]: https://github.com/rust-lang/rust/pull/82594
+[79078]: https://github.com/rust-lang/rust/pull/79078
 [cargo/9181]: https://github.com/rust-lang/cargo/pull/9181
 [`char::MAX`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.MAX
 [`char::REPLACEMENT_CHARACTER`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.REPLACEMENT_CHARACTER
@@ -1756,6 +1873,7 @@ Language
 - [You can now use `#[repr(transparent)]` on univariant `enum`s.][68122] Meaning
   that you can create an enum that has the exact layout and ABI of the type
   it contains.
+- [You can now use outer attribute procedural macros on inline modules.][64273]
 - [There are some *syntax-only* changes:][67131]
    - `default` is syntactically allowed before items in `trait` definitions.
    - Items in `impl`s (i.e. `const`s, `type`s, and `fn`s) may syntactically
@@ -1817,6 +1935,7 @@ Compatibility Notes
 [67935]: https://github.com/rust-lang/rust/pull/67935/
 [68339]: https://github.com/rust-lang/rust/pull/68339/
 [68122]: https://github.com/rust-lang/rust/pull/68122/
+[64273]: https://github.com/rust-lang/rust/pull/64273/
 [67712]: https://github.com/rust-lang/rust/pull/67712/
 [67887]: https://github.com/rust-lang/rust/pull/67887/
 [67131]: https://github.com/rust-lang/rust/pull/67131/

compiler/rustc_apfloat/src/lib.rs

@@ -35,7 +35,6 @@
 #![forbid(unsafe_code)]
 #![feature(iter_zip)]
 #![feature(nll)]
-#![cfg_attr(bootstrap, feature(or_patterns))]
 
 #[macro_use]
 extern crate alloc;

compiler/rustc_ast/src/ast.rs

@@ -278,7 +278,7 @@ impl ParenthesizedArgs {
             .cloned()
             .map(|input| AngleBracketedArg::Arg(GenericArg::Type(input)))
             .collect();
-        AngleBracketedArgs { span: self.span, args }
+        AngleBracketedArgs { span: self.inputs_span, args }
     }
 }
 
@@ -623,12 +623,13 @@ impl Pat {
             PatKind::Ident(_, _, Some(p)) => p.walk(it),
 
             // Walk into each field of struct.
-            PatKind::Struct(_, fields, _) => fields.iter().for_each(|field| field.pat.walk(it)),
+            PatKind::Struct(_, _, fields, _) => fields.iter().for_each(|field| field.pat.walk(it)),
 
             // Sequence of patterns.
-            PatKind::TupleStruct(_, s) | PatKind::Tuple(s) | PatKind::Slice(s) | PatKind::Or(s) => {
+            PatKind::TupleStruct(_, _, s)
-                s.iter().for_each(|p| p.walk(it))
+            | PatKind::Tuple(s)
-            }
+            | PatKind::Slice(s)
+            | PatKind::Or(s) => s.iter().for_each(|p| p.walk(it)),
 
             // Trivial wrappers over inner patterns.
             PatKind::Box(s) | PatKind::Ref(s, _) | PatKind::Paren(s) => s.walk(it),
@@ -701,10 +702,10 @@ pub enum PatKind {
 
     /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`).
     /// The `bool` is `true` in the presence of a `..`.
-    Struct(Path, Vec<PatField>, /* recovered */ bool),
+    Struct(Option<QSelf>, Path, Vec<PatField>, /* recovered */ bool),
 
     /// A tuple struct/variant pattern (`Variant(x, y, .., z)`).
-    TupleStruct(Path, Vec<P<Pat>>),
+    TupleStruct(Option<QSelf>, Path, Vec<P<Pat>>),
 
     /// An or-pattern `A | B | C`.
     /// Invariant: `pats.len() >= 2`.
@@ -1247,6 +1248,7 @@ pub enum StructRest {
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct StructExpr {
+    pub qself: Option<QSelf>,
     pub path: Path,
     pub fields: Vec<ExprField>,
     pub rest: StructRest,
@@ -1861,6 +1863,10 @@ pub enum TyKind {
     Never,
     /// A tuple (`(A, B, C, D,...)`).
     Tup(Vec<P<Ty>>),
+    /// An anonymous struct type i.e. `struct { foo: Type }`
+    AnonymousStruct(Vec<FieldDef>, bool),
+    /// An anonymous union type i.e. `union { bar: Type }`
+    AnonymousUnion(Vec<FieldDef>, bool),
     /// A path (`module::module::...::Type`), optionally
     /// "qualified", e.g., `<Vec<T> as SomeTrait>::SomeType`.
     ///
@@ -2279,14 +2285,6 @@ pub struct ForeignMod {
     pub items: Vec<P<ForeignItem>>,
 }
 
-/// Global inline assembly.
-///
-/// Also known as "module-level assembly" or "file-scoped assembly".
-#[derive(Clone, Encodable, Decodable, Debug, Copy)]
-pub struct GlobalAsm {
-    pub asm: Symbol,
-}
-
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct EnumDef {
     pub variants: Vec<Variant>,
@@ -2669,7 +2667,7 @@ pub enum ItemKind {
     /// E.g., `extern {}` or `extern "C" {}`.
     ForeignMod(ForeignMod),
     /// Module-level inline assembly (from `global_asm!()`).
-    GlobalAsm(GlobalAsm),
+    GlobalAsm(InlineAsm),
     /// A type alias (`type`).
     ///
     /// E.g., `type Foo = Bar<u8>;`.

compiler/rustc_ast/src/ast_like.rs

@@ -82,7 +82,8 @@ impl AstLike for crate::token::Nonterminal {
             Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
             Nonterminal::NtPath(path) => path.tokens_mut(),
             Nonterminal::NtVis(vis) => vis.tokens_mut(),
-            _ => panic!("Called tokens_mut on {:?}", self),
+            Nonterminal::NtBlock(block) => block.tokens_mut(),
+            Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None,
         }
     }
 }

compiler/rustc_ast/src/lib.rs

@@ -10,15 +10,13 @@
 )]
 #![feature(box_syntax)]
 #![feature(box_patterns)]
-#![cfg_attr(bootstrap, feature(const_fn))] // For the `transmute` in `P::new`
+#![cfg_attr(bootstrap, feature(const_fn_unsize))]
-#![cfg_attr(not(bootstrap), feature(const_fn_unsize))] // For the `transmute` in `P::new`
 #![feature(const_fn_transmute)]
-#![feature(const_panic)]
 #![feature(crate_visibility_modifier)]
 #![feature(iter_zip)]
 #![feature(label_break_value)]
 #![feature(nll)]
-#![cfg_attr(bootstrap, feature(or_patterns))]
+#![feature(min_specialization)]
 #![recursion_limit = "256"]
 
 #[macro_use]

compiler/rustc_ast/src/mut_visit.rs

@@ -484,6 +484,9 @@ pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
             visit_vec(bounds, |bound| vis.visit_param_bound(bound));
         }
         TyKind::MacCall(mac) => vis.visit_mac_call(mac),
+        TyKind::AnonymousStruct(fields, ..) | TyKind::AnonymousUnion(fields, ..) => {
+            fields.flat_map_in_place(|field| vis.flat_map_field_def(field));
+        }
     }
     vis.visit_span(span);
     visit_lazy_tts(tokens, vis);
@@ -965,7 +968,7 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) {
             ModKind::Unloaded => {}
         },
         ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
-        ItemKind::GlobalAsm(_ga) => {}
+        ItemKind::GlobalAsm(asm) => noop_visit_inline_asm(asm, vis),
         ItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => {
             vis.visit_generics(generics);
             visit_bounds(bounds, vis);
@@ -1136,7 +1139,8 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
             visit_opt(sub, |sub| vis.visit_pat(sub));
         }
         PatKind::Lit(e) => vis.visit_expr(e),
-        PatKind::TupleStruct(path, elems) => {
+        PatKind::TupleStruct(qself, path, elems) => {
+            vis.visit_qself(qself);
             vis.visit_path(path);
             visit_vec(elems, |elem| vis.visit_pat(elem));
         }
@@ -1144,7 +1148,8 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
             vis.visit_qself(qself);
             vis.visit_path(path);
         }
-        PatKind::Struct(path, fields, _etc) => {
+        PatKind::Struct(qself, path, fields, _etc) => {
+            vis.visit_qself(qself);
             vis.visit_path(path);
             fields.flat_map_in_place(|field| vis.flat_map_pat_field(field));
         }
@@ -1170,6 +1175,28 @@ pub fn noop_visit_anon_const<T: MutVisitor>(AnonConst { id, value }: &mut AnonCo
     vis.visit_expr(value);
 }
 
+fn noop_visit_inline_asm<T: MutVisitor>(asm: &mut InlineAsm, vis: &mut T) {
+    for (op, _) in &mut asm.operands {
+        match op {
+            InlineAsmOperand::In { expr, .. }
+            | InlineAsmOperand::InOut { expr, .. }
+            | InlineAsmOperand::Sym { expr, .. } => vis.visit_expr(expr),
+            InlineAsmOperand::Out { expr, .. } => {
+                if let Some(expr) = expr {
+                    vis.visit_expr(expr);
+                }
+            }
+            InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => {
+                vis.visit_expr(in_expr);
+                if let Some(out_expr) = out_expr {
+                    vis.visit_expr(out_expr);
+                }
+            }
+            InlineAsmOperand::Const { anon_const, .. } => vis.visit_anon_const(anon_const),
+        }
+    }
+}
+
 pub fn noop_visit_expr<T: MutVisitor>(
     Expr { kind, id, span, attrs, tokens }: &mut Expr,
     vis: &mut T,
@@ -1288,27 +1315,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
         ExprKind::Ret(expr) => {
             visit_opt(expr, |expr| vis.visit_expr(expr));
         }
-        ExprKind::InlineAsm(asm) => {
+        ExprKind::InlineAsm(asm) => noop_visit_inline_asm(asm, vis),
-            for (op, _) in &mut asm.operands {
-                match op {
-                    InlineAsmOperand::In { expr, .. }
-                    | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Sym { expr, .. } => vis.visit_expr(expr),
-                    InlineAsmOperand::Out { expr, .. } => {
-                        if let Some(expr) = expr {
-                            vis.visit_expr(expr);
-                        }
-                    }
-                    InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => {
-                        vis.visit_expr(in_expr);
-                        if let Some(out_expr) = out_expr {
-                            vis.visit_expr(out_expr);
-                        }
-                    }
-                    InlineAsmOperand::Const { anon_const, .. } => vis.visit_anon_const(anon_const),
-                }
-            }
-        }
         ExprKind::LlvmInlineAsm(asm) => {
             let LlvmInlineAsm {
                 asm: _,
@@ -1328,7 +1335,8 @@ pub fn noop_visit_expr<T: MutVisitor>(
         }
         ExprKind::MacCall(mac) => vis.visit_mac_call(mac),
         ExprKind::Struct(se) => {
-            let StructExpr { path, fields, rest } = se.deref_mut();
+            let StructExpr { qself, path, fields, rest } = se.deref_mut();
+            vis.visit_qself(qself);
             vis.visit_path(path);
             fields.flat_map_in_place(|field| vis.flat_map_expr_field(field));
             match rest {

compiler/rustc_ast/src/tokenstream.rs

@@ -218,8 +218,7 @@ impl AttrAnnotatedTokenStream {
                 AttrAnnotatedTokenTree::Attributes(data) => {
                     let mut outer_attrs = Vec::new();
                     let mut inner_attrs = Vec::new();
-                    let attrs: Vec<_> = data.attrs.clone().into();
+                    for attr in &data.attrs {
-                    for attr in attrs {
                         match attr.style {
                             crate::AttrStyle::Outer => {
                                 assert!(
@@ -264,7 +263,7 @@ impl AttrAnnotatedTokenStream {
                                 // so we never reach this code.
 
                                 let mut builder = TokenStreamBuilder::new();
-                                for inner_attr in &inner_attrs {
+                                for inner_attr in inner_attrs {
                                     builder.push(inner_attr.tokens().to_tokenstream());
                                 }
                                 builder.push(delim_tokens.clone());

compiler/rustc_ast/src/visit.rs

@@ -90,9 +90,6 @@ pub trait Visitor<'ast>: Sized {
     fn visit_foreign_item(&mut self, i: &'ast ForeignItem) {
         walk_foreign_item(self, i)
     }
-    fn visit_global_asm(&mut self, ga: &'ast GlobalAsm) {
-        walk_global_asm(self, ga)
-    }
     fn visit_item(&mut self, i: &'ast Item) {
         walk_item(self, i)
     }
@@ -299,7 +296,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) {
         ItemKind::ForeignMod(ref foreign_module) => {
             walk_list!(visitor, visit_foreign_item, &foreign_module.items);
         }
-        ItemKind::GlobalAsm(ref ga) => visitor.visit_global_asm(ga),
+        ItemKind::GlobalAsm(ref asm) => walk_inline_asm(visitor, asm),
         ItemKind::TyAlias(box TyAliasKind(_, ref generics, ref bounds, ref ty)) => {
             visitor.visit_generics(generics);
             walk_list!(visitor, visit_param_bound, bounds);
@@ -407,6 +404,9 @@ pub fn walk_ty<'a, V: Visitor<'a>>(visitor: &mut V, typ: &'a Ty) {
         TyKind::Typeof(ref expression) => visitor.visit_anon_const(expression),
         TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err => {}
         TyKind::MacCall(ref mac) => visitor.visit_mac_call(mac),
+        TyKind::AnonymousStruct(ref fields, ..) | TyKind::AnonymousUnion(ref fields, ..) => {
+            walk_list!(visitor, visit_field_def, fields)
+        }
         TyKind::Never | TyKind::CVarArgs => {}
     }
 }
@@ -497,7 +497,10 @@ pub fn walk_assoc_ty_constraint<'a, V: Visitor<'a>>(
 
 pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) {
     match pattern.kind {
-        PatKind::TupleStruct(ref path, ref elems) => {
+        PatKind::TupleStruct(ref opt_qself, ref path, ref elems) => {
+            if let Some(ref qself) = *opt_qself {
+                visitor.visit_ty(&qself.ty);
+            }
             visitor.visit_path(path, pattern.id);
             walk_list!(visitor, visit_pat, elems);
         }
@@ -507,7 +510,10 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) {
             }
             visitor.visit_path(path, pattern.id)
         }
-        PatKind::Struct(ref path, ref fields, _) => {
+        PatKind::Struct(ref opt_qself, ref path, ref fields, _) => {
+            if let Some(ref qself) = *opt_qself {
+                visitor.visit_ty(&qself.ty);
+            }
             visitor.visit_path(path, pattern.id);
             walk_list!(visitor, visit_pat_field, fields);
         }
@@ -557,10 +563,6 @@ pub fn walk_foreign_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a ForeignI
     }
 }
 
-pub fn walk_global_asm<'a, V: Visitor<'a>>(_: &mut V, _: &'a GlobalAsm) {
-    // Empty!
-}
-
 pub fn walk_param_bound<'a, V: Visitor<'a>>(visitor: &mut V, bound: &'a GenericBound) {
     match *bound {
         GenericBound::Trait(ref typ, ref modifier) => visitor.visit_poly_trait_ref(typ, modifier),
@@ -708,6 +710,28 @@ pub fn walk_anon_const<'a, V: Visitor<'a>>(visitor: &mut V, constant: &'a AnonCo
     visitor.visit_expr(&constant.value);
 }
 
+fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm) {
+    for (op, _) in &asm.operands {
+        match op {
+            InlineAsmOperand::In { expr, .. }
+            | InlineAsmOperand::InOut { expr, .. }
+            | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr),
+            InlineAsmOperand::Out { expr, .. } => {
+                if let Some(expr) = expr {
+                    visitor.visit_expr(expr);
+                }
+            }
+            InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => {
+                visitor.visit_expr(in_expr);
+                if let Some(out_expr) = out_expr {
+                    visitor.visit_expr(out_expr);
+                }
+            }
+            InlineAsmOperand::Const { anon_const, .. } => visitor.visit_anon_const(anon_const),
+        }
+    }
+}
+
 pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
     walk_list!(visitor, visit_attribute, expression.attrs.iter());
 
@@ -722,6 +746,9 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
             visitor.visit_anon_const(count)
         }
         ExprKind::Struct(ref se) => {
+            if let Some(ref qself) = se.qself {
+                visitor.visit_ty(&qself.ty);
+            }
             visitor.visit_path(&se.path, expression.id);
             walk_list!(visitor, visit_expr_field, &se.fields);
             match &se.rest {
@@ -830,29 +857,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
         }
         ExprKind::MacCall(ref mac) => visitor.visit_mac_call(mac),
         ExprKind::Paren(ref subexpression) => visitor.visit_expr(subexpression),
-        ExprKind::InlineAsm(ref ia) => {
+        ExprKind::InlineAsm(ref asm) => walk_inline_asm(visitor, asm),
-            for (op, _) in &ia.operands {
-                match op {
-                    InlineAsmOperand::In { expr, .. }
-                    | InlineAsmOperand::InOut { expr, .. }
-                    | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr),
-                    InlineAsmOperand::Out { expr, .. } => {
-                        if let Some(expr) = expr {
-                            visitor.visit_expr(expr);
-                        }
-                    }
-                    InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => {
-                        visitor.visit_expr(in_expr);
-                        if let Some(out_expr) = out_expr {
-                            visitor.visit_expr(out_expr);
-                        }
-                    }
-                    InlineAsmOperand::Const { anon_const, .. } => {
-                        visitor.visit_anon_const(anon_const)
-                    }
-                }
-            }
-        }
         ExprKind::LlvmInlineAsm(ref ia) => {
             for &(_, ref input) in &ia.inputs {
                 visitor.visit_expr(input)

compiler/rustc_ast_lowering/src/asm.rs

@@ -0,0 +1,329 @@
+use super::LoweringContext;
+
+use rustc_ast::*;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_errors::struct_span_err;
+use rustc_hir as hir;
+use rustc_span::{Span, Symbol};
+use rustc_target::asm;
+use std::collections::hash_map::Entry;
+use std::fmt::Write;
+
+impl<'a, 'hir> LoweringContext<'a, 'hir> {
+    crate fn lower_inline_asm(&mut self, sp: Span, asm: &InlineAsm) -> &'hir hir::InlineAsm<'hir> {
+        // Rustdoc needs to support asm! from foriegn architectures: don't try
+        // lowering the register contraints in this case.
+        let asm_arch = if self.sess.opts.actually_rustdoc { None } else { self.sess.asm_arch };
+        if asm_arch.is_none() && !self.sess.opts.actually_rustdoc {
+            struct_span_err!(self.sess, sp, E0472, "inline assembly is unsupported on this target")
+                .emit();
+        }
+        if asm.options.contains(InlineAsmOptions::ATT_SYNTAX)
+            && !matches!(asm_arch, Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64))
+            && !self.sess.opts.actually_rustdoc
+        {
+            self.sess
+                .struct_span_err(sp, "the `att_syntax` option is only supported on x86")
+                .emit();
+        }
+
+        // Lower operands to HIR. We use dummy register classes if an error
+        // occurs during lowering because we still need to be able to produce a
+        // valid HIR.
+        let sess = self.sess;
+        let operands: Vec<_> = asm
+            .operands
+            .iter()
+            .map(|(op, op_sp)| {
+                let lower_reg = |reg| match reg {
+                    InlineAsmRegOrRegClass::Reg(s) => {
+                        asm::InlineAsmRegOrRegClass::Reg(if let Some(asm_arch) = asm_arch {
+                            asm::InlineAsmReg::parse(
+                                asm_arch,
+                                |feature| sess.target_features.contains(&Symbol::intern(feature)),
+                                &sess.target,
+                                s,
+                            )
+                            .unwrap_or_else(|e| {
+                                let msg = format!("invalid register `{}`: {}", s.as_str(), e);
+                                sess.struct_span_err(*op_sp, &msg).emit();
+                                asm::InlineAsmReg::Err
+                            })
+                        } else {
+                            asm::InlineAsmReg::Err
+                        })
+                    }
+                    InlineAsmRegOrRegClass::RegClass(s) => {
+                        asm::InlineAsmRegOrRegClass::RegClass(if let Some(asm_arch) = asm_arch {
+                            asm::InlineAsmRegClass::parse(asm_arch, s).unwrap_or_else(|e| {
+                                let msg = format!("invalid register class `{}`: {}", s.as_str(), e);
+                                sess.struct_span_err(*op_sp, &msg).emit();
+                                asm::InlineAsmRegClass::Err
+                            })
+                        } else {
+                            asm::InlineAsmRegClass::Err
+                        })
+                    }
+                };
+
+                let op = match *op {
+                    InlineAsmOperand::In { reg, ref expr } => hir::InlineAsmOperand::In {
+                        reg: lower_reg(reg),
+                        expr: self.lower_expr_mut(expr),
+                    },
+                    InlineAsmOperand::Out { reg, late, ref expr } => hir::InlineAsmOperand::Out {
+                        reg: lower_reg(reg),
+                        late,
+                        expr: expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
+                    },
+                    InlineAsmOperand::InOut { reg, late, ref expr } => {
+                        hir::InlineAsmOperand::InOut {
+                            reg: lower_reg(reg),
+                            late,
+                            expr: self.lower_expr_mut(expr),
+                        }
+                    }
+                    InlineAsmOperand::SplitInOut { reg, late, ref in_expr, ref out_expr } => {
+                        hir::InlineAsmOperand::SplitInOut {
+                            reg: lower_reg(reg),
+                            late,
+                            in_expr: self.lower_expr_mut(in_expr),
+                            out_expr: out_expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
+                        }
+                    }
+                    InlineAsmOperand::Const { ref anon_const } => hir::InlineAsmOperand::Const {
+                        anon_const: self.lower_anon_const(anon_const),
+                    },
+                    InlineAsmOperand::Sym { ref expr } => {
+                        hir::InlineAsmOperand::Sym { expr: self.lower_expr_mut(expr) }
+                    }
+                };
+                (op, *op_sp)
+            })
+            .collect();
+
+        // Validate template modifiers against the register classes for the operands
+        for p in &asm.template {
+            if let InlineAsmTemplatePiece::Placeholder {
+                operand_idx,
+                modifier: Some(modifier),
+                span: placeholder_span,
+            } = *p
+            {
+                let op_sp = asm.operands[operand_idx].1;
+                match &operands[operand_idx].0 {
+                    hir::InlineAsmOperand::In { reg, .. }
+                    | hir::InlineAsmOperand::Out { reg, .. }
+                    | hir::InlineAsmOperand::InOut { reg, .. }
+                    | hir::InlineAsmOperand::SplitInOut { reg, .. } => {
+                        let class = reg.reg_class();
+                        if class == asm::InlineAsmRegClass::Err {
+                            continue;
+                        }
+                        let valid_modifiers = class.valid_modifiers(asm_arch.unwrap());
+                        if !valid_modifiers.contains(&modifier) {
+                            let mut err = sess.struct_span_err(
+                                placeholder_span,
+                                "invalid asm template modifier for this register class",
+                            );
+                            err.span_label(placeholder_span, "template modifier");
+                            err.span_label(op_sp, "argument");
+                            if !valid_modifiers.is_empty() {
+                                let mut mods = format!("`{}`", valid_modifiers[0]);
+                                for m in &valid_modifiers[1..] {
+                                    let _ = write!(mods, ", `{}`", m);
+                                }
+                                err.note(&format!(
+                                    "the `{}` register class supports \
+                                     the following template modifiers: {}",
+                                    class.name(),
+                                    mods
+                                ));
+                            } else {
+                                err.note(&format!(
+                                    "the `{}` register class does not support template modifiers",
+                                    class.name()
+                                ));
+                            }
+                            err.emit();
+                        }
+                    }
+                    hir::InlineAsmOperand::Const { .. } => {
+                        let mut err = sess.struct_span_err(
+                            placeholder_span,
+                            "asm template modifiers are not allowed for `const` arguments",
+                        );
+                        err.span_label(placeholder_span, "template modifier");
+                        err.span_label(op_sp, "argument");
+                        err.emit();
+                    }
+                    hir::InlineAsmOperand::Sym { .. } => {
+                        let mut err = sess.struct_span_err(
+                            placeholder_span,
+                            "asm template modifiers are not allowed for `sym` arguments",
+                        );
+                        err.span_label(placeholder_span, "template modifier");
+                        err.span_label(op_sp, "argument");
+                        err.emit();
+                    }
+                }
+            }
+        }
+
+        let mut used_input_regs = FxHashMap::default();
+        let mut used_output_regs = FxHashMap::default();
+        let mut required_features: Vec<&str> = vec![];
+        for (idx, &(ref op, op_sp)) in operands.iter().enumerate() {
+            if let Some(reg) = op.reg() {
+                // Make sure we don't accidentally carry features from the
+                // previous iteration.
+                required_features.clear();
+
+                let reg_class = reg.reg_class();
+                if reg_class == asm::InlineAsmRegClass::Err {
+                    continue;
+                }
+
+                // We ignore target feature requirements for clobbers: if the
+                // feature is disabled then the compiler doesn't care what we
+                // do with the registers.
+                //
+                // Note that this is only possible for explicit register
+                // operands, which cannot be used in the asm string.
+                let is_clobber = matches!(
+                    op,
+                    hir::InlineAsmOperand::Out {
+                        reg: asm::InlineAsmRegOrRegClass::Reg(_),
+                        late: _,
+                        expr: None
+                    }
+                );
+
+                if !is_clobber {
+                    // Validate register classes against currently enabled target
+                    // features. We check that at least one type is available for
+                    // the current target.
+                    for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) {
+                        if let Some(feature) = feature {
+                            if self.sess.target_features.contains(&Symbol::intern(feature)) {
+                                required_features.clear();
+                                break;
+                            } else {
+                                required_features.push(feature);
+                            }
+                        } else {
+                            required_features.clear();
+                            break;
+                        }
+                    }
+                    // We are sorting primitive strs here and can use unstable sort here
+                    required_features.sort_unstable();
+                    required_features.dedup();
+                    match &required_features[..] {
+                        [] => {}
+                        [feature] => {
+                            let msg = format!(
+                                "register class `{}` requires the `{}` target feature",
+                                reg_class.name(),
+                                feature
+                            );
+                            sess.struct_span_err(op_sp, &msg).emit();
+                        }
+                        features => {
+                            let msg = format!(
+                                "register class `{}` requires at least one target feature: {}",
+                                reg_class.name(),
+                                features.join(", ")
+                            );
+                            sess.struct_span_err(op_sp, &msg).emit();
+                        }
+                    }
+                }
+
+                // Check for conflicts between explicit register operands.
+                if let asm::InlineAsmRegOrRegClass::Reg(reg) = reg {
+                    let (input, output) = match op {
+                        hir::InlineAsmOperand::In { .. } => (true, false),
+
+                        // Late output do not conflict with inputs, but normal outputs do
+                        hir::InlineAsmOperand::Out { late, .. } => (!late, true),
+
+                        hir::InlineAsmOperand::InOut { .. }
+                        | hir::InlineAsmOperand::SplitInOut { .. } => (true, true),
+
+                        hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::Sym { .. } => {
+                            unreachable!()
+                        }
+                    };
+
+                    // Flag to output the error only once per operand
+                    let mut skip = false;
+                    reg.overlapping_regs(|r| {
+                        let mut check = |used_regs: &mut FxHashMap<asm::InlineAsmReg, usize>,
+                                         input| {
+                            match used_regs.entry(r) {
+                                Entry::Occupied(o) => {
+                                    if skip {
+                                        return;
+                                    }
+                                    skip = true;
+
+                                    let idx2 = *o.get();
+                                    let &(ref op2, op_sp2) = &operands[idx2];
+                                    let reg2 = match op2.reg() {
+                                        Some(asm::InlineAsmRegOrRegClass::Reg(r)) => r,
+                                        _ => unreachable!(),
+                                    };
+
+                                    let msg = format!(
+                                        "register `{}` conflicts with register `{}`",
+                                        reg.name(),
+                                        reg2.name()
+                                    );
+                                    let mut err = sess.struct_span_err(op_sp, &msg);
+                                    err.span_label(op_sp, &format!("register `{}`", reg.name()));
+                                    err.span_label(op_sp2, &format!("register `{}`", reg2.name()));
+
+                                    match (op, op2) {
+                                        (
+                                            hir::InlineAsmOperand::In { .. },
+                                            hir::InlineAsmOperand::Out { late, .. },
+                                        )
+                                        | (
+                                            hir::InlineAsmOperand::Out { late, .. },
+                                            hir::InlineAsmOperand::In { .. },
+                                        ) => {
+                                            assert!(!*late);
+                                            let out_op_sp = if input { op_sp2 } else { op_sp };
+                                            let msg = "use `lateout` instead of \
+                                                       `out` to avoid conflict";
+                                            err.span_help(out_op_sp, msg);
+                                        }
+                                        _ => {}
+                                    }
+
+                                    err.emit();
+                                }
+                                Entry::Vacant(v) => {
+                                    v.insert(idx);
+                                }
+                            }
+                        };
+                        if input {
+                            check(&mut used_input_regs, true);
+                        }
+                        if output {
+                            check(&mut used_output_regs, false);
+                        }
+                    });
+                }
+            }
+        }
+
+        let operands = self.arena.alloc_from_iter(operands);
+        let template = self.arena.alloc_from_iter(asm.template.iter().cloned());
+        let line_spans = self.arena.alloc_slice(&asm.line_spans[..]);
+        let hir_asm = hir::InlineAsm { template, operands, options: asm.options, line_spans };
+        self.arena.alloc(hir_asm)
+    }
+}

compiler/rustc_ast_lowering/src/expr.rs

@@ -3,7 +3,6 @@ use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericAr
 use rustc_ast::attr;
 use rustc_ast::ptr::P as AstP;
 use rustc_ast::*;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::struct_span_err;
@@ -15,9 +14,6 @@ use rustc_span::hygiene::ExpnId;
 use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{hygiene::ForLoopLoc, DUMMY_SP};
-use rustc_target::asm;
-use std::collections::hash_map::Entry;
-use std::fmt::Write;
 
 impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_exprs(&mut self, exprs: &[AstP<Expr>]) -> &'hir [hir::Expr<'hir>] {
@@ -222,7 +218,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     let e = e.as_ref().map(|x| self.lower_expr(x));
                     hir::ExprKind::Ret(e)
                 }
-                ExprKind::InlineAsm(ref asm) => self.lower_expr_asm(e.span, asm),
+                ExprKind::InlineAsm(ref asm) => {
+                    hir::ExprKind::InlineAsm(self.lower_inline_asm(e.span, asm))
+                }
                 ExprKind::LlvmInlineAsm(ref asm) => self.lower_expr_llvm_asm(asm),
                 ExprKind::Struct(ref se) => {
                     let rest = match &se.rest {
@@ -239,7 +237,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     hir::ExprKind::Struct(
                         self.arena.alloc(self.lower_qpath(
                             e.id,
-                            &None,
+                            &se.qself,
                             &se.path,
                             ParamMode::Optional,
                             ImplTraitContext::disallowed(),
@@ -338,7 +336,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let mut generic_args = vec![];
         for (idx, arg) in args.into_iter().enumerate() {
             if legacy_args_idx.contains(&idx) {
-                let parent_def_id = self.current_hir_id_owner.last().unwrap().0;
+                let parent_def_id = self.current_hir_id_owner.0;
                 let node_id = self.resolver.next_node_id();
 
                 // Add a definition for the in-band const def.
@@ -562,8 +560,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         )
     }
 
-    /// Desugar `try { <stmts>; <expr> }` into `{ <stmts>; ::std::ops::Try::from_ok(<expr>) }`,
+    /// Desugar `try { <stmts>; <expr> }` into `{ <stmts>; ::std::ops::Try::from_output(<expr>) }`,
-    /// `try { <stmts>; }` into `{ <stmts>; ::std::ops::Try::from_ok(()) }`
+    /// `try { <stmts>; }` into `{ <stmts>; ::std::ops::Try::from_output(()) }`
     /// and save the block id to use it as a break target for desugaring of the `?` operator.
     fn lower_expr_try_block(&mut self, body: &Block) -> hir::ExprKind<'hir> {
         self.with_catch_scope(body.id, |this| {
@@ -592,9 +590,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let ok_wrapped_span =
                 this.mark_span_with_reason(DesugaringKind::TryBlock, tail_expr.span, None);
 
-            // `::std::ops::Try::from_ok($tail_expr)`
+            // `::std::ops::Try::from_output($tail_expr)`
             block.expr = Some(this.wrap_in_try_constructor(
-                hir::LangItem::TryFromOk,
+                hir::LangItem::TryTraitFromOutput,
                 try_span,
                 tail_expr,
                 ok_wrapped_span,
@@ -1043,10 +1041,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
     /// It is not a complete check, but just tries to reject most paths early
     /// if they are not tuple structs.
     /// Type checking will take care of the full validation later.
-    fn extract_tuple_struct_path<'a>(&mut self, expr: &'a Expr) -> Option<&'a Path> {
+    fn extract_tuple_struct_path<'a>(
-        // For tuple struct destructuring, it must be a non-qualified path (like in patterns).
+        &mut self,
-        if let ExprKind::Path(None, path) = &expr.kind {
+        expr: &'a Expr,
-            // Does the path resolves to something disallowed in a tuple struct/variant pattern?
+    ) -> Option<(&'a Option<QSelf>, &'a Path)> {
+        if let ExprKind::Path(qself, path) = &expr.kind {
+            // Does the path resolve to something disallowed in a tuple struct/variant pattern?
             if let Some(partial_res) = self.resolver.get_partial_res(expr.id) {
                 if partial_res.unresolved_segments() == 0
                     && !partial_res.base_res().expected_in_tuple_struct_pat()
@@ -1054,7 +1054,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     return None;
                 }
             }
-            return Some(path);
+            return Some((qself, path));
         }
         None
     }
@@ -1090,7 +1090,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
             // Tuple structs.
             ExprKind::Call(callee, args) => {
-                if let Some(path) = self.extract_tuple_struct_path(callee) {
+                if let Some((qself, path)) = self.extract_tuple_struct_path(callee) {
                     let (pats, rest) = self.destructure_sequence(
                         args,
                         "tuple struct or variant",
@@ -1099,7 +1099,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     );
                     let qpath = self.lower_qpath(
                         callee.id,
-                        &None,
+                        qself,
                         path,
                         ParamMode::Optional,
                         ImplTraitContext::disallowed(),
@@ -1124,7 +1124,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }));
                 let qpath = self.lower_qpath(
                     lhs.id,
-                    &None,
+                    &se.qself,
                     &se.path,
                     ParamMode::Optional,
                     ImplTraitContext::disallowed(),
@@ -1236,9 +1236,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             (Some(..), Some(..), HalfOpen) => hir::LangItem::Range,
             (None, Some(..), Closed) => hir::LangItem::RangeToInclusive,
             (Some(..), Some(..), Closed) => unreachable!(),
-            (_, None, Closed) => {
+            (_, None, Closed) => self.diagnostic().span_fatal(span, "inclusive range with no end"),
-                self.diagnostic().span_fatal(span, "inclusive range with no end").raise()
-            }
         };
 
         let fields = self.arena.alloc_from_iter(
@@ -1331,319 +1329,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
         result
     }
 
-    fn lower_expr_asm(&mut self, sp: Span, asm: &InlineAsm) -> hir::ExprKind<'hir> {
-        // Rustdoc needs to support asm! from foriegn architectures: don't try
-        // lowering the register contraints in this case.
-        let asm_arch = if self.sess.opts.actually_rustdoc { None } else { self.sess.asm_arch };
-        if asm_arch.is_none() && !self.sess.opts.actually_rustdoc {
-            struct_span_err!(self.sess, sp, E0472, "asm! is unsupported on this target").emit();
-        }
-        if asm.options.contains(InlineAsmOptions::ATT_SYNTAX)
-            && !matches!(asm_arch, Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64))
-            && !self.sess.opts.actually_rustdoc
-        {
-            self.sess
-                .struct_span_err(sp, "the `att_syntax` option is only supported on x86")
-                .emit();
-        }
-
-        // Lower operands to HIR. We use dummy register classes if an error
-        // occurs during lowering because we still need to be able to produce a
-        // valid HIR.
-        let sess = self.sess;
-        let operands: Vec<_> = asm
-            .operands
-            .iter()
-            .map(|(op, op_sp)| {
-                let lower_reg = |reg| match reg {
-                    InlineAsmRegOrRegClass::Reg(s) => {
-                        asm::InlineAsmRegOrRegClass::Reg(if let Some(asm_arch) = asm_arch {
-                            asm::InlineAsmReg::parse(
-                                asm_arch,
-                                |feature| sess.target_features.contains(&Symbol::intern(feature)),
-                                &sess.target,
-                                s,
-                            )
-                            .unwrap_or_else(|e| {
-                                let msg = format!("invalid register `{}`: {}", s.as_str(), e);
-                                sess.struct_span_err(*op_sp, &msg).emit();
-                                asm::InlineAsmReg::Err
-                            })
-                        } else {
-                            asm::InlineAsmReg::Err
-                        })
-                    }
-                    InlineAsmRegOrRegClass::RegClass(s) => {
-                        asm::InlineAsmRegOrRegClass::RegClass(if let Some(asm_arch) = asm_arch {
-                            asm::InlineAsmRegClass::parse(asm_arch, s).unwrap_or_else(|e| {
-                                let msg = format!("invalid register class `{}`: {}", s.as_str(), e);
-                                sess.struct_span_err(*op_sp, &msg).emit();
-                                asm::InlineAsmRegClass::Err
-                            })
-                        } else {
-                            asm::InlineAsmRegClass::Err
-                        })
-                    }
-                };
-
-                let op = match *op {
-                    InlineAsmOperand::In { reg, ref expr } => hir::InlineAsmOperand::In {
-                        reg: lower_reg(reg),
-                        expr: self.lower_expr_mut(expr),
-                    },
-                    InlineAsmOperand::Out { reg, late, ref expr } => hir::InlineAsmOperand::Out {
-                        reg: lower_reg(reg),
-                        late,
-                        expr: expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
-                    },
-                    InlineAsmOperand::InOut { reg, late, ref expr } => {
-                        hir::InlineAsmOperand::InOut {
-                            reg: lower_reg(reg),
-                            late,
-                            expr: self.lower_expr_mut(expr),
-                        }
-                    }
-                    InlineAsmOperand::SplitInOut { reg, late, ref in_expr, ref out_expr } => {
-                        hir::InlineAsmOperand::SplitInOut {
-                            reg: lower_reg(reg),
-                            late,
-                            in_expr: self.lower_expr_mut(in_expr),
-                            out_expr: out_expr.as_ref().map(|expr| self.lower_expr_mut(expr)),
-                        }
-                    }
-                    InlineAsmOperand::Const { ref anon_const } => hir::InlineAsmOperand::Const {
-                        anon_const: self.lower_anon_const(anon_const),
-                    },
-                    InlineAsmOperand::Sym { ref expr } => {
-                        hir::InlineAsmOperand::Sym { expr: self.lower_expr_mut(expr) }
-                    }
-                };
-                (op, *op_sp)
-            })
-            .collect();
-
-        // Validate template modifiers against the register classes for the operands
-        for p in &asm.template {
-            if let InlineAsmTemplatePiece::Placeholder {
-                operand_idx,
-                modifier: Some(modifier),
-                span: placeholder_span,
-            } = *p
-            {
-                let op_sp = asm.operands[operand_idx].1;
-                match &operands[operand_idx].0 {
-                    hir::InlineAsmOperand::In { reg, .. }
-                    | hir::InlineAsmOperand::Out { reg, .. }
-                    | hir::InlineAsmOperand::InOut { reg, .. }
-                    | hir::InlineAsmOperand::SplitInOut { reg, .. } => {
-                        let class = reg.reg_class();
-                        if class == asm::InlineAsmRegClass::Err {
-                            continue;
-                        }
-                        let valid_modifiers = class.valid_modifiers(asm_arch.unwrap());
-                        if !valid_modifiers.contains(&modifier) {
-                            let mut err = sess.struct_span_err(
-                                placeholder_span,
-                                "invalid asm template modifier for this register class",
-                            );
-                            err.span_label(placeholder_span, "template modifier");
-                            err.span_label(op_sp, "argument");
-                            if !valid_modifiers.is_empty() {
-                                let mut mods = format!("`{}`", valid_modifiers[0]);
-                                for m in &valid_modifiers[1..] {
-                                    let _ = write!(mods, ", `{}`", m);
-                                }
-                                err.note(&format!(
-                                    "the `{}` register class supports \
-                                     the following template modifiers: {}",
-                                    class.name(),
-                                    mods
-                                ));
-                            } else {
-                                err.note(&format!(
-                                    "the `{}` register class does not support template modifiers",
-                                    class.name()
-                                ));
-                            }
-                            err.emit();
-                        }
-                    }
-                    hir::InlineAsmOperand::Const { .. } => {
-                        let mut err = sess.struct_span_err(
-                            placeholder_span,
-                            "asm template modifiers are not allowed for `const` arguments",
-                        );
-                        err.span_label(placeholder_span, "template modifier");
-                        err.span_label(op_sp, "argument");
-                        err.emit();
-                    }
-                    hir::InlineAsmOperand::Sym { .. } => {
-                        let mut err = sess.struct_span_err(
-                            placeholder_span,
-                            "asm template modifiers are not allowed for `sym` arguments",
-                        );
-                        err.span_label(placeholder_span, "template modifier");
-                        err.span_label(op_sp, "argument");
-                        err.emit();
-                    }
-                }
-            }
-        }
-
-        let mut used_input_regs = FxHashMap::default();
-        let mut used_output_regs = FxHashMap::default();
-        let mut required_features: Vec<&str> = vec![];
-        for (idx, &(ref op, op_sp)) in operands.iter().enumerate() {
-            if let Some(reg) = op.reg() {
-                // Make sure we don't accidentally carry features from the
-                // previous iteration.
-                required_features.clear();
-
-                let reg_class = reg.reg_class();
-                if reg_class == asm::InlineAsmRegClass::Err {
-                    continue;
-                }
-
-                // We ignore target feature requirements for clobbers: if the
-                // feature is disabled then the compiler doesn't care what we
-                // do with the registers.
-                //
-                // Note that this is only possible for explicit register
-                // operands, which cannot be used in the asm string.
-                let is_clobber = matches!(
-                    op,
-                    hir::InlineAsmOperand::Out {
-                        reg: asm::InlineAsmRegOrRegClass::Reg(_),
-                        late: _,
-                        expr: None
-                    }
-                );
-
-                if !is_clobber {
-                    // Validate register classes against currently enabled target
-                    // features. We check that at least one type is available for
-                    // the current target.
-                    for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) {
-                        if let Some(feature) = feature {
-                            if self.sess.target_features.contains(&Symbol::intern(feature)) {
-                                required_features.clear();
-                                break;
-                            } else {
-                                required_features.push(feature);
-                            }
-                        } else {
-                            required_features.clear();
-                            break;
-                        }
-                    }
-                    // We are sorting primitive strs here and can use unstable sort here
-                    required_features.sort_unstable();
-                    required_features.dedup();
-                    match &required_features[..] {
-                        [] => {}
-                        [feature] => {
-                            let msg = format!(
-                                "register class `{}` requires the `{}` target feature",
-                                reg_class.name(),
-                                feature
-                            );
-                            sess.struct_span_err(op_sp, &msg).emit();
-                        }
-                        features => {
-                            let msg = format!(
-                                "register class `{}` requires at least one target feature: {}",
-                                reg_class.name(),
-                                features.join(", ")
-                            );
-                            sess.struct_span_err(op_sp, &msg).emit();
-                        }
-                    }
-                }
-
-                // Check for conflicts between explicit register operands.
-                if let asm::InlineAsmRegOrRegClass::Reg(reg) = reg {
-                    let (input, output) = match op {
-                        hir::InlineAsmOperand::In { .. } => (true, false),
-                        // Late output do not conflict with inputs, but normal outputs do
-                        hir::InlineAsmOperand::Out { late, .. } => (!late, true),
-                        hir::InlineAsmOperand::InOut { .. }
-                        | hir::InlineAsmOperand::SplitInOut { .. } => (true, true),
-                        hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::Sym { .. } => {
-                            unreachable!()
-                        }
-                    };
-
-                    // Flag to output the error only once per operand
-                    let mut skip = false;
-                    reg.overlapping_regs(|r| {
-                        let mut check = |used_regs: &mut FxHashMap<asm::InlineAsmReg, usize>,
-                                         input| {
-                            match used_regs.entry(r) {
-                                Entry::Occupied(o) => {
-                                    if skip {
-                                        return;
-                                    }
-                                    skip = true;
-
-                                    let idx2 = *o.get();
-                                    let &(ref op2, op_sp2) = &operands[idx2];
-                                    let reg2 = match op2.reg() {
-                                        Some(asm::InlineAsmRegOrRegClass::Reg(r)) => r,
-                                        _ => unreachable!(),
-                                    };
-
-                                    let msg = format!(
-                                        "register `{}` conflicts with register `{}`",
-                                        reg.name(),
-                                        reg2.name()
-                                    );
-                                    let mut err = sess.struct_span_err(op_sp, &msg);
-                                    err.span_label(op_sp, &format!("register `{}`", reg.name()));
-                                    err.span_label(op_sp2, &format!("register `{}`", reg2.name()));
-
-                                    match (op, op2) {
-                                        (
-                                            hir::InlineAsmOperand::In { .. },
-                                            hir::InlineAsmOperand::Out { late, .. },
-                                        )
-                                        | (
-                                            hir::InlineAsmOperand::Out { late, .. },
-                                            hir::InlineAsmOperand::In { .. },
-                                        ) => {
-                                            assert!(!*late);
-                                            let out_op_sp = if input { op_sp2 } else { op_sp };
-                                            let msg = "use `lateout` instead of \
-                                                    `out` to avoid conflict";
-                                            err.span_help(out_op_sp, msg);
-                                        }
-                                        _ => {}
-                                    }
-
-                                    err.emit();
-                                }
-                                Entry::Vacant(v) => {
-                                    v.insert(idx);
-                                }
-                            }
-                        };
-                        if input {
-                            check(&mut used_input_regs, true);
-                        }
-                        if output {
-                            check(&mut used_output_regs, false);
-                        }
-                    });
-                }
-            }
-        }
-
-        let operands = self.arena.alloc_from_iter(operands);
-        let template = self.arena.alloc_from_iter(asm.template.iter().cloned());
-        let line_spans = self.arena.alloc_slice(&asm.line_spans[..]);
-        let hir_asm = hir::InlineAsm { template, operands, options: asm.options, line_spans };
-        hir::ExprKind::InlineAsm(self.arena.alloc(hir_asm))
-    }
-
     fn lower_expr_llvm_asm(&mut self, asm: &LlvmInlineAsm) -> hir::ExprKind<'hir> {
         let inner = hir::LlvmInlineAsmInner {
             inputs: asm.inputs.iter().map(|&(c, _)| c).collect(),
@@ -1896,14 +1581,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.allow_try_trait.clone(),
         );
 
-        // `Try::into_result(<expr>)`
+        // `Try::branch(<expr>)`
         let scrutinee = {
             // expand <expr>
             let sub_expr = self.lower_expr_mut(sub_expr);
 
             self.expr_call_lang_item_fn(
                 unstable_span,
-                hir::LangItem::TryIntoResult,
+                hir::LangItem::TryTraitBranch,
                 arena_vec![self; sub_expr],
             )
         };
@@ -1921,8 +1606,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         };
         let attrs = vec![attr];
 
-        // `Ok(val) => #[allow(unreachable_code)] val,`
+        // `ControlFlow::Continue(val) => #[allow(unreachable_code)] val,`
-        let ok_arm = {
+        let continue_arm = {
             let val_ident = Ident::with_dummy_span(sym::val);
             let (val_pat, val_pat_nid) = self.pat_ident(span, val_ident);
             let val_expr = self.arena.alloc(self.expr_ident_with_attrs(
@@ -1931,27 +1616,21 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 val_pat_nid,
                 ThinVec::from(attrs.clone()),
             ));
-            let ok_pat = self.pat_ok(span, val_pat);
+            let continue_pat = self.pat_cf_continue(unstable_span, val_pat);
-            self.arm(ok_pat, val_expr)
+            self.arm(continue_pat, val_expr)
         };
 
-        // `Err(err) => #[allow(unreachable_code)]
+        // `ControlFlow::Break(residual) =>
-        //              return Try::from_error(From::from(err)),`
+        //     #[allow(unreachable_code)]
-        let err_arm = {
+        //     return Try::from_residual(residual),`
-            let err_ident = Ident::with_dummy_span(sym::err);
+        let break_arm = {
-            let (err_local, err_local_nid) = self.pat_ident(try_span, err_ident);
+            let residual_ident = Ident::with_dummy_span(sym::residual);
-            let from_expr = {
+            let (residual_local, residual_local_nid) = self.pat_ident(try_span, residual_ident);
-                let err_expr = self.expr_ident_mut(try_span, err_ident, err_local_nid);
+            let residual_expr = self.expr_ident_mut(try_span, residual_ident, residual_local_nid);
-                self.expr_call_lang_item_fn(
+            let from_residual_expr = self.wrap_in_try_constructor(
-                    try_span,
+                hir::LangItem::TryTraitFromResidual,
-                    hir::LangItem::FromFrom,
+                try_span,
-                    arena_vec![self; err_expr],
+                self.arena.alloc(residual_expr),
-                )
-            };
-            let from_err_expr = self.wrap_in_try_constructor(
-                hir::LangItem::TryFromError,
-                unstable_span,
-                from_expr,
                 unstable_span,
             );
             let thin_attrs = ThinVec::from(attrs);
@@ -1962,25 +1641,25 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     try_span,
                     hir::ExprKind::Break(
                         hir::Destination { label: None, target_id },
-                        Some(from_err_expr),
+                        Some(from_residual_expr),
                     ),
                     thin_attrs,
                 ))
             } else {
                 self.arena.alloc(self.expr(
                     try_span,
-                    hir::ExprKind::Ret(Some(from_err_expr)),
+                    hir::ExprKind::Ret(Some(from_residual_expr)),
                     thin_attrs,
                 ))
             };
 
-            let err_pat = self.pat_err(try_span, err_local);
+            let break_pat = self.pat_cf_break(try_span, residual_local);
-            self.arm(err_pat, ret_expr)
+            self.arm(break_pat, ret_expr)
         };
 
         hir::ExprKind::Match(
             scrutinee,
-            arena_vec![self; err_arm, ok_arm],
+            arena_vec![self; break_arm, continue_arm],
             hir::MatchSource::TryDesugar,
         )
     }

compiler/rustc_ast_lowering/src/item.rs

@@ -329,7 +329,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
                 }
             }
-            ItemKind::GlobalAsm(ref ga) => hir::ItemKind::GlobalAsm(self.lower_global_asm(ga)),
+            ItemKind::GlobalAsm(ref asm) => {
+                hir::ItemKind::GlobalAsm(self.lower_inline_asm(span, asm))
+            }
             ItemKind::TyAlias(box TyAliasKind(_, ref gen, _, Some(ref ty))) => {
                 // We lower
                 //
@@ -746,10 +748,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_global_asm(&mut self, ga: &GlobalAsm) -> &'hir hir::GlobalAsm {
-        self.arena.alloc(hir::GlobalAsm { asm: ga.asm })
-    }
-
     fn lower_variant(&mut self, v: &Variant) -> hir::Variant<'hir> {
         let id = self.lower_node_id(v.id);
         self.lower_attrs(id, &v.attrs);
@@ -791,7 +789,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_field_def(&mut self, (index, f): (usize, &FieldDef)) -> hir::FieldDef<'hir> {
+    pub(super) fn lower_field_def(
+        &mut self,
+        (index, f): (usize, &FieldDef),
+    ) -> hir::FieldDef<'hir> {
         let ty = if let TyKind::Path(ref qself, ref path) = f.ty.kind {
             let t = self.lower_path_ty(
                 &f.ty,

compiler/rustc_ast_lowering/src/lib.rs

@@ -31,7 +31,6 @@
 //! in the HIR, especially for multiple identifiers.
 
 #![feature(crate_visibility_modifier)]
-#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_patterns)]
 #![feature(iter_zip)]
 #![recursion_limit = "256"]
@@ -44,9 +43,9 @@ use rustc_ast::walk_list;
 use rustc_ast::{self as ast, *};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
-use rustc_errors::struct_span_err;
+use rustc_errors::{struct_span_err, Applicability};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Namespace, PartialRes, PerNS, Res};
 use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId, CRATE_DEF_ID};
@@ -58,10 +57,11 @@ use rustc_session::lint::builtin::{BARE_TRAIT_OBJECTS, MISSING_ABI};
 use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer};
 use rustc_session::utils::{FlattenNonterminals, NtToTokenstream};
 use rustc_session::Session;
+use rustc_span::edition::Edition;
 use rustc_span::hygiene::ExpnId;
 use rustc_span::source_map::{respan, DesugaringKind};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{Span, DUMMY_SP};
 use rustc_target::spec::abi::Abi;
 
 use smallvec::{smallvec, SmallVec};
@@ -76,6 +76,7 @@ macro_rules! arena_vec {
     });
 }
 
+mod asm;
 mod expr;
 mod item;
 mod pat;
@@ -165,7 +166,7 @@ struct LoweringContext<'a, 'hir: 'a> {
 
     type_def_lifetime_params: DefIdMap<usize>,
 
-    current_hir_id_owner: Vec<(LocalDefId, u32)>,
+    current_hir_id_owner: (LocalDefId, u32),
     item_local_id_counters: NodeMap<u32>,
     node_id_to_hir_id: IndexVec<NodeId, Option<hir::HirId>>,
 
@@ -197,7 +198,7 @@ pub trait ResolverAstLowering {
 
     fn next_node_id(&mut self) -> NodeId;
 
-    fn trait_map(&self) -> &NodeMap<Vec<hir::TraitCandidate>>;
+    fn take_trait_map(&mut self) -> NodeMap<Vec<hir::TraitCandidate>>;
 
     fn opt_local_def_id(&self, node: NodeId) -> Option<LocalDefId>;
 
@@ -321,7 +322,7 @@ pub fn lower_crate<'a, 'hir>(
         anonymous_lifetime_mode: AnonymousLifetimeMode::PassThrough,
         type_def_lifetime_params: Default::default(),
         current_module: CRATE_DEF_ID,
-        current_hir_id_owner: vec![(CRATE_DEF_ID, 0)],
+        current_hir_id_owner: (CRATE_DEF_ID, 0),
         item_local_id_counters: Default::default(),
         node_id_to_hir_id: IndexVec::new(),
         generator_kind: None,
@@ -330,7 +331,7 @@ pub fn lower_crate<'a, 'hir>(
         lifetimes_to_define: Vec::new(),
         is_collecting_in_band_lifetimes: false,
         in_scope_lifetimes: Vec::new(),
-        allow_try_trait: Some([sym::try_trait][..].into()),
+        allow_try_trait: Some([sym::control_flow_enum, sym::try_trait_v2][..].into()),
         allow_gen_future: Some([sym::gen_future][..].into()),
     }
     .lower_crate(krate)
@@ -500,14 +501,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let proc_macros =
             c.proc_macros.iter().map(|id| self.node_id_to_hir_id[*id].unwrap()).collect();
 
-        let trait_map = self
+        let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default();
-            .resolver
+        for (k, v) in self.resolver.take_trait_map().into_iter() {
-            .trait_map()
+            if let Some(Some(hir_id)) = self.node_id_to_hir_id.get(k) {
-            .iter()
+                let map = trait_map.entry(hir_id.owner).or_default();
-            .filter_map(|(&k, v)| {
+                map.insert(hir_id.local_id, v.into_boxed_slice());
-                self.node_id_to_hir_id.get(k).and_then(|id| id.as_ref()).map(|id| (*id, v.clone()))
+            }
-            })
+        }
-            .collect();
 
         let mut def_id_to_hir_id = IndexVec::default();
 
@@ -594,9 +594,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             .insert(owner, HIR_ID_COUNTER_LOCKED)
             .unwrap_or_else(|| panic!("no `item_local_id_counters` entry for {:?}", owner));
         let def_id = self.resolver.local_def_id(owner);
-        self.current_hir_id_owner.push((def_id, counter));
+        let old_owner = std::mem::replace(&mut self.current_hir_id_owner, (def_id, counter));
         let ret = f(self);
-        let (new_def_id, new_counter) = self.current_hir_id_owner.pop().unwrap();
+        let (new_def_id, new_counter) =
+            std::mem::replace(&mut self.current_hir_id_owner, old_owner);
 
         debug_assert!(def_id == new_def_id);
         debug_assert!(new_counter >= counter);
@@ -614,8 +615,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     /// properly. Calling the method twice with the same `NodeId` is fine though.
     fn lower_node_id(&mut self, ast_node_id: NodeId) -> hir::HirId {
         self.lower_node_id_generic(ast_node_id, |this| {
-            let &mut (owner, ref mut local_id_counter) =
+            let &mut (owner, ref mut local_id_counter) = &mut this.current_hir_id_owner;
-                this.current_hir_id_owner.last_mut().unwrap();
             let local_id = *local_id_counter;
             *local_id_counter += 1;
             hir::HirId { owner, local_id: hir::ItemLocalId::from_u32(local_id) }
@@ -868,10 +868,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // wouldn't have been added yet.
                     let generics = this.lower_generics_mut(
                         generics,
-                        ImplTraitContext::Universal(
+                        ImplTraitContext::Universal(&mut params, this.current_hir_id_owner.0),
-                            &mut params,
-                            this.current_hir_id_owner.last().unwrap().0,
-                        ),
                     );
                     let res = f(this, &mut params);
                     (params, (generics, res))
@@ -1077,7 +1074,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
             AssocTyConstraintKind::Bound { ref bounds } => {
                 let mut capturable_lifetimes;
-                let mut parent_def_id = self.current_hir_id_owner.last().unwrap().0;
+                let mut parent_def_id = self.current_hir_id_owner.0;
                 // Piggy-back on the `impl Trait` context to figure out the correct behavior.
                 let (desugar_to_impl_trait, itctx) = match itctx {
                     // We are in the return position:
@@ -1198,7 +1195,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                             // Construct a AnonConst where the expr is the "ty"'s path.
 
-                            let parent_def_id = self.current_hir_id_owner.last().unwrap().0;
+                            let parent_def_id = self.current_hir_id_owner.0;
                             let node_id = self.resolver.next_node_id();
 
                             // Add a definition for the in-band const def.
@@ -1268,6 +1265,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let kind = match t.kind {
             TyKind::Infer => hir::TyKind::Infer,
             TyKind::Err => hir::TyKind::Err,
+            // FIXME(unnamed_fields): IMPLEMENTATION IN PROGRESS
+            TyKind::AnonymousStruct(ref _fields, _recovered) => {
+                self.sess.struct_span_err(t.span, "anonymous structs are unimplemented").emit();
+                hir::TyKind::Err
+            }
+            TyKind::AnonymousUnion(ref _fields, _recovered) => {
+                self.sess.struct_span_err(t.span, "anonymous unions are unimplemented").emit();
+                hir::TyKind::Err
+            }
             TyKind::Slice(ref ty) => hir::TyKind::Slice(self.lower_ty(ty, itctx)),
             TyKind::Ptr(ref mt) => hir::TyKind::Ptr(self.lower_mt(mt, itctx)),
             TyKind::Rptr(ref region, ref mt) => {
@@ -1814,10 +1820,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 if let Some((_, ibty)) = &mut in_band_ty_params {
                     this.lower_ty_direct(
                         &param.ty,
-                        ImplTraitContext::Universal(
+                        ImplTraitContext::Universal(ibty, this.current_hir_id_owner.0),
-                            ibty,
-                            this.current_hir_id_owner.last().unwrap().0,
-                        ),
                     )
                 } else {
                     this.lower_ty_direct(&param.ty, ImplTraitContext::disallowed())
@@ -2089,6 +2092,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             args: &[],
             bindings: arena_vec![self; self.output_ty_binding(span, output_ty)],
             parenthesized: false,
+            span_ext: DUMMY_SP,
         });
 
         hir::GenericBound::LangItemTrait(
@@ -2484,14 +2488,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.pat(span, hir::PatKind::Lit(expr))
     }
 
-    fn pat_ok(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
+    fn pat_cf_continue(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::ResultOk, field)
+        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowContinue, field)
     }
 
-    fn pat_err(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
+    fn pat_cf_break(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::ResultErr, field)
+        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowBreak, field)
     }
 
     fn pat_some(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
@@ -2736,13 +2740,26 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             .map(|snippet| snippet.starts_with("#["))
             .unwrap_or(true);
         if !is_macro_callsite {
-            self.resolver.lint_buffer().buffer_lint_with_diagnostic(
+            if span.edition() < Edition::Edition2021 {
-                BARE_TRAIT_OBJECTS,
+                self.resolver.lint_buffer().buffer_lint_with_diagnostic(
-                id,
+                    BARE_TRAIT_OBJECTS,
-                span,
+                    id,
-                "trait objects without an explicit `dyn` are deprecated",
+                    span,
-                BuiltinLintDiagnostics::BareTraitObject(span, is_global),
+                    "trait objects without an explicit `dyn` are deprecated",
-            )
+                    BuiltinLintDiagnostics::BareTraitObject(span, is_global),
+                )
+            } else {
+                let msg = "trait objects must include the `dyn` keyword";
+                let label = "add `dyn` keyword before this trait";
+                let mut err = struct_span_err!(self.sess, span, E0782, "{}", msg,);
+                err.span_suggestion_verbose(
+                    span.shrink_to_lo(),
+                    label,
+                    String::from("dyn "),
+                    Applicability::MachineApplicable,
+                );
+                err.emit();
+            }
         }
     }
 
@@ -2780,6 +2797,7 @@ struct GenericArgsCtor<'hir> {
     args: SmallVec<[hir::GenericArg<'hir>; 4]>,
     bindings: &'hir [hir::TypeBinding<'hir>],
     parenthesized: bool,
+    span: Span,
 }
 
 impl<'hir> GenericArgsCtor<'hir> {
@@ -2792,6 +2810,7 @@ impl<'hir> GenericArgsCtor<'hir> {
             args: arena.alloc_from_iter(self.args),
             bindings: self.bindings,
             parenthesized: self.parenthesized,
+            span_ext: self.span,
         }
     }
 }

compiler/rustc_ast_lowering/src/pat.rs

@@ -21,10 +21,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         break self.lower_pat_ident(pattern, binding_mode, ident, lower_sub);
                     }
                     PatKind::Lit(ref e) => break hir::PatKind::Lit(self.lower_expr(e)),
-                    PatKind::TupleStruct(ref path, ref pats) => {
+                    PatKind::TupleStruct(ref qself, ref path, ref pats) => {
                         let qpath = self.lower_qpath(
                             pattern.id,
-                            &None,
+                            qself,
                             path,
                             ParamMode::Optional,
                             ImplTraitContext::disallowed(),
@@ -47,10 +47,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         );
                         break hir::PatKind::Path(qpath);
                     }
-                    PatKind::Struct(ref path, ref fields, etc) => {
+                    PatKind::Struct(ref qself, ref path, ref fields, etc) => {
                         let qpath = self.lower_qpath(
                             pattern.id,
-                            &None,
+                            qself,
                             path,
                             ParamMode::Optional,
                             ImplTraitContext::disallowed(),

compiler/rustc_ast_lowering/src/path.rs

@@ -10,7 +10,7 @@ use rustc_hir::GenericArg;
 use rustc_session::lint::builtin::ELIDED_LIFETIMES_IN_PATHS;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_span::symbol::Ident;
-use rustc_span::Span;
+use rustc_span::{BytePos, Span, DUMMY_SP};
 
 use smallvec::smallvec;
 use tracing::debug;
@@ -24,6 +24,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         param_mode: ParamMode,
         mut itctx: ImplTraitContext<'_, 'hir>,
     ) -> hir::QPath<'hir> {
+        debug!("lower_qpath(id: {:?}, qself: {:?}, p: {:?})", id, qself, p);
         let qself_position = qself.as_ref().map(|q| q.position);
         let qself = qself.as_ref().map(|q| self.lower_ty(&q.ty, itctx.reborrow()));
 
@@ -222,6 +223,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         itctx: ImplTraitContext<'_, 'hir>,
         explicit_owner: Option<NodeId>,
     ) -> hir::PathSegment<'hir> {
+        debug!(
+            "path_span: {:?}, lower_path_segment(segment: {:?}, expected_lifetimes: {:?})",
+            path_span, segment, expected_lifetimes
+        );
         let (mut generic_args, infer_args) = if let Some(ref generic_args) = segment.args {
             let msg = "parenthesized type parameters may only be used with a `Fn` trait";
             match **generic_args {
@@ -262,23 +267,34 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 },
             }
         } else {
-            self.lower_angle_bracketed_parameter_data(&Default::default(), param_mode, itctx)
+            (
+                GenericArgsCtor {
+                    args: Default::default(),
+                    bindings: &[],
+                    parenthesized: false,
+                    span: path_span.shrink_to_hi(),
+                },
+                param_mode == ParamMode::Optional,
+            )
         };
 
         let has_lifetimes =
             generic_args.args.iter().any(|arg| matches!(arg, GenericArg::Lifetime(_)));
-        let first_generic_span = generic_args
-            .args
-            .iter()
-            .map(|a| a.span())
-            .chain(generic_args.bindings.iter().map(|b| b.span))
-            .next();
         if !generic_args.parenthesized && !has_lifetimes {
+            // Note: these spans are used for diagnostics when they can't be inferred.
+            // See rustc_resolve::late::lifetimes::LifetimeContext::add_missing_lifetime_specifiers_label
+            let elided_lifetime_span = if generic_args.span.is_empty() {
+                // If there are no brackets, use the identifier span.
+                segment.ident.span
+            } else if generic_args.is_empty() {
+                // If there are brackets, but not generic arguments, then use the opening bracket
+                generic_args.span.with_hi(generic_args.span.lo() + BytePos(1))
+            } else {
+                // Else use an empty span right after the opening bracket.
+                generic_args.span.with_lo(generic_args.span.lo() + BytePos(1)).shrink_to_lo()
+            };
             generic_args.args = self
-                .elided_path_lifetimes(
+                .elided_path_lifetimes(elided_lifetime_span, expected_lifetimes)
-                    first_generic_span.map_or(segment.ident.span, |s| s.shrink_to_lo()),
-                    expected_lifetimes,
-                )
                 .map(GenericArg::Lifetime)
                 .chain(generic_args.args.into_iter())
                 .collect();
@@ -287,15 +303,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let no_non_lt_args = generic_args.args.len() == expected_lifetimes;
                 let no_bindings = generic_args.bindings.is_empty();
                 let (incl_angl_brckt, insertion_sp, suggestion) = if no_non_lt_args && no_bindings {
-                    // If there are no (non-implicit) generic args or associated type
+                    // If there are no generic args, our suggestion can include the angle brackets.
-                    // bindings, our suggestion includes the angle brackets.
                     (true, path_span.shrink_to_hi(), format!("<{}>", anon_lt_suggestion))
                 } else {
-                    // Otherwise (sorry, this is kind of gross) we need to infer the
+                    // Otherwise we'll insert a `'_, ` right after the opening bracket.
-                    // place to splice in the `'_, ` from the generics that do exist.
+                    let span = generic_args
-                    let first_generic_span = first_generic_span
+                        .span
-                        .expect("already checked that non-lifetime args or bindings exist");
+                        .with_lo(generic_args.span.lo() + BytePos(1))
-                    (false, first_generic_span.shrink_to_lo(), format!("{}, ", anon_lt_suggestion))
+                        .shrink_to_lo();
+                    (false, span, format!("{}, ", anon_lt_suggestion))
                 };
                 match self.anonymous_lifetime_mode {
                     // In create-parameter mode we error here because we don't want to support
@@ -357,7 +373,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             hir_id: Some(id),
             res: Some(self.lower_res(res)),
             infer_args,
-            args: if generic_args.is_empty() {
+            args: if generic_args.is_empty() && generic_args.span.is_empty() {
                 None
             } else {
                 Some(self.arena.alloc(generic_args.into_generic_args(self.arena)))
@@ -390,7 +406,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
             AngleBracketedArg::Arg(_) => None,
         }));
-        let ctor = GenericArgsCtor { args, bindings, parenthesized: false };
+        let ctor = GenericArgsCtor { args, bindings, parenthesized: false, span: data.span };
         (ctor, !has_non_lt_args && param_mode == ParamMode::Optional)
     }
 
@@ -415,7 +431,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             let args = smallvec![GenericArg::Type(this.ty_tup(*inputs_span, inputs))];
             let binding = this.output_ty_binding(output_ty.span, output_ty);
             (
-                GenericArgsCtor { args, bindings: arena_vec![this; binding], parenthesized: true },
+                GenericArgsCtor {
+                    args,
+                    bindings: arena_vec![this; binding],
+                    parenthesized: true,
+                    span: data.inputs_span,
+                },
                 false,
             )
         })
@@ -431,7 +452,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let kind = hir::TypeBindingKind::Equality { ty };
         let args = arena_vec![self;];
         let bindings = arena_vec![self;];
-        let gen_args = self.arena.alloc(hir::GenericArgs { args, bindings, parenthesized: false });
+        let gen_args = self.arena.alloc(hir::GenericArgs {
+            args,
+            bindings,
+            parenthesized: false,
+            span_ext: DUMMY_SP,
+        });
         hir::TypeBinding { hir_id: self.next_id(), gen_args, span, ident, kind }
     }
 }

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -175,10 +175,30 @@ impl<'a> AstValidator<'a> {
                     }
                 }
             }
+            TyKind::AnonymousStruct(ref fields, ..) | TyKind::AnonymousUnion(ref fields, ..) => {
+                self.with_banned_assoc_ty_bound(|this| {
+                    walk_list!(this, visit_struct_field_def, fields)
+                });
+            }
             _ => visit::walk_ty(self, t),
         }
     }
 
+    fn visit_struct_field_def(&mut self, field: &'a FieldDef) {
+        if let Some(ident) = field.ident {
+            if ident.name == kw::Underscore {
+                self.check_anonymous_field(field);
+                self.visit_vis(&field.vis);
+                self.visit_ident(ident);
+                self.visit_ty_common(&field.ty);
+                self.walk_ty(&field.ty);
+                walk_list!(self, visit_attribute, &field.attrs);
+                return;
+            }
+        }
+        self.visit_field_def(field);
+    }
+
     fn err_handler(&self) -> &rustc_errors::Handler {
         &self.session.diagnostic()
     }
@@ -213,6 +233,66 @@ impl<'a> AstValidator<'a> {
         err.emit();
     }
 
+    fn check_anonymous_field(&self, field: &FieldDef) {
+        let FieldDef { ty, .. } = field;
+        match &ty.kind {
+            TyKind::AnonymousStruct(..) | TyKind::AnonymousUnion(..) => {
+                // We already checked for `kw::Underscore` before calling this function,
+                // so skip the check
+            }
+            TyKind::Path(..) => {
+                // If the anonymous field contains a Path as type, we can't determine
+                // if the path is a valid struct or union, so skip the check
+            }
+            _ => {
+                let msg = "unnamed fields can only have struct or union types";
+                let label = "not a struct or union";
+                self.err_handler()
+                    .struct_span_err(field.span, msg)
+                    .span_label(ty.span, label)
+                    .emit();
+            }
+        }
+    }
+
+    fn deny_anonymous_struct(&self, ty: &Ty) {
+        match &ty.kind {
+            TyKind::AnonymousStruct(..) => {
+                self.err_handler()
+                    .struct_span_err(
+                        ty.span,
+                        "anonymous structs are not allowed outside of unnamed struct or union fields",
+                    )
+                    .span_label(ty.span, "anonymous struct declared here")
+                    .emit();
+            }
+            TyKind::AnonymousUnion(..) => {
+                self.err_handler()
+                    .struct_span_err(
+                        ty.span,
+                        "anonymous unions are not allowed outside of unnamed struct or union fields",
+                    )
+                    .span_label(ty.span, "anonymous union declared here")
+                    .emit();
+            }
+            _ => {}
+        }
+    }
+
+    fn deny_anonymous_field(&self, field: &FieldDef) {
+        if let Some(ident) = field.ident {
+            if ident.name == kw::Underscore {
+                self.err_handler()
+                    .struct_span_err(
+                        field.span,
+                        "anonymous fields are not allowed outside of structs or unions",
+                    )
+                    .span_label(ident.span, "anonymous field declared here")
+                    .emit()
+            }
+        }
+    }
+
     fn check_decl_no_pat(decl: &FnDecl, mut report_err: impl FnMut(Span, Option<Ident>, bool)) {
         for Param { pat, .. } in &decl.inputs {
             match pat.kind {
@@ -732,6 +812,71 @@ impl<'a> AstValidator<'a> {
             )
             .emit();
     }
+
+    fn visit_ty_common(&mut self, ty: &'a Ty) {
+        match ty.kind {
+            TyKind::BareFn(ref bfty) => {
+                self.check_fn_decl(&bfty.decl, SelfSemantic::No);
+                Self::check_decl_no_pat(&bfty.decl, |span, _, _| {
+                    struct_span_err!(
+                        self.session,
+                        span,
+                        E0561,
+                        "patterns aren't allowed in function pointer types"
+                    )
+                    .emit();
+                });
+                self.check_late_bound_lifetime_defs(&bfty.generic_params);
+            }
+            TyKind::TraitObject(ref bounds, ..) => {
+                let mut any_lifetime_bounds = false;
+                for bound in bounds {
+                    if let GenericBound::Outlives(ref lifetime) = *bound {
+                        if any_lifetime_bounds {
+                            struct_span_err!(
+                                self.session,
+                                lifetime.ident.span,
+                                E0226,
+                                "only a single explicit lifetime bound is permitted"
+                            )
+                            .emit();
+                            break;
+                        }
+                        any_lifetime_bounds = true;
+                    }
+                }
+                self.no_questions_in_bounds(bounds, "trait object types", false);
+            }
+            TyKind::ImplTrait(_, ref bounds) => {
+                if self.is_impl_trait_banned {
+                    struct_span_err!(
+                        self.session,
+                        ty.span,
+                        E0667,
+                        "`impl Trait` is not allowed in path parameters"
+                    )
+                    .emit();
+                }
+
+                if let Some(outer_impl_trait_sp) = self.outer_impl_trait {
+                    struct_span_err!(
+                        self.session,
+                        ty.span,
+                        E0666,
+                        "nested `impl Trait` is not allowed"
+                    )
+                    .span_label(outer_impl_trait_sp, "outer `impl Trait`")
+                    .span_label(ty.span, "nested `impl Trait` here")
+                    .emit();
+                }
+
+                if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) {
+                    self.err_handler().span_err(ty.span, "at least one trait must be specified");
+                }
+            }
+            _ => {}
+        }
+    }
 }
 
 /// Checks that generic parameters are in the correct order,
@@ -793,8 +938,11 @@ fn validate_generic_param_order(
                 }
                 GenericParamKind::Type { default: None } => (),
                 GenericParamKind::Lifetime => (),
-                // FIXME(const_generics_defaults)
+                GenericParamKind::Const { ty: _, kw_span: _, default: Some(default) } => {
-                GenericParamKind::Const { ty: _, kw_span: _, default: _ } => (),
+                    ordered_params += " = ";
+                    ordered_params += &pprust::expr_to_string(&*default.value);
+                }
+                GenericParamKind::Const { ty: _, kw_span: _, default: None } => (),
             }
             first = false;
         }
@@ -814,7 +962,7 @@ fn validate_generic_param_order(
             span,
             &format!(
                 "reorder the parameters: lifetimes, {}",
-                if sess.features_untracked().const_generics {
+                if sess.features_untracked().unordered_const_ty_params() {
                     "then consts and types"
                 } else {
                     "then types, then consts"
@@ -850,72 +998,8 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
     }
 
     fn visit_ty(&mut self, ty: &'a Ty) {
-        match ty.kind {
+        self.visit_ty_common(ty);
-            TyKind::BareFn(ref bfty) => {
+        self.deny_anonymous_struct(ty);
-                self.check_fn_decl(&bfty.decl, SelfSemantic::No);
-                Self::check_decl_no_pat(&bfty.decl, |span, _, _| {
-                    struct_span_err!(
-                        self.session,
-                        span,
-                        E0561,
-                        "patterns aren't allowed in function pointer types"
-                    )
-                    .emit();
-                });
-                self.check_late_bound_lifetime_defs(&bfty.generic_params);
-            }
-            TyKind::TraitObject(ref bounds, ..) => {
-                let mut any_lifetime_bounds = false;
-                for bound in bounds {
-                    if let GenericBound::Outlives(ref lifetime) = *bound {
-                        if any_lifetime_bounds {
-                            struct_span_err!(
-                                self.session,
-                                lifetime.ident.span,
-                                E0226,
-                                "only a single explicit lifetime bound is permitted"
-                            )
-                            .emit();
-                            break;
-                        }
-                        any_lifetime_bounds = true;
-                    }
-                }
-                self.no_questions_in_bounds(bounds, "trait object types", false);
-            }
-            TyKind::ImplTrait(_, ref bounds) => {
-                if self.is_impl_trait_banned {
-                    struct_span_err!(
-                        self.session,
-                        ty.span,
-                        E0667,
-                        "`impl Trait` is not allowed in path parameters"
-                    )
-                    .emit();
-                }
-
-                if let Some(outer_impl_trait_sp) = self.outer_impl_trait {
-                    struct_span_err!(
-                        self.session,
-                        ty.span,
-                        E0666,
-                        "nested `impl Trait` is not allowed"
-                    )
-                    .span_label(outer_impl_trait_sp, "outer `impl Trait`")
-                    .span_label(ty.span, "nested `impl Trait` here")
-                    .emit();
-                }
-
-                if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) {
-                    self.err_handler().span_err(ty.span, "at least one trait must be specified");
-                }
-
-                self.walk_ty(ty);
-                return;
-            }
-            _ => {}
-        }
-
         self.walk_ty(ty)
     }
 
@@ -929,6 +1013,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         visit::walk_lifetime(self, lifetime);
     }
 
+    fn visit_field_def(&mut self, s: &'a FieldDef) {
+        self.deny_anonymous_field(s);
+        visit::walk_field_def(self, s)
+    }
+
     fn visit_item(&mut self, item: &'a Item) {
         if item.attrs.iter().any(|attr| self.session.is_proc_macro_attr(attr)) {
             self.has_proc_macro_decls = true;
@@ -1084,14 +1173,42 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     self.check_mod_file_item_asciionly(item.ident);
                 }
             }
-            ItemKind::Union(ref vdata, _) => {
+            ItemKind::Struct(ref vdata, ref generics) => match vdata {
-                if let VariantData::Tuple(..) | VariantData::Unit(..) = vdata {
+                // Duplicating the `Visitor` logic allows catching all cases
-                    self.err_handler()
+                // of `Anonymous(Struct, Union)` outside of a field struct or union.
-                        .span_err(item.span, "tuple and unit unions are not permitted");
+                //
+                // Inside `visit_ty` the validator catches every `Anonymous(Struct, Union)` it
+                // encounters, and only on `ItemKind::Struct` and `ItemKind::Union`
+                // it uses `visit_ty_common`, which doesn't contain that specific check.
+                VariantData::Struct(ref fields, ..) => {
+                    self.visit_vis(&item.vis);
+                    self.visit_ident(item.ident);
+                    self.visit_generics(generics);
+                    self.with_banned_assoc_ty_bound(|this| {
+                        walk_list!(this, visit_struct_field_def, fields);
+                    });
+                    walk_list!(self, visit_attribute, &item.attrs);
+                    return;
                 }
+                _ => {}
+            },
+            ItemKind::Union(ref vdata, ref generics) => {
                 if vdata.fields().is_empty() {
                     self.err_handler().span_err(item.span, "unions cannot have zero fields");
                 }
+                match vdata {
+                    VariantData::Struct(ref fields, ..) => {
+                        self.visit_vis(&item.vis);
+                        self.visit_ident(item.ident);
+                        self.visit_generics(generics);
+                        self.with_banned_assoc_ty_bound(|this| {
+                            walk_list!(this, visit_struct_field_def, fields);
+                        });
+                        walk_list!(self, visit_attribute, &item.attrs);
+                        return;
+                    }
+                    _ => {}
+                }
             }
             ItemKind::Const(def, .., None) => {
                 self.check_defaultness(item.span, def);

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -318,7 +318,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 }}
 
                 gate_doc!(
-                    include => external_doc
                     cfg => doc_cfg
                     masked => doc_masked
                     notable_trait => doc_notable_trait
@@ -326,6 +325,45 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 );
             }
         }
+
+        // Check for unstable modifiers on `#[link(..)]` attribute
+        if self.sess.check_name(attr, sym::link) {
+            for nested_meta in attr.meta_item_list().unwrap_or_default() {
+                if nested_meta.has_name(sym::modifiers) {
+                    gate_feature_post!(
+                        self,
+                        native_link_modifiers,
+                        nested_meta.span(),
+                        "native link modifiers are experimental"
+                    );
+
+                    if let Some(modifiers) = nested_meta.value_str() {
+                        for modifier in modifiers.as_str().split(',') {
+                            if let Some(modifier) = modifier.strip_prefix(&['+', '-'][..]) {
+                                macro_rules! gate_modifier { ($($name:literal => $feature:ident)*) => {
+                                    $(if modifier == $name {
+                                        let msg = concat!("`#[link(modifiers=\"", $name, "\")]` is unstable");
+                                        gate_feature_post!(
+                                            self,
+                                            $feature,
+                                            nested_meta.name_value_literal_span().unwrap(),
+                                            msg
+                                        );
+                                    })*
+                                }}
+
+                                gate_modifier!(
+                                    "bundle" => native_link_modifiers_bundle
+                                    "verbatim" => native_link_modifiers_verbatim
+                                    "whole-archive" => native_link_modifiers_whole_archive
+                                    "as-needed" => native_link_modifiers_as_needed
+                                );
+                            }
+                        }
+                    }
+                }
+            }
+        }
     }
 
     fn visit_item(&mut self, i: &'a ast::Item) {
@@ -667,16 +705,13 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
         "async closures are unstable",
         "to use an async block, remove the `||`: `async {`"
     );
+    gate_all!(more_qualified_paths, "usage of qualified paths in this context is experimental");
     gate_all!(generators, "yield syntax is experimental");
     gate_all!(raw_ref_op, "raw address of syntax is experimental");
     gate_all!(const_trait_bound_opt_out, "`?const` on trait bounds is experimental");
     gate_all!(const_trait_impl, "const trait impls are experimental");
     gate_all!(half_open_range_patterns, "half-open range patterns are unstable");
     gate_all!(inline_const, "inline-const is experimental");
-    gate_all!(
-        extended_key_value_attributes,
-        "arbitrary expressions in key-value attributes are unstable"
-    );
     gate_all!(
         const_generics_defaults,
         "default values for const generic parameters are experimental"
@@ -686,6 +721,7 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) {
         // involved, so we only emit errors where there are no other parsing errors.
         gate_all!(destructuring_assignment, "destructuring assignments are unstable");
     }
+    gate_all!(unnamed_fields, "unnamed fields are not yet fully implemented");
 
     // All uses of `gate_all!` below this point were added in #65742,
     // and subsequently disabled (with the non-early gating readded).

compiler/rustc_ast_passes/src/lib.rs

@@ -6,7 +6,6 @@
 
 #![feature(bindings_after_at)]
 #![feature(iter_is_partitioned)]
-#![feature(box_syntax)]
 #![feature(box_patterns)]
 #![recursion_limit = "256"]
 

compiler/rustc_ast_pretty/src/lib.rs

@@ -1,6 +1,5 @@
 #![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
-#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(box_patterns)]
 #![recursion_limit = "256"]
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -140,12 +140,15 @@ pub fn print_crate<'a>(
 // and also addresses some specific regressions described in #63896 and #73345.
 fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
     if let TokenTree::Token(token) = prev {
+        if matches!(token.kind, token::Dot) {
+            return false;
+        }
         if let token::DocComment(comment_kind, ..) = token.kind {
             return comment_kind != CommentKind::Line;
         }
     }
     match tt {
-        TokenTree::Token(token) => token.kind != token::Comma,
+        TokenTree::Token(token) => !matches!(token.kind, token::Comma | token::Not | token::Dot),
         TokenTree::Delimited(_, DelimToken::Paren, _) => {
             !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. }))
         }
@@ -955,6 +958,14 @@ impl<'a> State<'a> {
                 }
                 self.pclose();
             }
+            ast::TyKind::AnonymousStruct(ref fields, ..) => {
+                self.head("struct");
+                self.print_record_struct_body(&fields, ty.span);
+            }
+            ast::TyKind::AnonymousUnion(ref fields, ..) => {
+                self.head("union");
+                self.print_record_struct_body(&fields, ty.span);
+            }
             ast::TyKind::Paren(ref typ) => {
                 self.popen();
                 self.print_type(typ);
@@ -1168,9 +1179,9 @@ impl<'a> State<'a> {
                 self.print_foreign_mod(nmod, &item.attrs);
                 self.bclose(item.span);
             }
-            ast::ItemKind::GlobalAsm(ref ga) => {
+            ast::ItemKind::GlobalAsm(ref asm) => {
                 self.head(visibility_qualified(&item.vis, "global_asm!"));
-                self.s.word(ga.asm.to_string());
+                self.print_inline_asm(asm);
                 self.end();
             }
             ast::ItemKind::TyAlias(box ast::TyAliasKind(def, ref generics, ref bounds, ref ty)) => {
@@ -1390,6 +1401,24 @@ impl<'a> State<'a> {
         }
     }
 
+    crate fn print_record_struct_body(&mut self, fields: &[ast::FieldDef], span: rustc_span::Span) {
+        self.bopen();
+        self.hardbreak_if_not_bol();
+
+        for field in fields {
+            self.hardbreak_if_not_bol();
+            self.maybe_print_comment(field.span.lo());
+            self.print_outer_attributes(&field.attrs);
+            self.print_visibility(&field.vis);
+            self.print_ident(field.ident.unwrap());
+            self.word_nbsp(":");
+            self.print_type(&field.ty);
+            self.s.word(",");
+        }
+
+        self.bclose(span)
+    }
+
     crate fn print_struct(
         &mut self,
         struct_def: &ast::VariantData,
@@ -1419,24 +1448,10 @@ impl<'a> State<'a> {
                 self.end();
                 self.end(); // Close the outer-box.
             }
-            ast::VariantData::Struct(..) => {
+            ast::VariantData::Struct(ref fields, ..) => {
                 self.print_where_clause(&generics.where_clause);
                 self.nbsp();
-                self.bopen();
+                self.print_record_struct_body(fields, span);
-                self.hardbreak_if_not_bol();
-
-                for field in struct_def.fields() {
-                    self.hardbreak_if_not_bol();
-                    self.maybe_print_comment(field.span.lo());
-                    self.print_outer_attributes(&field.attrs);
-                    self.print_visibility(&field.vis);
-                    self.print_ident(field.ident.unwrap());
-                    self.word_nbsp(":");
-                    self.print_type(&field.ty);
-                    self.s.word(",");
-                }
-
-                self.bclose(span)
             }
         }
     }
@@ -1675,32 +1690,24 @@ impl<'a> State<'a> {
         }
     }
 
-    fn print_expr_vec(&mut self, exprs: &[P<ast::Expr>], attrs: &[ast::Attribute]) {
+    fn print_expr_vec(&mut self, exprs: &[P<ast::Expr>]) {
         self.ibox(INDENT_UNIT);
         self.s.word("[");
-        self.print_inner_attributes_inline(attrs);
         self.commasep_exprs(Inconsistent, exprs);
         self.s.word("]");
         self.end();
     }
 
-    fn print_expr_anon_const(&mut self, expr: &ast::AnonConst, attrs: &[ast::Attribute]) {
+    fn print_expr_anon_const(&mut self, expr: &ast::AnonConst) {
         self.ibox(INDENT_UNIT);
         self.s.word("const");
-        self.print_inner_attributes_inline(attrs);
         self.print_expr(&expr.value);
         self.end();
     }
 
-    fn print_expr_repeat(
+    fn print_expr_repeat(&mut self, element: &ast::Expr, count: &ast::AnonConst) {
-        &mut self,
-        element: &ast::Expr,
-        count: &ast::AnonConst,
-        attrs: &[ast::Attribute],
-    ) {
         self.ibox(INDENT_UNIT);
         self.s.word("[");
-        self.print_inner_attributes_inline(attrs);
         self.print_expr(element);
         self.word_space(";");
         self.print_expr(&count.value);
@@ -1710,14 +1717,17 @@ impl<'a> State<'a> {
 
     fn print_expr_struct(
         &mut self,
+        qself: &Option<ast::QSelf>,
         path: &ast::Path,
         fields: &[ast::ExprField],
         rest: &ast::StructRest,
-        attrs: &[ast::Attribute],
     ) {
-        self.print_path(path, true, 0);
+        if let Some(qself) = qself {
+            self.print_qpath(path, qself, true);
+        } else {
+            self.print_path(path, true, 0);
+        }
         self.s.word("{");
-        self.print_inner_attributes_inline(attrs);
         self.commasep_cmnt(
             Consistent,
             fields,
@@ -1752,9 +1762,8 @@ impl<'a> State<'a> {
         self.s.word("}");
     }
 
-    fn print_expr_tup(&mut self, exprs: &[P<ast::Expr>], attrs: &[ast::Attribute]) {
+    fn print_expr_tup(&mut self, exprs: &[P<ast::Expr>]) {
         self.popen();
-        self.print_inner_attributes_inline(attrs);
         self.commasep_exprs(Inconsistent, exprs);
         if exprs.len() == 1 {
             self.s.word(",");
@@ -1865,19 +1874,19 @@ impl<'a> State<'a> {
                 self.print_expr_maybe_paren(expr, parser::PREC_PREFIX);
             }
             ast::ExprKind::Array(ref exprs) => {
-                self.print_expr_vec(&exprs[..], attrs);
+                self.print_expr_vec(exprs);
             }
             ast::ExprKind::ConstBlock(ref anon_const) => {
-                self.print_expr_anon_const(anon_const, attrs);
+                self.print_expr_anon_const(anon_const);
             }
             ast::ExprKind::Repeat(ref element, ref count) => {
-                self.print_expr_repeat(element, count, attrs);
+                self.print_expr_repeat(element, count);
             }
             ast::ExprKind::Struct(ref se) => {
-                self.print_expr_struct(&se.path, &se.fields, &se.rest, attrs);
+                self.print_expr_struct(&se.qself, &se.path, &se.fields, &se.rest);
             }
             ast::ExprKind::Tup(ref exprs) => {
-                self.print_expr_tup(&exprs[..], attrs);
+                self.print_expr_tup(exprs);
             }
             ast::ExprKind::Call(ref func, ref args) => {
                 self.print_expr_call(func, &args[..]);
@@ -2082,117 +2091,8 @@ impl<'a> State<'a> {
                 }
             }
             ast::ExprKind::InlineAsm(ref a) => {
-                enum AsmArg<'a> {
-                    Template(String),
-                    Operand(&'a InlineAsmOperand),
-                    Options(InlineAsmOptions),
-                }
-
-                let mut args = vec![];
-                args.push(AsmArg::Template(InlineAsmTemplatePiece::to_string(&a.template)));
-                args.extend(a.operands.iter().map(|(o, _)| AsmArg::Operand(o)));
-                if !a.options.is_empty() {
-                    args.push(AsmArg::Options(a.options));
-                }
-
                 self.word("asm!");
-                self.popen();
+                self.print_inline_asm(a);
-                self.commasep(Consistent, &args, |s, arg| match arg {
-                    AsmArg::Template(template) => s.print_string(&template, ast::StrStyle::Cooked),
-                    AsmArg::Operand(op) => {
-                        let print_reg_or_class = |s: &mut Self, r: &InlineAsmRegOrRegClass| match r
-                        {
-                            InlineAsmRegOrRegClass::Reg(r) => {
-                                s.print_symbol(*r, ast::StrStyle::Cooked)
-                            }
-                            InlineAsmRegOrRegClass::RegClass(r) => s.word(r.to_string()),
-                        };
-                        match op {
-                            InlineAsmOperand::In { reg, expr } => {
-                                s.word("in");
-                                s.popen();
-                                print_reg_or_class(s, reg);
-                                s.pclose();
-                                s.space();
-                                s.print_expr(expr);
-                            }
-                            InlineAsmOperand::Out { reg, late, expr } => {
-                                s.word(if *late { "lateout" } else { "out" });
-                                s.popen();
-                                print_reg_or_class(s, reg);
-                                s.pclose();
-                                s.space();
-                                match expr {
-                                    Some(expr) => s.print_expr(expr),
-                                    None => s.word("_"),
-                                }
-                            }
-                            InlineAsmOperand::InOut { reg, late, expr } => {
-                                s.word(if *late { "inlateout" } else { "inout" });
-                                s.popen();
-                                print_reg_or_class(s, reg);
-                                s.pclose();
-                                s.space();
-                                s.print_expr(expr);
-                            }
-                            InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
-                                s.word(if *late { "inlateout" } else { "inout" });
-                                s.popen();
-                                print_reg_or_class(s, reg);
-                                s.pclose();
-                                s.space();
-                                s.print_expr(in_expr);
-                                s.space();
-                                s.word_space("=>");
-                                match out_expr {
-                                    Some(out_expr) => s.print_expr(out_expr),
-                                    None => s.word("_"),
-                                }
-                            }
-                            InlineAsmOperand::Const { anon_const } => {
-                                s.word("const");
-                                s.space();
-                                s.print_expr(&anon_const.value);
-                            }
-                            InlineAsmOperand::Sym { expr } => {
-                                s.word("sym");
-                                s.space();
-                                s.print_expr(expr);
-                            }
-                        }
-                    }
-                    AsmArg::Options(opts) => {
-                        s.word("options");
-                        s.popen();
-                        let mut options = vec![];
-                        if opts.contains(InlineAsmOptions::PURE) {
-                            options.push("pure");
-                        }
-                        if opts.contains(InlineAsmOptions::NOMEM) {
-                            options.push("nomem");
-                        }
-                        if opts.contains(InlineAsmOptions::READONLY) {
-                            options.push("readonly");
-                        }
-                        if opts.contains(InlineAsmOptions::PRESERVES_FLAGS) {
-                            options.push("preserves_flags");
-                        }
-                        if opts.contains(InlineAsmOptions::NORETURN) {
-                            options.push("noreturn");
-                        }
-                        if opts.contains(InlineAsmOptions::NOSTACK) {
-                            options.push("nostack");
-                        }
-                        if opts.contains(InlineAsmOptions::ATT_SYNTAX) {
-                            options.push("att_syntax");
-                        }
-                        s.commasep(Inconsistent, &options, |s, &opt| {
-                            s.word(opt);
-                        });
-                        s.pclose();
-                    }
-                });
-                self.pclose();
             }
             ast::ExprKind::LlvmInlineAsm(ref a) => {
                 self.s.word("llvm_asm!");
@@ -2253,7 +2153,6 @@ impl<'a> State<'a> {
             ast::ExprKind::MacCall(ref m) => self.print_mac(m),
             ast::ExprKind::Paren(ref e) => {
                 self.popen();
-                self.print_inner_attributes_inline(attrs);
                 self.print_expr(e);
                 self.pclose();
             }
@@ -2284,6 +2183,116 @@ impl<'a> State<'a> {
         self.end();
     }
 
+    fn print_inline_asm(&mut self, asm: &ast::InlineAsm) {
+        enum AsmArg<'a> {
+            Template(String),
+            Operand(&'a InlineAsmOperand),
+            Options(InlineAsmOptions),
+        }
+
+        let mut args = vec![];
+        args.push(AsmArg::Template(InlineAsmTemplatePiece::to_string(&asm.template)));
+        args.extend(asm.operands.iter().map(|(o, _)| AsmArg::Operand(o)));
+        if !asm.options.is_empty() {
+            args.push(AsmArg::Options(asm.options));
+        }
+
+        self.popen();
+        self.commasep(Consistent, &args, |s, arg| match arg {
+            AsmArg::Template(template) => s.print_string(&template, ast::StrStyle::Cooked),
+            AsmArg::Operand(op) => {
+                let print_reg_or_class = |s: &mut Self, r: &InlineAsmRegOrRegClass| match r {
+                    InlineAsmRegOrRegClass::Reg(r) => s.print_symbol(*r, ast::StrStyle::Cooked),
+                    InlineAsmRegOrRegClass::RegClass(r) => s.word(r.to_string()),
+                };
+                match op {
+                    InlineAsmOperand::In { reg, expr } => {
+                        s.word("in");
+                        s.popen();
+                        print_reg_or_class(s, reg);
+                        s.pclose();
+                        s.space();
+                        s.print_expr(expr);
+                    }
+                    InlineAsmOperand::Out { reg, late, expr } => {
+                        s.word(if *late { "lateout" } else { "out" });
+                        s.popen();
+                        print_reg_or_class(s, reg);
+                        s.pclose();
+                        s.space();
+                        match expr {
+                            Some(expr) => s.print_expr(expr),
+                            None => s.word("_"),
+                        }
+                    }
+                    InlineAsmOperand::InOut { reg, late, expr } => {
+                        s.word(if *late { "inlateout" } else { "inout" });
+                        s.popen();
+                        print_reg_or_class(s, reg);
+                        s.pclose();
+                        s.space();
+                        s.print_expr(expr);
+                    }
+                    InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
+                        s.word(if *late { "inlateout" } else { "inout" });
+                        s.popen();
+                        print_reg_or_class(s, reg);
+                        s.pclose();
+                        s.space();
+                        s.print_expr(in_expr);
+                        s.space();
+                        s.word_space("=>");
+                        match out_expr {
+                            Some(out_expr) => s.print_expr(out_expr),
+                            None => s.word("_"),
+                        }
+                    }
+                    InlineAsmOperand::Const { anon_const } => {
+                        s.word("const");
+                        s.space();
+                        s.print_expr(&anon_const.value);
+                    }
+                    InlineAsmOperand::Sym { expr } => {
+                        s.word("sym");
+                        s.space();
+                        s.print_expr(expr);
+                    }
+                }
+            }
+            AsmArg::Options(opts) => {
+                s.word("options");
+                s.popen();
+                let mut options = vec![];
+                if opts.contains(InlineAsmOptions::PURE) {
+                    options.push("pure");
+                }
+                if opts.contains(InlineAsmOptions::NOMEM) {
+                    options.push("nomem");
+                }
+                if opts.contains(InlineAsmOptions::READONLY) {
+                    options.push("readonly");
+                }
+                if opts.contains(InlineAsmOptions::PRESERVES_FLAGS) {
+                    options.push("preserves_flags");
+                }
+                if opts.contains(InlineAsmOptions::NORETURN) {
+                    options.push("noreturn");
+                }
+                if opts.contains(InlineAsmOptions::NOSTACK) {
+                    options.push("nostack");
+                }
+                if opts.contains(InlineAsmOptions::ATT_SYNTAX) {
+                    options.push("att_syntax");
+                }
+                s.commasep(Inconsistent, &options, |s, &opt| {
+                    s.word(opt);
+                });
+                s.pclose();
+            }
+        });
+        self.pclose();
+    }
+
     crate fn print_local_decl(&mut self, loc: &ast::Local) {
         self.print_pat(&loc.pat);
         if let Some(ref ty) = loc.ty {
@@ -2341,8 +2350,12 @@ impl<'a> State<'a> {
                     self.print_pat(p);
                 }
             }
-            PatKind::TupleStruct(ref path, ref elts) => {
+            PatKind::TupleStruct(ref qself, ref path, ref elts) => {
-                self.print_path(path, true, 0);
+                if let Some(qself) = qself {
+                    self.print_qpath(path, qself, true);
+                } else {
+                    self.print_path(path, true, 0);
+                }
                 self.popen();
                 self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p));
                 self.pclose();
@@ -2356,8 +2369,12 @@ impl<'a> State<'a> {
             PatKind::Path(Some(ref qself), ref path) => {
                 self.print_qpath(path, qself, false);
             }
-            PatKind::Struct(ref path, ref fields, etc) => {
+            PatKind::Struct(ref qself, ref path, ref fields, etc) => {
-                self.print_path(path, true, 0);
+                if let Some(qself) = qself {
+                    self.print_qpath(path, qself, true);
+                } else {
+                    self.print_path(path, true, 0);
+                }
                 self.nbsp();
                 self.word_space("{");
                 self.commasep_cmnt(

compiler/rustc_attr/src/lib.rs

@@ -4,8 +4,6 @@
 //! The goal is to move the definition of `MetaItem` and things that don't need to be in `syntax`
 //! to this crate.
 
-#![cfg_attr(bootstrap, feature(or_patterns))]
-
 #[macro_use]
 extern crate rustc_macros;
 

compiler/rustc_builtin_macros/src/asm.rs

@@ -8,9 +8,11 @@ use rustc_expand::base::{self, *};
 use rustc_parse::parser::Parser;
 use rustc_parse_format as parse;
 use rustc_session::lint;
+use rustc_span::symbol::Ident;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::{InnerSpan, Span};
 use rustc_target::asm::InlineAsmArch;
+use smallvec::smallvec;
 
 struct AsmArgs {
     templates: Vec<P<ast::Expr>>,
@@ -25,6 +27,7 @@ fn parse_args<'a>(
     ecx: &mut ExtCtxt<'a>,
     sp: Span,
     tts: TokenStream,
+    is_global_asm: bool,
 ) -> Result<AsmArgs, DiagnosticBuilder<'a>> {
     let mut p = ecx.new_parser_from_tts(tts);
 
@@ -33,7 +36,7 @@ fn parse_args<'a>(
     }
 
     // Detect use of the legacy llvm_asm! syntax (which used to be called asm!)
-    if p.look_ahead(1, |t| *t == token::Colon || *t == token::ModSep) {
+    if !is_global_asm && p.look_ahead(1, |t| *t == token::Colon || *t == token::ModSep) {
         let mut err =
             ecx.struct_span_err(sp, "the legacy LLVM-style asm! syntax is no longer supported");
         err.note("consider migrating to the new asm! syntax specified in RFC 2873");
@@ -84,7 +87,7 @@ fn parse_args<'a>(
 
         // Parse options
         if p.eat_keyword(sym::options) {
-            parse_options(&mut p, &mut args)?;
+            parse_options(&mut p, &mut args, is_global_asm)?;
             allow_templates = false;
             continue;
         }
@@ -103,19 +106,19 @@ fn parse_args<'a>(
         };
 
         let mut explicit_reg = false;
-        let op = if p.eat_keyword(kw::In) {
+        let op = if !is_global_asm && p.eat_keyword(kw::In) {
             let reg = parse_reg(&mut p, &mut explicit_reg)?;
             let expr = p.parse_expr()?;
             ast::InlineAsmOperand::In { reg, expr }
-        } else if p.eat_keyword(sym::out) {
+        } else if !is_global_asm && p.eat_keyword(sym::out) {
             let reg = parse_reg(&mut p, &mut explicit_reg)?;
             let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) };
             ast::InlineAsmOperand::Out { reg, expr, late: false }
-        } else if p.eat_keyword(sym::lateout) {
+        } else if !is_global_asm && p.eat_keyword(sym::lateout) {
             let reg = parse_reg(&mut p, &mut explicit_reg)?;
             let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) };
             ast::InlineAsmOperand::Out { reg, expr, late: true }
-        } else if p.eat_keyword(sym::inout) {
+        } else if !is_global_asm && p.eat_keyword(sym::inout) {
             let reg = parse_reg(&mut p, &mut explicit_reg)?;
             let expr = p.parse_expr()?;
             if p.eat(&token::FatArrow) {
@@ -125,7 +128,7 @@ fn parse_args<'a>(
             } else {
                 ast::InlineAsmOperand::InOut { reg, expr, late: false }
             }
-        } else if p.eat_keyword(sym::inlateout) {
+        } else if !is_global_asm && p.eat_keyword(sym::inlateout) {
             let reg = parse_reg(&mut p, &mut explicit_reg)?;
             let expr = p.parse_expr()?;
             if p.eat(&token::FatArrow) {
@@ -138,7 +141,7 @@ fn parse_args<'a>(
         } else if p.eat_keyword(kw::Const) {
             let anon_const = p.parse_anon_const_expr()?;
             ast::InlineAsmOperand::Const { anon_const }
-        } else if p.eat_keyword(sym::sym) {
+        } else if !is_global_asm && p.eat_keyword(sym::sym) {
             let expr = p.parse_expr()?;
             match expr.kind {
                 ast::ExprKind::Path(..) => {}
@@ -329,23 +332,27 @@ fn try_set_option<'a>(
     }
 }
 
-fn parse_options<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> Result<(), DiagnosticBuilder<'a>> {
+fn parse_options<'a>(
+    p: &mut Parser<'a>,
+    args: &mut AsmArgs,
+    is_global_asm: bool,
+) -> Result<(), DiagnosticBuilder<'a>> {
     let span_start = p.prev_token.span;
 
     p.expect(&token::OpenDelim(token::DelimToken::Paren))?;
 
     while !p.eat(&token::CloseDelim(token::DelimToken::Paren)) {
-        if p.eat_keyword(sym::pure) {
+        if !is_global_asm && p.eat_keyword(sym::pure) {
             try_set_option(p, args, sym::pure, ast::InlineAsmOptions::PURE);
-        } else if p.eat_keyword(sym::nomem) {
+        } else if !is_global_asm && p.eat_keyword(sym::nomem) {
             try_set_option(p, args, sym::nomem, ast::InlineAsmOptions::NOMEM);
-        } else if p.eat_keyword(sym::readonly) {
+        } else if !is_global_asm && p.eat_keyword(sym::readonly) {
             try_set_option(p, args, sym::readonly, ast::InlineAsmOptions::READONLY);
-        } else if p.eat_keyword(sym::preserves_flags) {
+        } else if !is_global_asm && p.eat_keyword(sym::preserves_flags) {
             try_set_option(p, args, sym::preserves_flags, ast::InlineAsmOptions::PRESERVES_FLAGS);
-        } else if p.eat_keyword(sym::noreturn) {
+        } else if !is_global_asm && p.eat_keyword(sym::noreturn) {
             try_set_option(p, args, sym::noreturn, ast::InlineAsmOptions::NORETURN);
-        } else if p.eat_keyword(sym::nostack) {
+        } else if !is_global_asm && p.eat_keyword(sym::nostack) {
             try_set_option(p, args, sym::nostack, ast::InlineAsmOptions::NOSTACK);
         } else if p.eat_keyword(sym::att_syntax) {
             try_set_option(p, args, sym::att_syntax, ast::InlineAsmOptions::ATT_SYNTAX);
@@ -388,7 +395,7 @@ fn parse_reg<'a>(
     Ok(result)
 }
 
-fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast::Expr> {
+fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, args: AsmArgs) -> Option<ast::InlineAsm> {
     let mut template = vec![];
     // Register operands are implicitly used since they are not allowed to be
     // referenced in the template string.
@@ -415,7 +422,7 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
                     if let Some(mut err) = err {
                         err.emit();
                     }
-                    return DummyResult::raw_expr(sp, true);
+                    return None;
                 }
             };
 
@@ -492,7 +499,7 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
                 e.span_label(err_sp, label);
             }
             e.emit();
-            return DummyResult::raw_expr(sp, true);
+            return None;
         }
 
         curarg = parser.curarg;
@@ -643,15 +650,7 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P<ast
         }
     }
 
-    let inline_asm =
+    Some(ast::InlineAsm { template, operands: args.operands, options: args.options, line_spans })
-        ast::InlineAsm { template, operands: args.operands, options: args.options, line_spans };
-    P(ast::Expr {
-        id: ast::DUMMY_NODE_ID,
-        kind: ast::ExprKind::InlineAsm(P(inline_asm)),
-        span: sp,
-        attrs: ast::AttrVec::new(),
-        tokens: None,
-    })
 }
 
 pub fn expand_asm<'cx>(
@@ -659,8 +658,53 @@ pub fn expand_asm<'cx>(
     sp: Span,
     tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'cx> {
-    match parse_args(ecx, sp, tts) {
+    match parse_args(ecx, sp, tts, false) {
-        Ok(args) => MacEager::expr(expand_preparsed_asm(ecx, sp, args)),
+        Ok(args) => {
+            let expr = if let Some(inline_asm) = expand_preparsed_asm(ecx, args) {
+                P(ast::Expr {
+                    id: ast::DUMMY_NODE_ID,
+                    kind: ast::ExprKind::InlineAsm(P(inline_asm)),
+                    span: sp,
+                    attrs: ast::AttrVec::new(),
+                    tokens: None,
+                })
+            } else {
+                DummyResult::raw_expr(sp, true)
+            };
+            MacEager::expr(expr)
+        }
+        Err(mut err) => {
+            err.emit();
+            DummyResult::any(sp)
+        }
+    }
+}
+
+pub fn expand_global_asm<'cx>(
+    ecx: &'cx mut ExtCtxt<'_>,
+    sp: Span,
+    tts: TokenStream,
+) -> Box<dyn base::MacResult + 'cx> {
+    match parse_args(ecx, sp, tts, true) {
+        Ok(args) => {
+            if let Some(inline_asm) = expand_preparsed_asm(ecx, args) {
+                MacEager::items(smallvec![P(ast::Item {
+                    ident: Ident::invalid(),
+                    attrs: Vec::new(),
+                    id: ast::DUMMY_NODE_ID,
+                    kind: ast::ItemKind::GlobalAsm(inline_asm),
+                    vis: ast::Visibility {
+                        span: sp.shrink_to_lo(),
+                        kind: ast::VisibilityKind::Inherited,
+                        tokens: None,
+                    },
+                    span: ecx.with_def_site_ctxt(sp),
+                    tokens: None,
+                })])
+            } else {
+                DummyResult::any(sp)
+            }
+        }
         Err(mut err) => {
             err.emit();
             DummyResult::any(sp)

compiler/rustc_builtin_macros/src/global_asm.rs

@@ -1,68 +0,0 @@
-//! Module-level assembly support.
-//!
-//! The macro defined here allows you to specify "top-level",
-//! "file-scoped", or "module-level" assembly. These synonyms
-//! all correspond to LLVM's module-level inline assembly instruction.
-//!
-//! For example, `global_asm!("some assembly here")` codegens to
-//! LLVM's `module asm "some assembly here"`. All of LLVM's caveats
-//! therefore apply.
-
-use rustc_ast as ast;
-use rustc_ast::ptr::P;
-use rustc_ast::token;
-use rustc_ast::tokenstream::TokenStream;
-use rustc_errors::DiagnosticBuilder;
-use rustc_expand::base::{self, *};
-use rustc_span::symbol::Ident;
-use rustc_span::Span;
-use smallvec::smallvec;
-
-pub fn expand_global_asm<'cx>(
-    cx: &'cx mut ExtCtxt<'_>,
-    sp: Span,
-    tts: TokenStream,
-) -> Box<dyn base::MacResult + 'cx> {
-    match parse_global_asm(cx, sp, tts) {
-        Ok(Some(global_asm)) => MacEager::items(smallvec![P(ast::Item {
-            ident: Ident::invalid(),
-            attrs: Vec::new(),
-            id: ast::DUMMY_NODE_ID,
-            kind: ast::ItemKind::GlobalAsm(global_asm),
-            vis: ast::Visibility {
-                span: sp.shrink_to_lo(),
-                kind: ast::VisibilityKind::Inherited,
-                tokens: None,
-            },
-            span: cx.with_def_site_ctxt(sp),
-            tokens: None,
-        })]),
-        Ok(None) => DummyResult::any(sp),
-        Err(mut err) => {
-            err.emit();
-            DummyResult::any(sp)
-        }
-    }
-}
-
-fn parse_global_asm<'a>(
-    cx: &mut ExtCtxt<'a>,
-    sp: Span,
-    tts: TokenStream,
-) -> Result<Option<ast::GlobalAsm>, DiagnosticBuilder<'a>> {
-    let mut p = cx.new_parser_from_tts(tts);
-
-    if p.token == token::Eof {
-        let mut err = cx.struct_span_err(sp, "macro requires a string literal as an argument");
-        err.span_label(sp, "string literal required");
-        return Err(err);
-    }
-
-    let expr = p.parse_expr()?;
-    let (asm, _) = match expr_to_string(cx, expr, "inline assembly must be a string literal") {
-        Some((s, st)) => (s, st),
-        None => return Ok(None),
-    };
-
-    Ok(Some(ast::GlobalAsm { asm }))
-}

compiler/rustc_builtin_macros/src/lib.rs

@@ -9,7 +9,6 @@
 #![feature(decl_macro)]
 #![feature(iter_zip)]
 #![feature(nll)]
-#![cfg_attr(bootstrap, feature(or_patterns))]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_quote)]
 #![recursion_limit = "256"]
@@ -20,6 +19,7 @@ use crate::deriving::*;
 
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
 use rustc_expand::proc_macro::BangProcMacro;
+use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::symbol::sym;
 
 mod asm;
@@ -36,7 +36,6 @@ mod env;
 mod format;
 mod format_foreign;
 mod global_allocator;
-mod global_asm;
 mod llvm_asm;
 mod log_syntax;
 mod panic;
@@ -74,7 +73,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         file: source_util::expand_file,
         format_args_nl: format::expand_format_args_nl,
         format_args: format::expand_format_args,
-        global_asm: global_asm::expand_global_asm,
+        global_asm: asm::expand_global_asm,
         include_bytes: source_util::expand_include_bytes,
         include_str: source_util::expand_include_str,
         include: source_util::expand_include,
@@ -114,5 +113,8 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     }
 
     let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
-    register(sym::quote, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })));
+    register(
+        sym::quote,
+        SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client, krate: LOCAL_CRATE })),
+    );
 }

compiler/rustc_builtin_macros/src/source_util.rs

@@ -61,7 +61,9 @@ pub fn expand_file(
 
     let topmost = cx.expansion_cause().unwrap_or(sp);
     let loc = cx.source_map().lookup_char_pos(topmost.lo());
-    base::MacEager::expr(cx.expr_str(topmost, Symbol::intern(&loc.file.name.to_string())))
+    base::MacEager::expr(
+        cx.expr_str(topmost, Symbol::intern(&loc.file.name.prefer_remapped().to_string_lossy())),
+    )
 }
 
 pub fn expand_stringify(

compiler/rustc_builtin_macros/src/test.rs

@@ -254,6 +254,10 @@ pub fn expand_test_or_bench(
                                         "allow_fail",
                                         cx.expr_bool(sp, should_fail(&cx.sess, &item)),
                                     ),
+                                    // compile_fail: true | false
+                                    field("compile_fail", cx.expr_bool(sp, false)),
+                                    // no_run: true | false
+                                    field("no_run", cx.expr_bool(sp, false)),
                                     // should_panic: ...
                                     field(
                                         "should_panic",

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -80,3 +80,10 @@ jobs:
       with:
         name: cg_clif-${{ runner.os }}
         path: cg_clif.tar.xz
+
+    - name: Upload prebuilt cg_clif (cross compile)
+      if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      uses: actions/upload-artifact@v2
+      with:
+        name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
+        path: cg_clif.tar.xz

compiler/rustc_codegen_cranelift/.vscode/settings.json

@@ -1,6 +1,6 @@
 {
     // source for rustc_* is not included in the rust-src component; disable the errors about this
-    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
+    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"],
     "rust-analyzer.assist.importMergeBehavior": "last",
     "rust-analyzer.cargo.runBuildScripts": true,
     "rust-analyzer.linkedProjects": [

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -25,12 +25,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
-[[package]]
-name = "byteorder"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -39,18 +33,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
- "byteorder",
  "cranelift-bforest",
  "cranelift-codegen-meta",
  "cranelift-codegen-shared",
@@ -60,13 +53,12 @@ dependencies = [
  "regalloc",
  "smallvec",
  "target-lexicon",
- "thiserror",
 ]
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -74,18 +66,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -95,15 +87,14 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
  "cranelift-entity",
  "cranelift-module",
  "cranelift-native",
- "errno",
  "libc",
  "log",
  "region",
@@ -113,20 +104,19 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
  "cranelift-entity",
  "log",
- "thiserror",
 ]
 
 [[package]]
 name = "cranelift-native"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "cranelift-codegen",
  "target-lexicon",
@@ -134,8 +124,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.72.0"
+version = "0.74.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0"
+source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,38 +144,11 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "errno"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067"
-dependencies = [
- "gcc",
- "libc",
-]
-
-[[package]]
-name = "gcc"
-version = "0.3.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
-
 [[package]]
 name = "gimli"
-version = "0.23.0"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
+checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189"
 dependencies = [
  "indexmap",
 ]
@@ -242,32 +205,14 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.23.0"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
+checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170"
 dependencies = [
  "crc32fast",
  "indexmap",
 ]
 
-[[package]]
-name = "proc-macro2"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
-dependencies = [
- "unicode-xid",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
-dependencies = [
- "proc-macro2",
-]
-
 [[package]]
 name = "regalloc"
 version = "0.0.31"
@@ -306,6 +251,7 @@ dependencies = [
  "cranelift-frontend",
  "cranelift-jit",
  "cranelift-module",
+ "cranelift-native",
  "cranelift-object",
  "gimli",
  "indexmap",
@@ -321,48 +267,11 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
-[[package]]
-name = "syn"
-version = "1.0.60"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-xid",
-]
-
 [[package]]
 name = "target-lexicon"
-version = "0.11.2"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95"
+checksum = "64ae3b39281e4b14b8123bdbaddd472b7dfe215e444181f2f9d2443c2444f834"
-
-[[package]]
-name = "thiserror"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
 
 [[package]]
 name = "winapi"

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -9,14 +9,15 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] }
+cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", features = ["unwind"] }
-cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
+cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" }
-cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
+cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" }
-cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
+cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" }
-cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
+cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", optional = true }
-target-lexicon = "0.11.0"
+cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" }
-gimli = { version = "0.23.0", default-features = false, features = ["write"]}
+target-lexicon = "0.12.0"
-object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
+gimli = { version = "0.24.0", default-features = false, features = ["write"]}
+object = { version = "0.24.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
@@ -24,10 +25,11 @@ libloading = { version = "0.6.0", optional = true }
 smallvec = "1.6.1"
 
 # Uncomment to use local checkout of cranelift
-#[patch."https://github.com/bytecodealliance/wasmtime/"]
+#[patch."https://github.com/bytecodealliance/wasmtime.git"]
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
 #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" }
 #cranelift-module = { path = "../wasmtime/cranelift/module" }
+#cranelift-native = { path = "../wasmtime/cranelift/native" }
 #cranelift-jit = { path = "../wasmtime/cranelift/jit" }
 #cranelift-object = { path = "../wasmtime/cranelift/object" }
 
@@ -68,13 +70,5 @@ debug = false
 opt-level = 0
 debug = false
 
-[profile.dev.package.syn]
-opt-level = 0
-debug = false
-
-[profile.release.package.syn]
-opt-level = 0
-debug = false
-
 [package.metadata.rust-analyzer]
 rustc_private = true

compiler/rustc_codegen_cranelift/Readme.md

@@ -44,9 +44,10 @@ This will build your project with rustc_codegen_cranelift instead of the usual L
 
 For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md).
 
-## Env vars
+## Configuration
 
-See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift.
+See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all
+configuration options.
 
 ## Not yet supported
 

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -40,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "cc"
-version = "1.0.67"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
 
 [[package]]
 name = "cfg-if"
@@ -56,7 +56,7 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.39"
+version = "0.1.43"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.91"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
+checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -167,6 +167,7 @@ dependencies = [
 name = "panic_abort"
 version = "0.0.0"
 dependencies = [
+ "alloc",
  "cfg-if",
  "compiler_builtins",
  "core",
@@ -194,9 +195,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-demangle"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
+checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -242,10 +243,22 @@ dependencies = [
  "panic_abort",
  "panic_unwind",
  "rustc-demangle",
+ "std_detect",
  "unwind",
  "wasi",
 ]
 
+[[package]]
+name = "std_detect"
+version = "0.1.5"
+dependencies = [
+ "cfg-if",
+ "compiler_builtins",
+ "libc",
+ "rustc-std-workspace-alloc",
+ "rustc-std-workspace-core",
+]
+
 [[package]]
 name = "sysroot"
 version = "0.0.0"

compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh

@@ -32,7 +32,7 @@ popd
 git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned"
 pushd compiler-builtins
 git checkout -- .
-git checkout 0.1.39
+git checkout 0.1.43
 git apply ../../crate_patches/000*-compiler-builtins-*.patch
 popd
 

compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch

@@ -1,35 +0,0 @@
-From 7078cca3cb614e1e82da428380b4e16fc3afef46 Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Thu, 21 Jan 2021 14:46:36 +0100
-Subject: [PATCH] Remove rotate_left from Int
-
----
- src/int/mod.rs | 5 -----
- 1 file changed, 5 deletions(-)
-
-diff --git a/src/int/mod.rs b/src/int/mod.rs
-index 06054c8..3bea17b 100644
---- a/src/int/mod.rs
-+++ b/src/int/mod.rs
-@@ -85,7 +85,6 @@ pub trait Int:
-     fn wrapping_sub(self, other: Self) -> Self;
-     fn wrapping_shl(self, other: u32) -> Self;
-     fn wrapping_shr(self, other: u32) -> Self;
--    fn rotate_left(self, other: u32) -> Self;
-     fn overflowing_add(self, other: Self) -> (Self, bool);
-     fn aborting_div(self, other: Self) -> Self;
-     fn aborting_rem(self, other: Self) -> Self;
-@@ -209,10 +208,6 @@ macro_rules! int_impl_common {
-             <Self>::wrapping_shr(self, other)
-         }
- 
--        fn rotate_left(self, other: u32) -> Self {
--            <Self>::rotate_left(self, other)
--        }
--
-         fn overflowing_add(self, other: Self) -> (Self, bool) {
-             <Self>::overflowing_add(self, other)
-         }
--- 
-2.26.2.7.g19db9cfb68
-

compiler/rustc_codegen_cranelift/docs/env_vars.md

@@ -1,15 +0,0 @@
-# List of env vars recognized by cg_clif
-
-<dl>
-    <dt>CG_CLIF_JIT_ARGS</dt>
-    <dd>When JIT mode is enable pass these arguments to the program.</dd>
-    <dt>CG_CLIF_INCR_CACHE_DISABLED</dt>
-    <dd>Don't cache object files in the incremental cache. Useful during development of cg_clif
-    to make it possible to use incremental mode for all analyses performed by rustc without caching
-    object files when their content should have been changed by a change to cg_clif.</dd>
-    <dt>CG_CLIF_DISPLAY_CG_TIME</dt>
-    <dd>If "1", display the time it took to perform codegen for a crate.</dd>
-    <dt>CG_CLIF_ENABLE_VERIFIER</dt>
-    <dd>Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once
-    before passing the clif ir to Cranelift for compilation.</dt>
-</dl>

compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs

@@ -11,6 +11,22 @@ unsafe extern "C" fn my_puts(s: *const i8) {
     puts(s);
 }
 
+macro_rules! assert {
+    ($e:expr) => {
+        if !$e {
+            panic(stringify!(! $e));
+        }
+    };
+}
+
+macro_rules! assert_eq {
+    ($l:expr, $r: expr) => {
+        if $l != $r {
+            panic(stringify!($l != $r));
+        }
+    }
+}
+
 #[lang = "termination"]
 trait Termination {
     fn report(self) -> i32;
@@ -20,8 +36,9 @@ impl Termination for () {
     fn report(self) -> i32 {
         unsafe {
             NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44
-            *NUM_REF as i32
+            assert_eq!(*NUM_REF as i32, 44);
         }
+        0
     }
 }
 
@@ -82,29 +99,12 @@ fn start<T: Termination + 'static>(
         unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); }
     }
 
-    main().report();
+    main().report() as isize
-    0
 }
 
 static mut NUM: u8 = 6 * 7;
 static NUM_REF: &'static u8 = unsafe { &NUM };
 
-macro_rules! assert {
-    ($e:expr) => {
-        if !$e {
-            panic(stringify!(! $e));
-        }
-    };
-}
-
-macro_rules! assert_eq {
-    ($l:expr, $r: expr) => {
-        if $l != $r {
-            panic(stringify!($l != $r));
-        }
-    }
-}
-
 struct Unique<T: ?Sized> {
     pointer: *const T,
     _marker: PhantomData<T>,
@@ -296,6 +296,11 @@ fn main() {
     unsafe {
         global_asm_test();
     }
+
+    // Both statics have a reference that points to the same anonymous allocation.
+    static REF1: &u8 = &42;
+    static REF2: &u8 = REF1;
+    assert_eq!(*REF1, *REF2);
 }
 
 #[cfg(all(not(jit), target_os = "linux"))]

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -48,6 +48,8 @@ fn main() {
     assert_eq!(2.3f32.copysign(-1.0), -2.3f32);
     println!("{}", 2.3f32.powf(2.0));
 
+    assert_eq!(i64::MAX.checked_mul(2), None);
+
     assert_eq!(-128i8, (-128i8).saturating_sub(1));
     assert_eq!(127i8, 127i8.saturating_sub(-128));
     assert_eq!(-128i8, (-128i8).saturating_add(-128));
@@ -84,6 +86,7 @@ fn main() {
     assert_eq!(houndred_i128 as f64, 100.0);
     assert_eq!(houndred_f32 as i128, 100);
     assert_eq!(houndred_f64 as i128, 100);
+    assert_eq!(1u128.rotate_left(2), 4);
 
     // Test signed 128bit comparing
     let max = usize::MAX as i128;
@@ -184,20 +187,6 @@ unsafe fn test_mm_slli_si128() {
     );
     let r = _mm_slli_si128(a, 16);
     assert_eq_m128i(r, _mm_set1_epi8(0));
-
-    #[rustfmt::skip]
-    let a = _mm_setr_epi8(
-        1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-    );
-    let r = _mm_slli_si128(a, -1);
-    assert_eq_m128i(_mm_set1_epi8(0), r);
-
-    #[rustfmt::skip]
-    let a = _mm_setr_epi8(
-        1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-    );
-    let r = _mm_slli_si128(a, -0x80000000);
-    assert_eq_m128i(r, _mm_set1_epi8(0));
 }
 
 #[cfg(target_arch = "x86_64")]
@@ -292,7 +281,7 @@ unsafe fn test_mm_extract_epi8() {
         8, 9, 10, 11, 12, 13, 14, 15
     );
     let r1 = _mm_extract_epi8(a, 0);
-    let r2 = _mm_extract_epi8(a, 19);
+    let r2 = _mm_extract_epi8(a, 3);
     assert_eq!(r1, 0xFF);
     assert_eq!(r2, 3);
 }

compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch

@@ -39,46 +39,6 @@ index a35897e..f0bf645 100644
  
  pub fn decode_finite<T: DecodableFloat>(v: T) -> Decoded {
      match decode(v).1 {
-diff --git a/library/core/tests/num/int_macros.rs b/library/core/tests/num/int_macros.rs
-index 0475aeb..9558198 100644
---- a/library/core/tests/num/int_macros.rs
-+++ b/library/core/tests/num/int_macros.rs
-@@ -88,6 +88,7 @@ mod tests {
-                 assert_eq!(x.trailing_ones(), 0);
-             }
- 
-+            /*
-             #[test]
-             fn test_rotate() {
-                 assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A);
-@@ -112,6 +113,7 @@ mod tests {
-                 assert_eq!(B.rotate_left(128), B);
-                 assert_eq!(C.rotate_left(128), C);
-             }
-+            */
- 
-             #[test]
-             fn test_swap_bytes() {
-diff --git a/library/core/tests/num/uint_macros.rs b/library/core/tests/num/uint_macros.rs
-index 04ed14f..a6e372e 100644
---- a/library/core/tests/num/uint_macros.rs
-+++ b/library/core/tests/num/uint_macros.rs
-@@ -52,6 +52,7 @@ mod tests {
-                 assert_eq!(x.trailing_ones(), 0);
-             }
- 
-+            /*
-             #[test]
-             fn test_rotate() {
-                 assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A);
-@@ -76,6 +77,7 @@ mod tests {
-                 assert_eq!(B.rotate_left(128), B);
-                 assert_eq!(C.rotate_left(128), C);
-             }
-+            */
- 
-             #[test]
-             fn test_swap_bytes() {
 diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs
 index 1a6be3a..42dbd59 100644
 --- a/library/core/tests/ptr.rs

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2021-03-29"
+channel = "nightly-2021-05-26"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

compiler/rustc_codegen_cranelift/scripts/ext_config.sh

@@ -5,7 +5,7 @@
 set -e
 
 export CG_CLIF_DISPLAY_CG_TIME=1
-export CG_CLIF_INCR_CACHE_DISABLED=1
+export CG_CLIF_DISABLE_INCR_CACHE=1
 
 export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
 export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE}

compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh

@@ -24,18 +24,6 @@ index 5bd1147cad5..10d68a2ff14 100644
 +
  [patch."https://github.com/rust-lang/rust-clippy"]
  clippy_lints = { path = "src/tools/clippy/clippy_lints" }
-diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
-index 23e689fcae7..5f077b765b6 100644
---- a/compiler/rustc_data_structures/Cargo.toml
-+++ b/compiler/rustc_data_structures/Cargo.toml
-@@ -32,7 +32,6 @@ tempfile = "3.0.5"
-
- [dependencies.parking_lot]
- version = "0.11"
--features = ["nightly"]
-
- [target.'cfg(windows)'.dependencies]
- winapi = { version = "0.3", features = ["fileapi", "psapi"] }
 diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
 index d95b5b7f17f..00b6f0e3635 100644
 --- a/library/alloc/Cargo.toml
@@ -44,11 +32,12 @@ index d95b5b7f17f..00b6f0e3635 100644
 
  [dependencies]
  core = { path = "../core" }
--compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
+-compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
++compiler_builtins = { version = "0.1.43", features = ['rustc-dep-of-std', 'no-asm'] }
 
  [dev-dependencies]
  rand = "0.7"
+ rand_xorshift = "0.2"
 EOF
 
 cat > config.toml <<EOF

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -38,6 +38,7 @@ rm src/test/ui/threads-sendsync/task-stderr.rs
 rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
 rm src/test/ui/drop/drop-trait-enum.rs
 rm src/test/ui/numbers-arithmetic/issue-8460.rs
+rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
 
 rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
 rm src/test/ui/init-large-type.rs # same
@@ -47,6 +48,7 @@ rm src/test/ui/issues/issue-51947.rs # same
 rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
 rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
 rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/consts/issue-33537.rs # same
 rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
 rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
 rm src/test/ui/generator/size-moved-locals.rs # same
@@ -56,11 +58,13 @@ rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and n
 
 rm src/test/incremental/hashes/inline_asm.rs # inline asm
 rm src/test/incremental/issue-72386.rs # same
-rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
 rm src/test/incremental/issue-49482.rs # same
 rm src/test/incremental/issue-54059.rs # same
 rm src/test/incremental/lto.rs # requires lto
 
+rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/
+rm -r src/test/run-make/unstable-flag-required # same
+
 rm src/test/pretty/asm.rs # inline asm
 rm src/test/pretty/raw-str-nonexpr.rs # same
 
@@ -68,6 +72,7 @@ rm -r src/test/run-pass-valgrind/unsized-locals
 
 rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
 rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/match/issue-82392.rs # differing error
 rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
 
 rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition

compiler/rustc_codegen_cranelift/scripts/tests.sh

@@ -116,6 +116,7 @@ function extended_sysroot_tests() {
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
     cargo clean
+    export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning
     # Make sure `[codegen mono items] start` doesn't poison the diff
     ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -63,16 +63,16 @@ pub(crate) fn import_function<'tcx>(
     module: &mut dyn Module,
     inst: Instance<'tcx>,
 ) -> FuncId {
-    let name = tcx.symbol_name(inst).name.to_string();
+    let name = tcx.symbol_name(inst).name;
     let sig = get_function_sig(tcx, module.isa().triple(), inst);
-    module.declare_function(&name, Linkage::Import, &sig).unwrap()
+    module.declare_function(name, Linkage::Import, &sig).unwrap()
 }
 
 impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     /// Instance must be monomorphized
     pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
-        let func_id = import_function(self.tcx, self.cx.module, inst);
+        let func_id = import_function(self.tcx, self.module, inst);
-        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
+        let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
 
         if self.clif_comments.enabled() {
             self.add_comment(func_ref, format!("{:?}", inst));
@@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         args: &[Value],
     ) -> &[Value] {
         let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
-        let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
+        let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
-        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
+        let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
         if self.clif_comments.enabled() {
             self.add_comment(call_inst, format!("easy_call {}", name));
@@ -295,7 +295,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
 pub(crate) fn codegen_terminator_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     span: Span,
-    current_block: Block,
     func: &Operand<'tcx>,
     args: &[Operand<'tcx>],
     destination: Option<(Place<'tcx>, BasicBlock)>,
@@ -357,7 +356,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD))
         .unwrap_or(false);
     if is_cold {
-        fx.cold_blocks.insert(current_block);
+        // FIXME Mark current_block block as cold once Cranelift supports it
     }
 
     // Unpack arguments tuple for closures

compiler/rustc_codegen_cranelift/src/allocator.rs

@@ -11,9 +11,9 @@ use rustc_span::symbol::sym;
 pub(crate) fn codegen(
     tcx: TyCtxt<'_>,
     module: &mut impl Module,
-    unwind_context: &mut UnwindContext<'_>,
+    unwind_context: &mut UnwindContext,
 ) -> bool {
-    let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| {
+    let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| {
         use rustc_middle::middle::dependency_format::Linkage;
         list.iter().any(|&linkage| linkage == Linkage::Dynamic)
     });
@@ -29,7 +29,7 @@ pub(crate) fn codegen(
 
 fn codegen_inner(
     module: &mut impl Module,
-    unwind_context: &mut UnwindContext<'_>,
+    unwind_context: &mut UnwindContext,
     kind: AllocatorKind,
 ) {
     let usize_ty = module.target_config().pointer_type();

compiler/rustc_codegen_cranelift/src/archive.rs

@@ -85,8 +85,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
         ));
     }
 
-    fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) {
+    fn add_native_library(&mut self, name: rustc_span::symbol::Symbol, verbatim: bool) {
-        let location = find_library(name, &self.lib_search_paths, self.sess);
+        let location = find_library(name, verbatim, &self.lib_search_paths, self.sess);
         self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| {
             panic!("failed to add native library {}: {}", location.to_string_lossy(), e);
         });
@@ -160,7 +160,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             };
 
             if !self.no_builtin_ranlib {
-                match object::File::parse(&data) {
+                match object::File::parse(&*data) {
                     Ok(object) => {
                         symbol_table.insert(
                             entry_name.as_bytes().to_vec(),
@@ -254,6 +254,15 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             }
         }
     }
+
+    fn inject_dll_import_lib(
+        &mut self,
+        _lib_name: &str,
+        _dll_imports: &[rustc_middle::middle::cstore::DllImport],
+        _tmpdir: &rustc_data_structures::temp_dir::MaybeTempDir,
+    ) {
+        bug!("injecting dll imports is not supported");
+    }
 }
 
 impl<'a> ArArchiveBuilder<'a> {

compiler/rustc_codegen_cranelift/src/backend.rs

@@ -5,23 +5,23 @@ use std::convert::{TryFrom, TryInto};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_session::Session;
 
+use cranelift_codegen::isa::TargetIsa;
 use cranelift_module::FuncId;
+use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
 
 use object::write::*;
 use object::{RelocationEncoding, SectionKind, SymbolFlags};
 
-use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
-
 use gimli::SectionId;
 
 use crate::debuginfo::{DebugReloc, DebugRelocName};
 
 pub(crate) trait WriteMetadata {
-    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, is_like_osx: bool);
+    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>);
 }
 
 impl WriteMetadata for object::write::Object {
-    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) {
+    fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>) {
         let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
         let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
         let offset = self.append_section_data(section_id, &data, 1);
@@ -113,7 +113,7 @@ impl WriteDebugInfo for ObjectProduct {
 }
 
 pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
-    let triple = crate::build_isa(sess).triple().clone();
+    let triple = crate::target_triple(sess);
 
     let binary_format = match triple.binary_format {
         target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
@@ -141,13 +141,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
     metadata_object.write().unwrap()
 }
 
-pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule {
+pub(crate) fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
-    let mut builder = ObjectBuilder::new(
+    let mut builder =
-        crate::build_isa(sess),
+        ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
-        name + ".o",
-        cranelift_module::default_libcall_names(),
-    )
-    .unwrap();
     // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
     // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections
     // can easily double the amount of time necessary to perform linking.

compiler/rustc_codegen_cranelift/src/base.rs

@@ -6,9 +6,14 @@ use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiExt;
 use rustc_target::abi::call::FnAbi;
 
+use crate::constant::ConstantCx;
 use crate::prelude::*;
 
-pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
+pub(crate) fn codegen_fn<'tcx>(
+    cx: &mut crate::CodegenCx<'tcx>,
+    module: &mut dyn Module,
+    instance: Instance<'tcx>,
+) {
     let tcx = cx.tcx;
 
     let _inst_guard =
@@ -18,9 +23,9 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
     let mir = tcx.instance_mir(instance.def);
 
     // Declare function
-    let name = tcx.symbol_name(instance).name.to_string();
+    let symbol_name = tcx.symbol_name(instance);
-    let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
+    let sig = get_function_sig(tcx, module.isa().triple(), instance);
-    let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
+    let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap();
 
     cx.cached_context.clear();
 
@@ -39,15 +44,19 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
         (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
 
     // Make FunctionCx
-    let pointer_type = cx.module.target_config().pointer_type();
+    let pointer_type = module.target_config().pointer_type();
     let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
 
     let mut fx = FunctionCx {
         cx,
+        module,
         tcx,
         pointer_type,
+        vtables: FxHashMap::default(),
+        constants_cx: ConstantCx::new(),
 
         instance,
+        symbol_name,
         mir,
         fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])),
 
@@ -55,7 +64,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
         block_map,
         local_map: IndexVec::with_capacity(mir.local_decls.len()),
         caller_location: None, // set by `codegen_fn_prelude`
-        cold_blocks: EntitySet::new(),
 
         clif_comments,
         source_info_set: indexmap::IndexSet::new(),
@@ -90,7 +98,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
     let mut clif_comments = fx.clif_comments;
     let source_info_set = fx.source_info_set;
     let local_map = fx.local_map;
-    let cold_blocks = fx.cold_blocks;
+
+    fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
 
     // Store function in context
     let context = &mut cx.cached_context;
@@ -101,32 +110,24 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
     // Verify function
     verify_func(tcx, &clif_comments, &context.func);
 
-    // Perform rust specific optimizations
-    tcx.sess.time("optimize clif ir", || {
-        crate::optimize::optimize_function(
-            tcx,
-            instance,
-            context,
-            &cold_blocks,
-            &mut clif_comments,
-        );
-    });
-
     // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
     // instruction, which doesn't have an encoding.
     context.compute_cfg();
     context.compute_domtree();
-    context.eliminate_unreachable_code(cx.module.isa()).unwrap();
+    context.eliminate_unreachable_code(module.isa()).unwrap();
-    context.dce(cx.module.isa()).unwrap();
+    context.dce(module.isa()).unwrap();
     // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
     // invalidate it when it would change.
     context.domtree.clear();
 
-    context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
+    // Perform rust specific optimizations
+    tcx.sess.time("optimize clif ir", || {
+        crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments);
+    });
 
     // Define function
-    let module = &mut cx.module;
     tcx.sess.time("define function", || {
+        context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
         module
             .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap()
@@ -136,7 +137,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
     crate::pretty_clif::write_clif_file(
         tcx,
         "opt",
-        Some(cx.module.isa()),
+        Some(module.isa()),
         instance,
         &context,
         &clif_comments,
@@ -145,13 +146,13 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
     if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
         crate::pretty_clif::write_ir_file(
             tcx,
-            &format!("{}.vcode", tcx.symbol_name(instance).name),
+            || format!("{}.vcode", tcx.symbol_name(instance).name),
             |file| file.write_all(disasm.as_bytes()),
         )
     }
 
     // Define debuginfo for function
-    let isa = cx.module.isa();
+    let isa = module.isa();
     let debug_context = &mut cx.debug_context;
     let unwind_context = &mut cx.unwind_context;
     tcx.sess.time("generate debug info", || {
@@ -159,7 +160,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
             debug_context.define_function(
                 instance,
                 func_id,
-                &name,
+                symbol_name.name,
                 isa,
                 context,
                 &source_info_set,
@@ -205,9 +206,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             // Unwinding after panicking is not supported
             continue;
 
-            // FIXME once unwinding is supported uncomment next lines
+            // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do
-            // // Unwinding is unlikely to happen, so mark cleanup block's as cold.
+            // so for cleanup blocks.
-            // fx.cold_blocks.insert(block);
         }
 
         fx.bcx.ins().nop();
@@ -262,7 +262,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
 
                 let target = fx.get_block(*target);
                 let failure = fx.bcx.create_block();
-                fx.cold_blocks.insert(failure);
+                // FIXME Mark failure block as cold once Cranelift supports it
 
                 if *expected {
                     fx.bcx.ins().brz(cond, failure, &[]);
@@ -355,14 +355,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                 from_hir_call: _,
             } => {
                 fx.tcx.sess.time("codegen call", || {
-                    crate::abi::codegen_terminator_call(
+                    crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination)
-                        fx,
-                        *fn_span,
-                        block,
-                        func,
-                        args,
-                        *destination,
-                    )
                 });
             }
             TerminatorKind::InlineAsm {
@@ -664,7 +657,7 @@ fn codegen_stmt<'tcx>(
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
                         let val = operand.load_scalar(fx);
-                        fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times);
+                        fx.bcx.call_memset(fx.module.target_config(), addr, val, times);
                     } else {
                         let loop_block = fx.bcx.create_block();
                         let loop_block2 = fx.bcx.create_block();
@@ -750,85 +743,15 @@ fn codegen_stmt<'tcx>(
         | StatementKind::AscribeUserType(..) => {}
 
         StatementKind::LlvmInlineAsm(asm) => {
-            use rustc_span::symbol::Symbol;
+            match asm.asm.asm.as_str().trim() {
-            let LlvmInlineAsm { asm, outputs, inputs } = &**asm;
-            let rustc_hir::LlvmInlineAsmInner {
-                asm: asm_code,         // Name
-                outputs: output_names, // Vec<LlvmInlineAsmOutput>
-                inputs: input_names,   // Vec<Name>
-                clobbers,              // Vec<Name>
-                volatile,              // bool
-                alignstack,            // bool
-                dialect: _,
-                asm_str_style: _,
-            } = asm;
-            match asm_code.as_str().trim() {
                 "" => {
                     // Black box
                 }
-                "mov %rbx, %rsi\n                  cpuid\n                  xchg %rbx, %rsi" => {
+                _ => fx.tcx.sess.span_fatal(
-                    assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]);
+                    stmt.source_info.span,
-                    assert_eq!(output_names.len(), 4);
+                    "Legacy `llvm_asm!` inline assembly is not supported. \
-                    for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() {
+                    Try using the new `asm!` instead.",
-                        assert_eq!(&output_names[i].constraint.as_str(), c);
+                ),
-                        assert!(!output_names[i].is_rw);
-                        assert!(!output_names[i].is_indirect);
-                    }
-
-                    assert_eq!(clobbers, &[]);
-
-                    assert!(!volatile);
-                    assert!(!alignstack);
-
-                    assert_eq!(inputs.len(), 2);
-                    let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax
-                    let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx
-
-                    let (eax, ebx, ecx, edx) =
-                        crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf);
-
-                    assert_eq!(outputs.len(), 4);
-                    codegen_place(fx, outputs[0])
-                        .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
-                    codegen_place(fx, outputs[1])
-                        .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
-                    codegen_place(fx, outputs[2])
-                        .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
-                    codegen_place(fx, outputs[3])
-                        .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
-                }
-                "xgetbv" => {
-                    assert_eq!(input_names, &[Symbol::intern("{ecx}")]);
-
-                    assert_eq!(output_names.len(), 2);
-                    for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() {
-                        assert_eq!(&output_names[i].constraint.as_str(), c);
-                        assert!(!output_names[i].is_rw);
-                        assert!(!output_names[i].is_indirect);
-                    }
-
-                    assert_eq!(clobbers, &[]);
-
-                    assert!(!volatile);
-                    assert!(!alignstack);
-
-                    crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported");
-                }
-                // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
-                _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => {
-                    crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
-                }
-                _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => {
-                    crate::trap::trap_unimplemented(fx, "Alloca is not supported");
-                }
-                // Used in sys::windows::abort_internal
-                "int $$0x29" => {
-                    crate::trap::trap_unimplemented(fx, "Windows abort");
-                }
-                _ => fx
-                    .tcx
-                    .sess
-                    .span_fatal(stmt.source_info.span, "Inline assembly is not supported"),
             }
         }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
@@ -844,7 +767,7 @@ fn codegen_stmt<'tcx>(
             let elem_size: u64 = pointee.size.bytes();
             let bytes =
                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
-            fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
+            fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes);
         }
     }
 }
@@ -946,7 +869,7 @@ pub(crate) fn codegen_operand<'tcx>(
 pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) {
     let location = fx.get_caller_location(span).load_scalar(fx);
 
-    let msg_ptr = fx.anonymous_str("assert", msg_str);
+    let msg_ptr = fx.anonymous_str(msg_str);
     let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len, location];
 

compiler/rustc_codegen_cranelift/src/common.rs

@@ -1,8 +1,10 @@
 use rustc_index::vec::IndexVec;
+use rustc_middle::ty::SymbolName;
 use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{Integer, Primitive};
 use rustc_target::spec::{HasTargetSpec, Target};
 
+use crate::constant::ConstantCx;
 use crate::prelude::*;
 
 pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@@ -226,12 +228,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
     }
 }
 
-pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
+pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
-    pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>,
+    pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
+    pub(crate) module: &'m mut dyn Module,
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) pointer_type: Type, // Cached from module
+    pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>,
+    pub(crate) constants_cx: ConstantCx,
 
     pub(crate) instance: Instance<'tcx>,
+    pub(crate) symbol_name: SymbolName<'tcx>,
     pub(crate) mir: &'tcx Body<'tcx>,
     pub(crate) fn_abi: Option<FnAbi<'tcx, Ty<'tcx>>>,
 
@@ -242,9 +248,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
     /// When `#[track_caller]` is used, the implicit caller location is stored in this variable.
     pub(crate) caller_location: Option<CValue<'tcx>>,
 
-    /// See [`crate::optimize::code_layout`] for more information.
-    pub(crate) cold_blocks: EntitySet<Block>,
-
     pub(crate) clif_comments: crate::pretty_clif::CommentWriter,
     pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>,
 
@@ -331,7 +334,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
         let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
         let const_loc = self.tcx.const_caller_location((
-            rustc_span::symbol::Symbol::intern(&caller.file.name.to_string()),
+            rustc_span::symbol::Symbol::intern(
+                &caller.file.name.prefer_remapped().to_string_lossy(),
+            ),
             caller.line as u32,
             caller.col_display as u32 + 1,
         ));
@@ -339,28 +344,18 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     }
 
     pub(crate) fn triple(&self) -> &target_lexicon::Triple {
-        self.cx.module.isa().triple()
+        self.module.isa().triple()
     }
 
-    pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value {
+    pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value {
-        use std::collections::hash_map::DefaultHasher;
-        use std::hash::{Hash, Hasher};
-
-        let mut hasher = DefaultHasher::new();
-        msg.hash(&mut hasher);
-        let msg_hash = hasher.finish();
         let mut data_ctx = DataContext::new();
         data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice());
-        let msg_id = self
+        let msg_id = self.module.declare_anonymous_data(false, false).unwrap();
-            .cx
-            .module
-            .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false)
-            .unwrap();
 
         // Ignore DuplicateDefinition error, as the data will be the same
-        let _ = self.cx.module.define_data(msg_id, &data_ctx);
+        let _ = self.module.define_data(msg_id, &data_ctx);
 
-        let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
+        let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func);
         if self.clif_comments.enabled() {
             self.add_comment(local_msg_id, msg);
         }

compiler/rustc_codegen_cranelift/src/config.rs

@@ -0,0 +1,116 @@
+use std::env;
+use std::str::FromStr;
+
+fn bool_env_var(key: &str) -> bool {
+    env::var(key).as_ref().map(|val| &**val) == Ok("1")
+}
+
+/// The mode to use for compilation.
+#[derive(Copy, Clone, Debug)]
+pub enum CodegenMode {
+    /// AOT compile the crate. This is the default.
+    Aot,
+    /// JIT compile and execute the crate.
+    Jit,
+    /// JIT compile and execute the crate, but only compile functions the first time they are used.
+    JitLazy,
+}
+
+impl FromStr for CodegenMode {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "aot" => Ok(CodegenMode::Aot),
+            "jit" => Ok(CodegenMode::Jit),
+            "jit-lazy" => Ok(CodegenMode::JitLazy),
+            _ => Err(format!("Unknown codegen mode `{}`", s)),
+        }
+    }
+}
+
+/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars.
+#[derive(Clone, Debug)]
+pub struct BackendConfig {
+    /// Should the crate be AOT compiled or JIT executed.
+    ///
+    /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`.
+    pub codegen_mode: CodegenMode,
+
+    /// When JIT mode is enable pass these arguments to the program.
+    ///
+    /// Defaults to the value of `CG_CLIF_JIT_ARGS`.
+    pub jit_args: Vec<String>,
+
+    /// Display the time it took to perform codegen for a crate.
+    ///
+    /// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise.
+    /// Can be set using `-Cllvm-args=display_cg_time=...`.
+    pub display_cg_time: bool,
+
+    /// The register allocator to use.
+    ///
+    /// Defaults to the value of `CG_CLIF_REGALLOC` or `backtracking` otherwise. Can be set using
+    /// `-Cllvm-args=regalloc=...`.
+    pub regalloc: String,
+
+    /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run
+    /// once before passing the clif ir to Cranelift for compilation.
+    ///
+    /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is
+    /// compiled with debug assertions enabled or false otherwise. Can be set using
+    /// `-Cllvm-args=enable_verifier=...`.
+    pub enable_verifier: bool,
+
+    /// Don't cache object files in the incremental cache. Useful during development of cg_clif
+    /// to make it possible to use incremental mode for all analyses performed by rustc without
+    /// caching object files when their content should have been changed by a change to cg_clif.
+    ///
+    /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false
+    /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`.
+    pub disable_incr_cache: bool,
+}
+
+impl Default for BackendConfig {
+    fn default() -> Self {
+        BackendConfig {
+            codegen_mode: CodegenMode::Aot,
+            jit_args: {
+                let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
+                args.split(' ').map(|arg| arg.to_string()).collect()
+            },
+            display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"),
+            regalloc: std::env::var("CG_CLIF_REGALLOC")
+                .unwrap_or_else(|_| "backtracking".to_string()),
+            enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
+            disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),
+        }
+    }
+}
+
+impl BackendConfig {
+    /// Parse the configuration passed in using `-Cllvm-args`.
+    pub fn from_opts(opts: &[String]) -> Result<Self, String> {
+        fn parse_bool(name: &str, value: &str) -> Result<bool, String> {
+            value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name))
+        }
+
+        let mut config = BackendConfig::default();
+        for opt in opts {
+            if let Some((name, value)) = opt.split_once('=') {
+                match name {
+                    "mode" => config.codegen_mode = value.parse()?,
+                    "display_cg_time" => config.display_cg_time = parse_bool(name, value)?,
+                    "regalloc" => config.regalloc = value.to_string(),
+                    "enable_verifier" => config.enable_verifier = parse_bool(name, value)?,
+                    "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?,
+                    _ => return Err(format!("Unknown option `{}`", name)),
+                }
+            } else {
+                return Err(format!("Invalid option `{}`", opt));
+            }
+        }
+
+        Ok(config)
+    }
+}

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -2,11 +2,13 @@
 
 use rustc_span::DUMMY_SP;
 
-use rustc_data_structures::fx::FxHashSet;
+use rustc_ast::Mutability;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::ErrorReported;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{
-    read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer, Scalar,
+    alloc_range, read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc,
+    Scalar,
 };
 use rustc_middle::ty::ConstKind;
 
@@ -15,10 +17,10 @@ use cranelift_module::*;
 
 use crate::prelude::*;
 
-#[derive(Default)]
 pub(crate) struct ConstantCx {
     todo: Vec<TodoItem>,
     done: FxHashSet<DataId>,
+    anon_allocs: FxHashMap<AllocId, DataId>,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -28,6 +30,10 @@ enum TodoItem {
 }
 
 impl ConstantCx {
+    pub(crate) fn new() -> Self {
+        ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() }
+    }
+
     pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) {
         //println!("todo {:?}", self.todo);
         define_all_allocs(tcx, module, &mut self);
@@ -74,8 +80,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
     all_constants_ok
 }
 
-pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
+pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) {
+    let mut constants_cx = ConstantCx::new();
     constants_cx.todo.push(TodoItem::Static(def_id));
+    constants_cx.finalize(tcx, module);
 }
 
 pub(crate) fn codegen_tls_ref<'tcx>(
@@ -83,8 +91,8 @@ pub(crate) fn codegen_tls_ref<'tcx>(
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CValue<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
+    let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
-    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+    let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {
         fx.add_comment(local_data_id, format!("tls {:?}", def_id));
     }
@@ -97,8 +105,8 @@ fn codegen_static_ref<'tcx>(
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CPlace<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
+    let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
-    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+    let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {
         fx.add_comment(local_data_id, format!("{:?}", def_id));
     }
@@ -169,9 +177,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                 let mut alloc = Allocation::from_bytes(
                     std::iter::repeat(0).take(size.bytes_usize()).collect::<Vec<u8>>(),
                     align,
+                    Mutability::Not,
                 );
-                let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used
+                alloc.write_scalar(fx, alloc_range(Size::ZERO, size), x.into()).unwrap();
-                alloc.write_scalar(fx, ptr, x.into(), size).unwrap();
                 let alloc = fx.tcx.intern_const_alloc(alloc);
                 return CValue::by_ref(pointer_for_allocation(fx, alloc), layout);
             }
@@ -182,28 +190,31 @@ pub(crate) fn codegen_const_value<'tcx>(
                     let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id);
                     let base_addr = match alloc_kind {
                         Some(GlobalAlloc::Memory(alloc)) => {
-                            fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
+                            fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
-                            let data_id =
+                            let data_id = data_id_for_alloc_id(
-                                data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
+                                &mut fx.constants_cx,
+                                fx.module,
+                                ptr.alloc_id,
+                                alloc.mutability,
+                            );
                             let local_data_id =
-                                fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+                                fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
                             if fx.clif_comments.enabled() {
                                 fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
                             }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         Some(GlobalAlloc::Function(instance)) => {
-                            let func_id =
+                            let func_id = crate::abi::import_function(fx.tcx, fx.module, instance);
-                                crate::abi::import_function(fx.tcx, fx.cx.module, instance);
                             let local_func_id =
-                                fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
+                                fx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
                             fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
                         }
                         Some(GlobalAlloc::Static(def_id)) => {
                             assert!(fx.tcx.is_static(def_id));
-                            let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
+                            let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
                             let local_data_id =
-                                fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+                                fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
                             if fx.clif_comments.enabled() {
                                 fx.add_comment(local_data_id, format!("{:?}", def_id));
                             }
@@ -243,10 +254,11 @@ fn pointer_for_allocation<'tcx>(
     alloc: &'tcx Allocation,
 ) -> crate::pointer::Pointer {
     let alloc_id = fx.tcx.create_memory_alloc(alloc);
-    fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
+    fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
-    let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
+    let data_id =
+        data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability);
 
-    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+    let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {
         fx.add_comment(local_data_id, format!("{:?}", alloc_id));
     }
@@ -255,18 +267,14 @@ fn pointer_for_allocation<'tcx>(
 }
 
 fn data_id_for_alloc_id(
+    cx: &mut ConstantCx,
     module: &mut dyn Module,
     alloc_id: AllocId,
     mutability: rustc_hir::Mutability,
 ) -> DataId {
-    module
+    *cx.anon_allocs.entry(alloc_id).or_insert_with(|| {
-        .declare_data(
+        module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap()
-            &format!(".L__alloc_{:x}", alloc_id.0),
+    })
-            Linkage::Local,
-            mutability == rustc_hir::Mutability::Mut,
-            false,
-        )
-        .unwrap()
 }
 
 fn data_id_for_static(
@@ -344,7 +352,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                     GlobalAlloc::Memory(alloc) => alloc,
                     GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(),
                 };
-                let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability);
+                let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability);
                 (data_id, alloc, None)
             }
             TodoItem::Static(def_id) => {
@@ -368,8 +376,19 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
         data_ctx.set_align(alloc.align.bytes());
 
         if let Some(section_name) = section_name {
-            // FIXME set correct segment for Mach-O files
+            let (segment_name, section_name) = if tcx.sess.target.is_like_osx {
-            data_ctx.set_segment_section("", &*section_name);
+                if let Some(names) = section_name.split_once(',') {
+                    names
+                } else {
+                    tcx.sess.fatal(&format!(
+                        "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma",
+                        section_name
+                    ));
+                }
+            } else {
+                ("", &*section_name)
+            };
+            data_ctx.set_segment_section(segment_name, section_name);
         }
 
         let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
@@ -397,7 +416,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                 }
                 GlobalAlloc::Memory(target_alloc) => {
                     cx.todo.push(TodoItem::Alloc(reloc));
-                    data_id_for_alloc_id(module, reloc, target_alloc.mutability)
+                    data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability)
                 }
                 GlobalAlloc::Static(def_id) => {
                     if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)
@@ -419,8 +438,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
             data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64);
         }
 
-        // FIXME don't duplicate definitions in lazy jit mode
+        module.define_data(data_id, &data_ctx).unwrap();
-        let _ = module.define_data(data_id, &data_ctx);
         cx.done.insert(data_id);
     }
 
@@ -432,12 +450,89 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
     operand: &Operand<'tcx>,
 ) -> Option<ConstValue<'tcx>> {
     match operand {
-        Operand::Copy(_) | Operand::Move(_) => None,
         Operand::Constant(const_) => match const_.literal {
             ConstantKind::Ty(const_) => {
                 fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value()
             }
             ConstantKind::Val(val, _) => Some(val),
         },
+        // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored
+        // inside a temporary before being passed to the intrinsic requiring the const argument.
+        // This code tries to find a single constant defining definition of the referenced local.
+        Operand::Copy(place) | Operand::Move(place) => {
+            if !place.projection.is_empty() {
+                return None;
+            }
+            let mut computed_const_val = None;
+            for bb_data in fx.mir.basic_blocks() {
+                for stmt in &bb_data.statements {
+                    match &stmt.kind {
+                        StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => {
+                            match &local_and_rvalue.1 {
+                                Rvalue::Cast(CastKind::Misc, operand, ty) => {
+                                    if computed_const_val.is_some() {
+                                        return None; // local assigned twice
+                                    }
+                                    if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) {
+                                        return None;
+                                    }
+                                    let const_val = mir_operand_get_const_val(fx, operand)?;
+                                    if fx.layout_of(ty).size
+                                        != const_val.try_to_scalar_int()?.size()
+                                    {
+                                        return None;
+                                    }
+                                    computed_const_val = Some(const_val);
+                                }
+                                Rvalue::Use(operand) => {
+                                    computed_const_val = mir_operand_get_const_val(fx, operand)
+                                }
+                                _ => return None,
+                            }
+                        }
+                        StatementKind::SetDiscriminant { place: stmt_place, variant_index: _ }
+                            if &**stmt_place == place =>
+                        {
+                            return None;
+                        }
+                        StatementKind::LlvmInlineAsm(_) | StatementKind::CopyNonOverlapping(_) => {
+                            return None;
+                        } // conservative handling
+                        StatementKind::Assign(_)
+                        | StatementKind::FakeRead(_)
+                        | StatementKind::SetDiscriminant { .. }
+                        | StatementKind::StorageLive(_)
+                        | StatementKind::StorageDead(_)
+                        | StatementKind::Retag(_, _)
+                        | StatementKind::AscribeUserType(_, _)
+                        | StatementKind::Coverage(_)
+                        | StatementKind::Nop => {}
+                    }
+                }
+                match &bb_data.terminator().kind {
+                    TerminatorKind::Goto { .. }
+                    | TerminatorKind::SwitchInt { .. }
+                    | TerminatorKind::Resume
+                    | TerminatorKind::Abort
+                    | TerminatorKind::Return
+                    | TerminatorKind::Unreachable
+                    | TerminatorKind::Drop { .. }
+                    | TerminatorKind::Assert { .. } => {}
+                    TerminatorKind::DropAndReplace { .. }
+                    | TerminatorKind::Yield { .. }
+                    | TerminatorKind::GeneratorDrop
+                    | TerminatorKind::FalseEdge { .. }
+                    | TerminatorKind::FalseUnwind { .. } => unreachable!(),
+                    TerminatorKind::InlineAsm { .. } => return None,
+                    TerminatorKind::Call { destination: Some((call_place, _)), .. }
+                        if call_place == place =>
+                    {
+                        return None;
+                    }
+                    TerminatorKind::Call { .. } => {}
+                }
+            }
+            computed_const_val
+        }
     }
 }

compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs

@@ -66,7 +66,7 @@ fn line_program_add_file(
 ) -> FileId {
     match &file.name {
         FileName::Real(path) => {
-            let (dir_path, file_name) = split_path_dir_and_file(path.stable_name());
+            let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available());
             let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
             let file_name = osstr_as_utf8_bytes(file_name);
 
@@ -87,7 +87,7 @@ fn line_program_add_file(
         filename => {
             let dir_id = line_program.default_directory();
             let dummy_file_name = LineString::new(
-                filename.to_string().into_bytes(),
+                filename.prefer_remapped().to_string().into_bytes(),
                 line_program.encoding(),
                 line_strings,
             );

compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs

@@ -64,7 +64,7 @@ impl<'tcx> DebugContext<'tcx> {
         // FIXME: how to get version when building out of tree?
         // Normally this would use option_env!("CFG_VERSION").
         let producer = format!("cg_clif (rustc {})", "unknown version");
-        let comp_dir = tcx.sess.working_dir.0.to_string_lossy().into_owned();
+        let comp_dir = tcx.sess.working_dir.to_string_lossy(false).into_owned();
         let (name, file_info) = match tcx.sess.local_crate_source_file.clone() {
             Some(path) => {
                 let name = path.to_string_lossy().into_owned();

compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs

@@ -5,17 +5,19 @@ use crate::prelude::*;
 use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa};
 
 use gimli::write::{Address, CieId, EhFrame, FrameTable, Section};
+use gimli::RunTimeEndian;
 
 use crate::backend::WriteDebugInfo;
 
-pub(crate) struct UnwindContext<'tcx> {
+pub(crate) struct UnwindContext {
-    tcx: TyCtxt<'tcx>,
+    endian: RunTimeEndian,
     frame_table: FrameTable,
     cie_id: Option<CieId>,
 }
 
-impl<'tcx> UnwindContext<'tcx> {
+impl UnwindContext {
-    pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
+    pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
+        let endian = super::target_endian(tcx);
         let mut frame_table = FrameTable::default();
 
         let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
@@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> {
             None
         };
 
-        UnwindContext { tcx, frame_table, cie_id }
+        UnwindContext { endian, frame_table, cie_id }
     }
 
     pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
@@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> {
     }
 
     pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) {
-        let mut eh_frame =
+        let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian));
-            EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
         self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
 
         if !eh_frame.0.writer.slice().is_empty() {
@@ -70,17 +71,16 @@ impl<'tcx> UnwindContext<'tcx> {
         }
     }
 
-    #[cfg(feature = "jit")]
+    #[cfg(all(feature = "jit", windows))]
-    pub(crate) unsafe fn register_jit(
+    pub(crate) unsafe fn register_jit(self, _jit_module: &cranelift_jit::JITModule) {}
-        self,
+
-        jit_module: &cranelift_jit::JITModule,
+    #[cfg(all(feature = "jit", not(windows)))]
-    ) -> Option<UnwindRegistry> {
+    pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) {
-        let mut eh_frame =
+        let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian));
-            EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
         self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
 
         if eh_frame.0.writer.slice().is_empty() {
-            return None;
+            return;
         }
 
         let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module);
@@ -88,7 +88,10 @@ impl<'tcx> UnwindContext<'tcx> {
         // GCC expects a terminating "empty" length, so write a 0 length at the end of the table.
         eh_frame.extend(&[0, 0, 0, 0]);
 
-        let mut registrations = Vec::new();
+        // FIXME support unregistering unwind tables once cranelift-jit supports deallocating
+        // individual functions
+        #[allow(unused_variables)]
+        let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame);
 
         // =======================================================================
         // Everything after this line up to the end of the file is loosly based on
@@ -96,8 +99,8 @@ impl<'tcx> UnwindContext<'tcx> {
         #[cfg(target_os = "macos")]
         {
             // On macOS, `__register_frame` takes a pointer to a single FDE
-            let start = eh_frame.as_ptr();
+            let start = eh_frame;
-            let end = start.add(eh_frame.len());
+            let end = start.add(eh_frame_len);
             let mut current = start;
 
             // Walk all of the entries in the frame table and register them
@@ -107,7 +110,6 @@ impl<'tcx> UnwindContext<'tcx> {
                 // Skip over the CIE
                 if current != start {
                     __register_frame(current);
-                    registrations.push(current as usize);
                 }
 
                 // Move to the next table entry (+4 because the length itself is not inclusive)
@@ -117,41 +119,12 @@ impl<'tcx> UnwindContext<'tcx> {
         #[cfg(not(target_os = "macos"))]
         {
             // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0
-            let ptr = eh_frame.as_ptr();
+            __register_frame(eh_frame);
-            __register_frame(ptr);
-            registrations.push(ptr as usize);
         }
-
-        Some(UnwindRegistry { _frame_table: eh_frame, registrations })
     }
 }
 
-/// Represents a registry of function unwind information for System V ABI.
-pub(crate) struct UnwindRegistry {
-    _frame_table: Vec<u8>,
-    registrations: Vec<usize>,
-}
-
 extern "C" {
     // libunwind import
     fn __register_frame(fde: *const u8);
-    fn __deregister_frame(fde: *const u8);
-}
-
-impl Drop for UnwindRegistry {
-    fn drop(&mut self) {
-        unsafe {
-            // libgcc stores the frame entries as a linked list in decreasing sort order
-            // based on the PC value of the registered entry.
-            //
-            // As we store the registrations in increasing order, it would be O(N^2) to
-            // deregister in that order.
-            //
-            // To ensure that we just pop off the first element in the list upon every
-            // deregistration, walk our list of registrations backwards.
-            for fde in self.registrations.iter().rev() {
-                __deregister_frame(*fde as *const _);
-            }
-        }
-    }
 }

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -3,6 +3,7 @@
 
 use std::path::PathBuf;
 
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_codegen_ssa::back::linker::LinkerInfo;
 use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -16,12 +17,6 @@ use cranelift_object::ObjectModule;
 
 use crate::{prelude::*, BackendConfig};
 
-fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
-    let module = crate::backend::make_module(tcx.sess, name);
-    assert_eq!(pointer_ty(tcx), module.target_config().pointer_type());
-    module
-}
-
 struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
 
 impl<HCX> HashStable<HCX> for ModuleCodegenResult {
@@ -32,11 +27,12 @@ impl<HCX> HashStable<HCX> for ModuleCodegenResult {
 
 fn emit_module(
     tcx: TyCtxt<'_>,
+    backend_config: &BackendConfig,
     name: String,
     kind: ModuleKind,
     module: ObjectModule,
     debug: Option<DebugContext<'_>>,
-    unwind_context: UnwindContext<'_>,
+    unwind_context: UnwindContext,
 ) -> ModuleCodegenResult {
     let mut product = module.finish();
 
@@ -46,13 +42,13 @@ fn emit_module(
 
     unwind_context.emit(&mut product);
 
-    let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name));
+    let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name));
     let obj = product.object.write().unwrap();
     if let Err(err) = std::fs::write(&tmp_file, obj) {
         tcx.sess.fatal(&format!("error writing object file: {}", err));
     }
 
-    let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() {
+    let work_product = if backend_config.disable_incr_cache {
         None
     } else {
         rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
@@ -77,9 +73,8 @@ fn reuse_workproduct_for_cgu(
     let mut object = None;
     let work_product = cgu.work_product(tcx);
     if let Some(saved_file) = &work_product.saved_file {
-        let obj_out = tcx
+        let obj_out =
-            .output_filenames(LOCAL_CRATE)
+            tcx.output_filenames(()).temp_path(OutputType::Object, Some(&cgu.name().as_str()));
-            .temp_path(OutputType::Object, Some(&cgu.name().as_str()));
         object = Some(obj_out.clone());
         let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, &saved_file);
         if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) {
@@ -110,77 +105,82 @@ fn module_codegen(
     let cgu = tcx.codegen_unit(cgu_name);
     let mono_items = cgu.items_in_deterministic_order(tcx);
 
-    let mut module = new_module(tcx, cgu_name.as_str().to_string());
+    let isa = crate::build_isa(tcx.sess, &backend_config);
+    let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string());
 
     let mut cx = crate::CodegenCx::new(
         tcx,
-        backend_config,
+        backend_config.clone(),
-        &mut module,
+        module.isa(),
         tcx.sess.opts.debuginfo != DebugInfo::None,
     );
-    super::predefine_mono_items(&mut cx, &mono_items);
+    super::predefine_mono_items(tcx, &mut module, &mono_items);
     for (mono_item, _) in mono_items {
         match mono_item {
             MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
+                cx.tcx
-            }
+                    .sess
-            MonoItem::Static(def_id) => {
+                    .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst));
-                crate::constant::codegen_static(&mut cx.constants_cx, def_id)
             }
+            MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id),
             MonoItem::GlobalAsm(item_id) => {
                 let item = cx.tcx.hir().item(item_id);
-                if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
+                if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
-                    cx.global_asm.push_str(&*asm.as_str());
+                    if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
-                    cx.global_asm.push_str("\n\n");
+                        cx.global_asm.push_str("\n.intel_syntax noprefix\n");
+                    } else {
+                        cx.global_asm.push_str("\n.att_syntax\n");
+                    }
+                    for piece in asm.template {
+                        match *piece {
+                            InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s),
+                            InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
+                        }
+                    }
+                    cx.global_asm.push_str("\n.att_syntax\n\n");
                 } else {
                     bug!("Expected GlobalAsm found {:?}", item);
                 }
             }
         }
     }
-    let (global_asm, debug, mut unwind_context) =
+    crate::main_shim::maybe_create_entry_wrapper(
-        tcx.sess.time("finalize CodegenCx", || cx.finalize());
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
-
-    let codegen_result = emit_module(
         tcx,
-        cgu.name().as_str().to_string(),
+        &mut module,
-        ModuleKind::Regular,
+        &mut cx.unwind_context,
-        module,
+        false,
-        debug,
+        cgu.is_primary(),
-        unwind_context,
     );
 
-    codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
+    let debug_context = cx.debug_context;
+    let unwind_context = cx.unwind_context;
+    let codegen_result = tcx.sess.time("write object file", || {
+        emit_module(
+            tcx,
+            &backend_config,
+            cgu.name().as_str().to_string(),
+            ModuleKind::Regular,
+            module,
+            debug_context,
+            unwind_context,
+        )
+    });
+
+    codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm);
 
     codegen_result
 }
 
-pub(super) fn run_aot(
+pub(crate) fn run_aot(
     tcx: TyCtxt<'_>,
     backend_config: BackendConfig,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
-    use rustc_span::symbol::sym;
-
-    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
-    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
-    let windows_subsystem = subsystem.map(|subsystem| {
-        if subsystem != sym::windows && subsystem != sym::console {
-            tcx.sess.fatal(&format!(
-                "invalid windows subsystem `{}`, only \
-                                    `windows` and `console` are allowed",
-                subsystem
-            ));
-        }
-        subsystem.to_string()
-    });
-
     let mut work_products = FxHashMap::default();
 
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
-        tcx.collect_and_partition_mono_items(LOCAL_CRATE).1
+        tcx.collect_and_partition_mono_items(()).1
     } else {
         // If only `--emit metadata` is used, we shouldn't perform any codegen.
         // Also `tcx.collect_and_partition_mono_items` may panic in that case.
@@ -193,14 +193,14 @@ pub(super) fn run_aot(
         }
     }
 
-    let modules = super::time(tcx, "codegen mono items", || {
+    let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
         cgus.iter()
             .map(|cgu| {
                 let cgu_reuse = determine_cgu_reuse(tcx, cgu);
                 tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
 
                 match cgu_reuse {
-                    _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {}
+                    _ if backend_config.disable_incr_cache => {}
                     CguReuse::No => {}
                     CguReuse::PreLto => {
                         return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products);
@@ -212,7 +212,7 @@ pub(super) fn run_aot(
                 let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
                     dep_node,
                     tcx,
-                    (backend_config, cgu.name()),
+                    (backend_config.clone(), cgu.name()),
                     module_codegen,
                     rustc_middle::dep_graph::hash_result,
                 );
@@ -228,7 +228,10 @@ pub(super) fn run_aot(
 
     tcx.sess.abort_if_errors();
 
-    let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
+    let isa = crate::build_isa(tcx.sess, &backend_config);
+    let mut allocator_module =
+        crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string());
+    assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
     let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
     let created_alloc_shim =
         crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
@@ -236,6 +239,7 @@ pub(super) fn run_aot(
     let allocator_module = if created_alloc_shim {
         let ModuleCodegenResult(module, work_product) = emit_module(
             tcx,
+            &backend_config,
             "allocator_shim".to_string(),
             ModuleKind::Allocator,
             allocator_module,
@@ -261,9 +265,8 @@ pub(super) fn run_aot(
                 .as_str()
                 .to_string();
 
-            let tmp_file = tcx
+            let tmp_file =
-                .output_filenames(LOCAL_CRATE)
+                tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
-                .temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
 
             let obj = crate::backend::with_object(tcx.sess, &metadata_cgu_name, |object| {
                 crate::metadata::write_metadata(tcx, object);
@@ -289,13 +292,11 @@ pub(super) fn run_aot(
 
     Box::new((
         CodegenResults {
-            crate_name: tcx.crate_name(LOCAL_CRATE),
             modules,
             allocator_module,
             metadata_module,
             metadata,
-            windows_subsystem,
+            linker_info: LinkerInfo::new(tcx, crate::target_triple(tcx.sess).to_string()),
-            linker_info: LinkerInfo::new(tcx),
             crate_info: CrateInfo::new(tcx),
         },
         work_products,
@@ -338,8 +339,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
         .collect::<Vec<_>>()
         .join("\n");
 
-    let output_object_file =
+    let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name));
-        tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name));
 
     // Assemble `global_asm`
     let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -1,4 +1,4 @@
-//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
+//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object
 //! files.
 
 use std::cell::RefCell;
@@ -8,34 +8,65 @@ use std::os::raw::{c_char, c_int};
 use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
-use rustc_session::config::EntryFnType;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
 use crate::{prelude::*, BackendConfig};
 use crate::{CodegenCx, CodegenMode};
 
-thread_local! {
+struct JitState {
-    pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
+    backend_config: BackendConfig,
-    pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
+    jit_module: JITModule,
 }
 
-pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
+thread_local! {
-    if !tcx.sess.opts.output_types.should_codegen() {
+    static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
-        tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
+}
-    }
 
+fn create_jit_module<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    backend_config: &BackendConfig,
+    hotswap: bool,
+) -> (JITModule, CodegenCx<'tcx>) {
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
-    let mut jit_builder =
+    let isa = crate::build_isa(tcx.sess, backend_config);
-        JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
+    let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
-    jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
+    jit_builder.hotswap(hotswap);
     crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
-    assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
-    let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
+    let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false);
+
+    crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
+    crate::main_shim::maybe_create_entry_wrapper(
+        tcx,
+        &mut jit_module,
+        &mut cx.unwind_context,
+        true,
+        true,
+    );
+
+    (jit_module, cx)
+}
+
+pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
+    if !tcx.sess.opts.output_types.should_codegen() {
+        tcx.sess.fatal("JIT mode doesn't work with `cargo check`");
+    }
+
+    if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) {
+        tcx.sess.fatal("can't jit non-executable crate");
+    }
+
+    let (mut jit_module, mut cx) = create_jit_module(
+        tcx,
+        &backend_config,
+        matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
+    );
+
+    let (_, cgus) = tcx.collect_and_partition_mono_items(());
     let mono_items = cgus
         .iter()
         .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
@@ -44,52 +75,45 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
+    super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
-
+        super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
-    super::time(tcx, "codegen mono items", || {
-        super::predefine_mono_items(&mut cx, &mono_items);
         for (mono_item, _) in mono_items {
             match mono_item {
                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
+                        cx.tcx.sess.time("codegen fn", || {
+                            crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
+                        });
                     }
-                    CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
+                    CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
                 },
                 MonoItem::Static(def_id) => {
-                    crate::constant::codegen_static(&mut cx.constants_cx, def_id);
+                    crate::constant::codegen_static(tcx, &mut jit_module, def_id);
                 }
                 MonoItem::GlobalAsm(item_id) => {
-                    let item = cx.tcx.hir().item(item_id);
+                    let item = tcx.hir().item(item_id);
                     tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode");
                 }
             }
         }
     });
 
-    let (global_asm, _debug, mut unwind_context) =
+    if !cx.global_asm.is_empty() {
-        tcx.sess.time("finalize CodegenCx", || cx.finalize());
-    jit_module.finalize_definitions();
-
-    if !global_asm.is_empty() {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
-
     tcx.sess.abort_if_errors();
 
     jit_module.finalize_definitions();
-    let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
+    unsafe { cx.unwind_context.register_jit(&jit_module) };
 
     println!(
         "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
     );
 
-    let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
     let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
-        .chain(args.split(' '))
+        .chain(backend_config.jit_args.iter().map(|arg| &**arg))
         .map(|arg| CString::new(arg).unwrap())
         .collect::<Vec<_>>();
     let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
@@ -98,61 +122,27 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     // useful as some dynamic linkers use it as a marker to jump over.
     argv.push(std::ptr::null());
 
-    BACKEND_CONFIG.with(|tls_backend_config| {
+    let start_sig = Signature {
-        assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
+        params: vec![
+            AbiParam::new(jit_module.target_config().pointer_type()),
+            AbiParam::new(jit_module.target_config().pointer_type()),
+        ],
+        returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
+        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+    };
+    let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
+    let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
+
+    LAZY_JIT_STATE.with(|lazy_jit_state| {
+        let mut lazy_jit_state = lazy_jit_state.borrow_mut();
+        assert!(lazy_jit_state.is_none());
+        *lazy_jit_state = Some(JitState { backend_config, jit_module });
     });
 
-    let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
+    let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-    let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
+        unsafe { ::std::mem::transmute(finalized_start) };
-
+    let ret = f(args.len() as c_int, argv.as_ptr());
-    match entry_ty {
+    std::process::exit(ret);
-        EntryFnType::Main => {
-            // FIXME set program arguments somehow
-
-            let main_sig = Signature {
-                params: vec![],
-                returns: vec![],
-                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-            };
-            let main_func_id = jit_module
-                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
-                .unwrap();
-            let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
-
-            CURRENT_MODULE.with(|current_module| {
-                assert!(current_module.borrow_mut().replace(jit_module).is_none())
-            });
-
-            let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
-            f();
-            std::process::exit(0);
-        }
-        EntryFnType::Start => {
-            let start_sig = Signature {
-                params: vec![
-                    AbiParam::new(jit_module.target_config().pointer_type()),
-                    AbiParam::new(jit_module.target_config().pointer_type()),
-                ],
-                returns: vec![AbiParam::new(
-                    jit_module.target_config().pointer_type(), /*isize*/
-                )],
-                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-            };
-            let start_func_id = jit_module
-                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
-                .unwrap();
-            let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
-
-            CURRENT_MODULE.with(|current_module| {
-                assert!(current_module.borrow_mut().replace(jit_module).is_none())
-            });
-
-            let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-                unsafe { ::std::mem::transmute(finalized_start) };
-            let ret = f(args.len() as c_int, argv.as_ptr());
-            std::process::exit(ret);
-        }
-    }
 }
 
 #[no_mangle]
@@ -161,24 +151,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
         // lift is used to ensure the correct lifetime for instance.
         let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
 
-        CURRENT_MODULE.with(|jit_module| {
+        LAZY_JIT_STATE.with(|lazy_jit_state| {
-            let mut jit_module = jit_module.borrow_mut();
+            let mut lazy_jit_state = lazy_jit_state.borrow_mut();
-            let jit_module = jit_module.as_mut().unwrap();
+            let lazy_jit_state = lazy_jit_state.as_mut().unwrap();
-            let backend_config =
+            let jit_module = &mut lazy_jit_state.jit_module;
-                BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
+            let backend_config = lazy_jit_state.backend_config.clone();
 
-            let name = tcx.symbol_name(instance).name.to_string();
+            let name = tcx.symbol_name(instance).name;
             let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
-            let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
+            let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap();
             jit_module.prepare_for_function_redefine(func_id).unwrap();
 
-            let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
+            let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
-            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
+            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
 
-            let (global_asm, _debug_context, unwind_context) = cx.finalize();
+            assert!(cx.global_asm.is_empty());
-            assert!(global_asm.is_empty());
             jit_module.finalize_definitions();
-            std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
+            unsafe { cx.unwind_context.register_jit(&jit_module) };
             jit_module.get_finalized_function(func_id)
         })
     })
@@ -190,7 +179,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     let mut dylib_paths = Vec::new();
 
     let crate_info = CrateInfo::new(tcx);
-    let formats = tcx.dependency_formats(LOCAL_CRATE);
+    let formats = tcx.dependency_formats(());
     let data = &formats
         .iter()
         .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
@@ -218,7 +207,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
         use object::{Object, ObjectSymbol};
         let lib = libloading::Library::new(&path).unwrap();
         let obj = std::fs::read(path).unwrap();
-        let obj = object::File::parse(&obj).unwrap();
+        let obj = object::File::parse(&*obj).unwrap();
         imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| {
             let name = symbol.name().unwrap().to_string();
             if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
@@ -248,35 +237,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     imported_symbols
 }
 
-fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
     let tcx = cx.tcx;
 
-    let pointer_type = cx.module.target_config().pointer_type();
+    let pointer_type = module.target_config().pointer_type();
 
-    let name = tcx.symbol_name(inst).name.to_string();
+    let name = tcx.symbol_name(inst).name;
-    let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
+    let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst);
-    let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
+    let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap();
 
     let instance_ptr = Box::into_raw(Box::new(inst));
 
-    let jit_fn = cx
+    let jit_fn = module
-        .module
         .declare_function(
             "__clif_jit_fn",
             Linkage::Import,
             &Signature {
-                call_conv: cx.module.target_config().default_call_conv,
+                call_conv: module.target_config().default_call_conv,
                 params: vec![AbiParam::new(pointer_type)],
                 returns: vec![AbiParam::new(pointer_type)],
             },
         )
         .unwrap();
 
-    let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
+    cx.cached_context.clear();
-    let mut builder_ctx = FunctionBuilderContext::new();
+    let trampoline = &mut cx.cached_context.func;
-    let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
+    trampoline.signature = sig.clone();
 
-    let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
+    let mut builder_ctx = FunctionBuilderContext::new();
+    let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx);
+
+    let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func);
     let sig_ref = trampoline_builder.func.import_signature(sig);
 
     let entry_block = trampoline_builder.create_block();
@@ -291,10 +282,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
     trampoline_builder.ins().return_(&ret_vals);
 
-    cx.module
+    module
         .define_function(
             func_id,
-            &mut Context::for_function(trampoline),
+            &mut cx.cached_context,
             &mut NullTrapSink {},
             &mut NullStackMapSink {},
         )

compiler/rustc_codegen_cranelift/src/driver/mod.rs

@@ -1,63 +1,37 @@
-//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions
+//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and
-//! like JIT executing or writing object files.
+//! performing any further actions like JIT executing or writing object files.
+//!
+//! [`codegen_fn`]: crate::base::codegen_fn
+//! [`codegen_static`]: crate::constant::codegen_static
 
-use std::any::Any;
-
-use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility};
 
 use crate::prelude::*;
-use crate::CodegenMode;
 
-mod aot;
+pub(crate) mod aot;
 #[cfg(feature = "jit")]
-mod jit;
+pub(crate) mod jit;
-
-pub(crate) fn codegen_crate(
-    tcx: TyCtxt<'_>,
-    metadata: EncodedMetadata,
-    need_metadata_module: bool,
-    backend_config: crate::BackendConfig,
-) -> Box<dyn Any> {
-    tcx.sess.abort_if_errors();
-
-    match backend_config.codegen_mode {
-        CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module),
-        CodegenMode::Jit | CodegenMode::JitLazy => {
-            let is_executable =
-                tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable);
-            if !is_executable {
-                tcx.sess.fatal("can't jit non-executable crate");
-            }
-
-            #[cfg(feature = "jit")]
-            let _: ! = jit::run_jit(tcx, backend_config);
-
-            #[cfg(not(feature = "jit"))]
-            tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
-        }
-    }
-}
 
 fn predefine_mono_items<'tcx>(
-    cx: &mut crate::CodegenCx<'_, 'tcx>,
+    tcx: TyCtxt<'tcx>,
+    module: &mut dyn Module,
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
-    cx.tcx.sess.time("predefine functions", || {
+    tcx.sess.time("predefine functions", || {
-        let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
+        let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE);
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
                 MonoItem::Fn(instance) => {
-                    let name = cx.tcx.symbol_name(instance).name.to_string();
+                    let name = tcx.symbol_name(instance).name;
                     let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
-                    let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
+                    let sig = get_function_sig(tcx, module.isa().triple(), instance);
                     let linkage = crate::linkage::get_clif_linkage(
                         mono_item,
                         linkage,
                         visibility,
                         is_compiler_builtins,
                     );
-                    cx.module.declare_function(&name, linkage, &sig).unwrap();
+                    module.declare_function(name, linkage, &sig).unwrap();
                 }
                 MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
             }
@@ -65,8 +39,8 @@ fn predefine_mono_items<'tcx>(
     });
 }
 
-fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
+fn time<R>(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R {
-    if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") {
+    if display {
         println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
         let before = std::time::Instant::now();
         let res = tcx.sess.time(name, f);

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -24,6 +24,76 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         let true_ = fx.bcx.ins().iconst(types::I32, 1);
         fx.bcx.ins().trapnz(true_, TrapCode::User(1));
         return;
+    } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
+        && matches!(
+            template[1],
+            InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ }
+        )
+        && template[2] == InlineAsmTemplatePiece::String("\n".to_string())
+        && template[3] == InlineAsmTemplatePiece::String("cpuid".to_string())
+        && template[4] == InlineAsmTemplatePiece::String("\n".to_string())
+        && template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string())
+        && matches!(
+            template[6],
+            InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ }
+        )
+    {
+        assert_eq!(operands.len(), 4);
+        let (leaf, eax_place) = match operands[1] {
+            InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
+                let reg = expect_reg(reg);
+                assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax));
+                (
+                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
+                    crate::base::codegen_place(fx, out_place.unwrap()),
+                )
+            }
+            _ => unreachable!(),
+        };
+        let ebx_place = match operands[0] {
+            InlineAsmOperand::Out { reg, late: true, place } => {
+                assert_eq!(
+                    reg,
+                    InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
+                        X86InlineAsmRegClass::reg
+                    ))
+                );
+                crate::base::codegen_place(fx, place.unwrap())
+            }
+            _ => unreachable!(),
+        };
+        let (sub_leaf, ecx_place) = match operands[2] {
+            InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
+                let reg = expect_reg(reg);
+                assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx));
+                (
+                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
+                    crate::base::codegen_place(fx, out_place.unwrap()),
+                )
+            }
+            _ => unreachable!(),
+        };
+        let edx_place = match operands[3] {
+            InlineAsmOperand::Out { reg, late: true, place } => {
+                let reg = expect_reg(reg);
+                assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx));
+                crate::base::codegen_place(fx, place.unwrap())
+            }
+            _ => unreachable!(),
+        };
+
+        let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf);
+
+        eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
+        ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
+        ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
+        edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
+        return;
+    } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
+        // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
+        crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
+    } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
+        crate::trap::trap_unimplemented(fx, "Alloca is not supported");
     }
 
     let mut slot_size = Size::from_bytes(0);
@@ -92,8 +162,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 
     let inline_asm_index = fx.inline_asm_index;
     fx.inline_asm_index += 1;
-    let asm_name =
+    let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index);
-        format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index);
 
     let generated_asm = generate_asm_wrapper(
         &asm_name,
@@ -202,7 +271,6 @@ fn call_inline_asm<'tcx>(
     }
 
     let inline_asm_func = fx
-        .cx
         .module
         .declare_function(
             asm_name,
@@ -214,7 +282,7 @@ fn call_inline_asm<'tcx>(
             },
         )
         .unwrap();
-    let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
+    let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {
         fx.add_comment(inline_asm_func, asm_name);
     }

compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs

@@ -8,10 +8,11 @@ use crate::prelude::*;
 pub(crate) fn codegen_cpuid_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     leaf: Value,
-    _subleaf: Value,
+    _sub_leaf: Value,
 ) -> (Value, Value, Value, Value) {
     let leaf_0 = fx.bcx.create_block();
     let leaf_1 = fx.bcx.create_block();
+    let leaf_7 = fx.bcx.create_block();
     let leaf_8000_0000 = fx.bcx.create_block();
     let leaf_8000_0001 = fx.bcx.create_block();
     let unsupported_leaf = fx.bcx.create_block();
@@ -25,6 +26,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
     let mut switch = cranelift_frontend::Switch::new();
     switch.set_entry(0, leaf_0);
     switch.set_entry(1, leaf_1);
+    switch.set_entry(7, leaf_7);
     switch.set_entry(0x8000_0000, leaf_8000_0000);
     switch.set_entry(0x8000_0001, leaf_8000_0001);
     switch.emit(&mut fx.bcx, leaf, unsupported_leaf);
@@ -43,6 +45,11 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
     let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
     fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]);
 
+    fx.bcx.switch_to_block(leaf_7);
+    // This leaf technically has subleaves, but we just return zero for all subleaves.
+    let zero = fx.bcx.ins().iconst(types::I32, 0);
+    fx.bcx.ins().jump(dest, &[zero, zero, zero, zero]);
+
     fx.bcx.switch_to_block(leaf_8000_0000);
     let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0);
     let zero = fx.bcx.ins().iconst(types::I32, 0);

compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

@@ -22,7 +22,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
 
         // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
-        llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) {
+        "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) {
             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
             let lane_ty = fx.clif_type(lane_ty).unwrap();
             assert!(lane_count <= 32);
@@ -51,7 +51,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
             ret.write_cvalue(fx, res);
         };
-        llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
+        "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) {
             let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
             let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
                 0 => FloatCC::Equal,
@@ -81,7 +81,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
             });
         };
-        llvm.x86.sse2.psrli.d, (c a, o imm8) {
+        "llvm.x86.sse2.psrli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
                 let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
@@ -91,7 +91,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
                 CValue::by_val(res_lane, res_lane_layout)
             });
         };
-        llvm.x86.sse2.pslli.d, (c a, o imm8) {
+        "llvm.x86.sse2.pslli.d", (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
                 let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
@@ -101,7 +101,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
                 CValue::by_val(res_lane, res_lane_layout)
             });
         };
-        llvm.x86.sse2.storeu.dq, (v mem_addr, c a) {
+        "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) {
             // FIXME correctly handle the unalignment
             let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
             dest.write_cvalue(fx, a);

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -8,23 +8,25 @@ mod simd;
 pub(crate) use cpuid::codegen_cpuid_call;
 pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
+use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_span::symbol::{kw, sym};
+
 use crate::prelude::*;
 use cranelift_codegen::ir::AtomicRmwOp;
-use rustc_middle::ty::print::with_no_trimmed_paths;
 
 macro intrinsic_pat {
     (_) => {
         _
     },
     ($name:ident) => {
-        stringify!($name)
+        sym::$name
+    },
+    (kw.$name:ident) => {
+        kw::$name
     },
     ($name:literal) => {
-        stringify!($name)
+        $name
     },
-    ($x:ident . $($xs:tt).*) => {
-        concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
-    }
 }
 
 macro intrinsic_arg {
@@ -87,7 +89,7 @@ macro call_intrinsic_match {
     )*) => {
         match $intrinsic {
             $(
-                stringify!($name) => {
+                sym::$name => {
                     assert!($substs.is_noop());
                     if let [$(ref $arg),*] = *$args {
                         let ($($arg,)*) = (
@@ -400,18 +402,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     let def_id = instance.def_id();
     let substs = instance.substs;
 
-    let intrinsic = fx.tcx.item_name(def_id).as_str();
+    let intrinsic = fx.tcx.item_name(def_id);
-    let intrinsic = &intrinsic[..];
 
     let ret = match destination {
         Some((place, _)) => place,
         None => {
             // Insert non returning intrinsics here
             match intrinsic {
-                "abort" => {
+                sym::abort => {
                     trap_abort(fx, "Called intrinsic::abort.");
                 }
-                "transmute" => {
+                sym::transmute => {
                     crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
                 }
                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
@@ -420,7 +421,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         }
     };
 
-    if intrinsic.starts_with("simd_") {
+    if intrinsic.as_str().starts_with("simd_") {
         self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
@@ -470,8 +471,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         sinf64(flt) -> f64 => sin,
         cosf32(flt) -> f32 => cosf,
         cosf64(flt) -> f64 => cos,
-        tanf32(flt) -> f32 => tanf,
-        tanf64(flt) -> f64 => tan,
     }
 
     intrinsic_match! {
@@ -496,12 +495,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 count
             };
 
-            if intrinsic.contains("nonoverlapping") {
+            if intrinsic == sym::copy_nonoverlapping {
                 // FIXME emit_small_memcpy
-                fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
             } else {
                 // FIXME emit_small_memmove
-                fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
             }
         };
         // NOTE: the volatile variants have src and dst swapped
@@ -515,12 +514,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             };
 
             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
-            if intrinsic.contains("nonoverlapping") {
+            if intrinsic == sym::volatile_copy_nonoverlapping_memory {
                 // FIXME emit_small_memcpy
-                fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
             } else {
                 // FIXME emit_small_memmove
-                fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
+                fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
             }
         };
         size_of_val, <T> (c ptr) {
@@ -552,27 +551,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
         };
 
-        _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
+        unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem
+        | unchecked_shl | unchecked_shr, (c x, c y) {
             // FIXME trap on overflow
             let bin_op = match intrinsic {
-                "unchecked_add" => BinOp::Add,
+                sym::unchecked_add => BinOp::Add,
-                "unchecked_sub" => BinOp::Sub,
+                sym::unchecked_sub => BinOp::Sub,
-                "unchecked_div" | "exact_div" => BinOp::Div,
+                sym::unchecked_div | sym::exact_div => BinOp::Div,
-                "unchecked_rem" => BinOp::Rem,
+                sym::unchecked_rem => BinOp::Rem,
-                "unchecked_shl" => BinOp::Shl,
+                sym::unchecked_shl => BinOp::Shl,
-                "unchecked_shr" => BinOp::Shr,
+                sym::unchecked_shr => BinOp::Shr,
-                _ => unreachable!("intrinsic {}", intrinsic),
+                _ => unreachable!(),
             };
             let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
             ret.write_cvalue(fx, res);
         };
-        _ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
+        add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) {
             assert_eq!(x.layout().ty, y.layout().ty);
             let bin_op = match intrinsic {
-                "add_with_overflow" => BinOp::Add,
+                sym::add_with_overflow => BinOp::Add,
-                "sub_with_overflow" => BinOp::Sub,
+                sym::sub_with_overflow => BinOp::Sub,
-                "mul_with_overflow" => BinOp::Mul,
+                sym::mul_with_overflow => BinOp::Mul,
-                _ => unreachable!("intrinsic {}", intrinsic),
+                _ => unreachable!(),
             };
 
             let res = crate::num::codegen_checked_int_binop(
@@ -583,12 +583,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             );
             ret.write_cvalue(fx, res);
         };
-        _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
+        saturating_add | saturating_sub, <T> (c lhs, c rhs) {
             assert_eq!(lhs.layout().ty, rhs.layout().ty);
             let bin_op = match intrinsic {
-                "saturating_add" => BinOp::Add,
+                sym::saturating_add => BinOp::Add,
-                "saturating_sub" => BinOp::Sub,
+                sym::saturating_sub => BinOp::Sub,
-                _ => unreachable!("intrinsic {}", intrinsic),
+                _ => unreachable!(),
             };
 
             let signed = type_sign(T);
@@ -609,15 +609,15 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 
             let val = match (intrinsic, signed) {
-                ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
+                (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val),
-                ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
+                (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
-                ("saturating_add", true) => {
+                (sym::saturating_add, true) => {
                     let rhs = rhs.load_scalar(fx);
                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
                     fx.bcx.ins().select(has_overflow, sat_val, val)
                 }
-                ("saturating_sub", true) => {
+                (sym::saturating_sub, true) => {
                     let rhs = rhs.load_scalar(fx);
                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
@@ -632,11 +632,21 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         };
         rotate_left, <T>(v x, v y) {
             let layout = fx.layout_of(T);
+            let y = if fx.bcx.func.dfg.value_type(y) == types::I128 {
+                fx.bcx.ins().ireduce(types::I64, y)
+            } else {
+                y
+            };
             let res = fx.bcx.ins().rotl(x, y);
             ret.write_cvalue(fx, CValue::by_val(res, layout));
         };
         rotate_right, <T>(v x, v y) {
             let layout = fx.layout_of(T);
+            let y = if fx.bcx.func.dfg.value_type(y) == types::I128 {
+                fx.bcx.ins().ireduce(types::I64, y)
+            } else {
+                y
+            };
             let res = fx.bcx.ins().rotr(x, y);
             ret.write_cvalue(fx, CValue::by_val(res, layout));
         };
@@ -670,7 +680,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let dst_ptr = dst.load_scalar(fx);
             // FIXME make the memset actually volatile when switching to emit_small_memset
             // FIXME use emit_small_memset
-            fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count);
+            fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
         };
         ctlz | ctlz_nonzero, <T> (v arg) {
             // FIXME trap on `ctlz_nonzero` with zero arg.
@@ -806,7 +816,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 return;
             }
 
-            if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
+            if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
                 with_no_trimmed_paths(|| crate::base::codegen_panic(
                     fx,
                     &format!("attempted to zero-initialize type `{}`, which is invalid", T),
@@ -815,7 +825,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 return;
             }
 
-            if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
+            if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
                 with_no_trimmed_paths(|| crate::base::codegen_panic(
                     fx,
                     &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
@@ -827,7 +837,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
 
         volatile_load | unaligned_volatile_load, (c ptr) {
             // Cranelift treats loads as volatile by default
-            // FIXME ignore during stack2reg optimization
             // FIXME correctly handle unaligned_volatile_load
             let inner_layout =
                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
@@ -836,7 +845,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         };
         volatile_store | unaligned_volatile_store, (v ptr, c val) {
             // Cranelift treats stores as volatile by default
-            // FIXME ignore during stack2reg optimization
             // FIXME correctly handle unaligned_volatile_store
             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
             dest.write_cvalue(fx, val);
@@ -878,14 +886,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, caller_location);
         };
 
-        _ if intrinsic.starts_with("atomic_fence"), () {
+        _ if intrinsic.as_str().starts_with("atomic_fence"), () {
             fx.bcx.ins().fence();
         };
-        _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
+        _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () {
             // FIXME use a compiler fence once Cranelift supports it
             fx.bcx.ins().fence();
         };
-        _ if intrinsic.starts_with("atomic_load"), <T> (v ptr) {
+        _ if intrinsic.as_str().starts_with("atomic_load"), <T> (v ptr) {
             validate_atomic_type!(fx, intrinsic, span, T);
             let ty = fx.clif_type(T).unwrap();
 
@@ -894,14 +902,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(T));
             ret.write_cvalue(fx, val);
         };
-        _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
+        _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 
             let val = val.load_scalar(fx);
 
             fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
         };
-        _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) {
+        _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) {
             let layout = new.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -913,7 +921,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
+        _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
             let layout = new.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
 
@@ -927,7 +935,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, ret_val)
         };
 
-        _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) {
+        _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) {
             let layout = amount.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -939,7 +947,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) {
+        _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) {
             let layout = amount.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -951,7 +959,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -963,7 +971,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -975,7 +983,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -989,7 +997,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         };
 
         // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647
-        _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1001,7 +1009,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1013,7 +1021,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1025,7 +1033,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1037,7 +1045,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) {
+        _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) {
             let layout = src.layout();
             validate_atomic_type!(fx, intrinsic, span, layout.ty);
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1071,7 +1079,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, val);
         };
 
-        try, (v f, v data, v _catch_fn) {
+        kw.Try, (v f, v data, v _catch_fn) {
             // FIXME once unwinding is supported, change this to actually catch panics
             let f_sig = fx.bcx.func.import_signature(Signature {
                 call_conv: CallConv::triple_default(fx.triple()),
@@ -1088,11 +1096,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
 
         fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
             let res = crate::num::codegen_float_binop(fx, match intrinsic {
-                "fadd_fast" => BinOp::Add,
+                sym::fadd_fast => BinOp::Add,
-                "fsub_fast" => BinOp::Sub,
+                sym::fsub_fast => BinOp::Sub,
-                "fmul_fast" => BinOp::Mul,
+                sym::fmul_fast => BinOp::Mul,
-                "fdiv_fast" => BinOp::Div,
+                sym::fdiv_fast => BinOp::Div,
-                "frem_fast" => BinOp::Rem,
+                sym::frem_fast => BinOp::Rem,
                 _ => unreachable!(),
             }, x, y);
             ret.write_cvalue(fx, res);

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -13,8 +13,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     let def_id = instance.def_id();
     let substs = instance.substs;
 
-    let intrinsic = fx.tcx.item_name(def_id).as_str();
+    let intrinsic = fx.tcx.item_name(def_id);
-    let intrinsic = &intrinsic[..];
 
     intrinsic_match! {
         fx, intrinsic, substs, args,
@@ -65,10 +64,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
-        _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
+        _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
             validate_simd_type!(fx, intrinsic, span, x.layout().ty);
 
-            let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap();
+            let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap();
 
             assert_eq!(x.layout(), y.layout());
             let layout = x.layout();
@@ -87,9 +86,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
                 let idx_bytes = match idx_const {
                     ConstValue::ByRef { alloc, offset } => {
-                        let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
                         let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
-                        alloc.get_bytes(fx, ptr, size).unwrap()
+                        alloc.get_bytes(fx, alloc_range(offset, size)).unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
                 };

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -1,4 +1,4 @@
-#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
@@ -14,6 +14,7 @@ extern crate rustc_fs_util;
 extern crate rustc_hir;
 extern crate rustc_incremental;
 extern crate rustc_index;
+extern crate rustc_metadata;
 extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
@@ -23,20 +24,19 @@ extern crate rustc_target;
 extern crate rustc_driver;
 
 use std::any::Any;
-use std::str::FromStr;
 
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
 use rustc_errors::ErrorReported;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
-use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
+use rustc_middle::middle::cstore::EncodedMetadata;
-use rustc_middle::ty::query::Providers;
 use rustc_session::config::OutputFilenames;
 use rustc_session::Session;
 
+use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::settings::{self, Configurable};
 
-use crate::constant::ConstantCx;
+pub use crate::config::*;
 use crate::prelude::*;
 
 mod abi;
@@ -49,6 +49,7 @@ mod cast;
 mod codegen_i128;
 mod common;
 mod compiler_builtins;
+mod config;
 mod constant;
 mod debuginfo;
 mod discriminant;
@@ -87,7 +88,6 @@ mod prelude {
 
     pub(crate) use rustc_index::vec::Idx;
 
-    pub(crate) use cranelift_codegen::entity::EntitySet;
     pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
     pub(crate) use cranelift_codegen::ir::function::Function;
     pub(crate) use cranelift_codegen::ir::types;
@@ -119,95 +119,36 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
     }
 }
 
-struct CodegenCx<'m, 'tcx: 'm> {
+/// The codegen context holds any information shared between the codegen of individual functions
+/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
+struct CodegenCx<'tcx> {
     tcx: TyCtxt<'tcx>,
-    module: &'m mut dyn Module,
     global_asm: String,
-    constants_cx: ConstantCx,
     cached_context: Context,
-    vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>,
     debug_context: Option<DebugContext<'tcx>>,
-    unwind_context: UnwindContext<'tcx>,
+    unwind_context: UnwindContext,
 }
 
-impl<'m, 'tcx> CodegenCx<'m, 'tcx> {
+impl<'tcx> CodegenCx<'tcx> {
     fn new(
         tcx: TyCtxt<'tcx>,
         backend_config: BackendConfig,
-        module: &'m mut dyn Module,
+        isa: &dyn TargetIsa,
         debug_info: bool,
     ) -> Self {
-        let unwind_context = UnwindContext::new(
+        assert_eq!(pointer_ty(tcx), isa.pointer_type());
-            tcx,
+
-            module.isa(),
+        let unwind_context =
-            matches!(backend_config.codegen_mode, CodegenMode::Aot),
+            UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
-        );
+        let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None };
-        let debug_context =
-            if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None };
         CodegenCx {
             tcx,
-            module,
             global_asm: String::new(),
-            constants_cx: ConstantCx::default(),
             cached_context: Context::new(),
-            vtables: FxHashMap::default(),
             debug_context,
             unwind_context,
         }
     }
-
-    fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
-        self.constants_cx.finalize(self.tcx, self.module);
-        (self.global_asm, self.debug_context, self.unwind_context)
-    }
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum CodegenMode {
-    Aot,
-    Jit,
-    JitLazy,
-}
-
-impl Default for CodegenMode {
-    fn default() -> Self {
-        CodegenMode::Aot
-    }
-}
-
-impl FromStr for CodegenMode {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "aot" => Ok(CodegenMode::Aot),
-            "jit" => Ok(CodegenMode::Jit),
-            "jit-lazy" => Ok(CodegenMode::JitLazy),
-            _ => Err(format!("Unknown codegen mode `{}`", s)),
-        }
-    }
-}
-
-#[derive(Copy, Clone, Debug, Default)]
-pub struct BackendConfig {
-    pub codegen_mode: CodegenMode,
-}
-
-impl BackendConfig {
-    fn from_opts(opts: &[String]) -> Result<Self, String> {
-        let mut config = BackendConfig::default();
-        for opt in opts {
-            if let Some((name, value)) = opt.split_once('=') {
-                match name {
-                    "mode" => config.codegen_mode = value.parse()?,
-                    _ => return Err(format!("Unknown option `{}`", name)),
-                }
-            } else {
-                return Err(format!("Invalid option `{}`", opt));
-            }
-        }
-        Ok(config)
-    }
 }
 
 pub struct CraneliftCodegenBackend {
@@ -223,30 +164,37 @@ impl CodegenBackend for CraneliftCodegenBackend {
         }
     }
 
-    fn metadata_loader(&self) -> Box<dyn MetadataLoader + Sync> {
-        Box::new(crate::metadata::CraneliftMetadataLoader)
-    }
-
-    fn provide(&self, _providers: &mut Providers) {}
-    fn provide_extern(&self, _providers: &mut Providers) {}
-
     fn target_features(&self, _sess: &Session) -> Vec<rustc_span::Symbol> {
         vec![]
     }
 
+    fn print_version(&self) {
+        println!("Cranelift version: {}", cranelift_codegen::VERSION);
+    }
+
     fn codegen_crate(
         &self,
         tcx: TyCtxt<'_>,
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
-        let config = if let Some(config) = self.config {
+        tcx.sess.abort_if_errors();
+        let config = if let Some(config) = self.config.clone() {
             config
         } else {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                 .unwrap_or_else(|err| tcx.sess.fatal(&err))
         };
-        driver::codegen_crate(tcx, metadata, need_metadata_module, config)
+        match config.codegen_mode {
+            CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
+            CodegenMode::Jit | CodegenMode::JitLazy => {
+                #[cfg(feature = "jit")]
+                let _: ! = driver::jit::run_jit(tcx, config);
+
+                #[cfg(not(feature = "jit"))]
+                tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
+            }
+        }
     }
 
     fn join_codegen(
@@ -267,13 +215,11 @@ impl CodegenBackend for CraneliftCodegenBackend {
     ) -> Result<(), ErrorReported> {
         use rustc_codegen_ssa::back::link::link_binary;
 
-        let target_cpu = crate::target_triple(sess).to_string();
         link_binary::<crate::archive::ArArchiveBuilder<'_>>(
             sess,
             &codegen_results,
             outputs,
-            &codegen_results.crate_name.as_str(),
+            &codegen_results.crate_info.local_crate_name.as_str(),
-            &target_cpu,
         );
 
         Ok(())
@@ -284,7 +230,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
     sess.target.llvm_target.parse().unwrap()
 }
 
-fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
+fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
     use target_lexicon::BinaryFormat;
 
     let target_triple = crate::target_triple(sess);
@@ -292,9 +238,8 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     let mut flags_builder = settings::builder();
     flags_builder.enable("is_pic").unwrap();
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
-    let enable_verifier =
+    let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
-        cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
+    flags_builder.set("enable_verifier", enable_verifier).unwrap();
-    flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
 
     let tls_model = match target_triple.binary_format {
         BinaryFormat::Elf => "elf_gd",
@@ -308,6 +253,8 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
 
     flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
 
+    flags_builder.set("regalloc", &backend_config.regalloc).unwrap();
+
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
@@ -322,10 +269,30 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     let flags = settings::Flags::new(flags_builder);
 
     let variant = cranelift_codegen::isa::BackendVariant::MachInst;
-    let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
+
-    // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
+    let isa_builder = match sess.opts.cg.target_cpu.as_deref() {
-    // is interpreted as `bsr`.
+        Some("native") => {
-    isa_builder.enable("nehalem").unwrap();
+            let builder = cranelift_native::builder_with_options(variant, true).unwrap();
+            builder
+        }
+        Some(value) => {
+            let mut builder =
+                cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
+            if let Err(_) = builder.enable(value) {
+                sess.fatal("The specified target cpu isn't currently supported by Cranelift.");
+            }
+            builder
+        }
+        None => {
+            let mut builder =
+                cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
+            // Don't use "haswell" as the default, as it implies `has_lzcnt`.
+            // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
+            builder.enable("nehalem").unwrap();
+            builder
+        }
+    };
+
     isa_builder.finish(flags)
 }
 

compiler/rustc_codegen_cranelift/src/linkage.rs

@@ -13,6 +13,7 @@ pub(crate) fn get_clif_linkage(
         (RLinkage::External, Visibility::Default) => Linkage::Export,
         (RLinkage::Internal, Visibility::Default) => Linkage::Local,
         (RLinkage::External, Visibility::Hidden) => Linkage::Hidden,
+        (RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible,
         _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility),
     }
 }

compiler/rustc_codegen_cranelift/src/main_shim.rs

@@ -1,6 +1,9 @@
 use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
+use rustc_middle::ty::subst::GenericArg;
+use rustc_middle::ty::AssocKind;
 use rustc_session::config::EntryFnType;
+use rustc_span::symbol::Ident;
 
 use crate::prelude::*;
 
@@ -9,9 +12,11 @@ use crate::prelude::*;
 pub(crate) fn maybe_create_entry_wrapper(
     tcx: TyCtxt<'_>,
     module: &mut impl Module,
-    unwind_context: &mut UnwindContext<'_>,
+    unwind_context: &mut UnwindContext,
+    is_jit: bool,
+    is_primary_cgu: bool,
 ) {
-    let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) {
+    let (main_def_id, is_main_fn) = match tcx.entry_fn(()) {
         Some((def_id, entry_ty)) => (
             def_id,
             match entry_ty {
@@ -22,19 +27,24 @@ pub(crate) fn maybe_create_entry_wrapper(
         None => return,
     };
 
-    let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
+    if main_def_id.is_local() {
-    if module.get_name(&*tcx.symbol_name(instance).name).is_none() {
+        let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
+        if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() {
+            return;
+        }
+    } else if !is_primary_cgu {
         return;
     }
 
-    create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item);
+    create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn);
 
     fn create_entry_fn(
         tcx: TyCtxt<'_>,
         m: &mut impl Module,
-        unwind_context: &mut UnwindContext<'_>,
+        unwind_context: &mut UnwindContext,
         rust_main_def_id: DefId,
-        use_start_lang_item: bool,
+        ignore_lang_start_wrapper: bool,
+        is_main_fn: bool,
     ) {
         let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
         // Given that `main()` has no arguments,
@@ -57,9 +67,9 @@ pub(crate) fn maybe_create_entry_wrapper(
 
         let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
 
-        let main_name = tcx.symbol_name(instance).name.to_string();
+        let main_name = tcx.symbol_name(instance).name;
         let main_sig = get_function_sig(tcx, m.isa().triple(), instance);
-        let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap();
+        let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap();
 
         let mut ctx = Context::new();
         ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig);
@@ -74,7 +84,47 @@ pub(crate) fn maybe_create_entry_wrapper(
 
             let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
 
-            let call_inst = if use_start_lang_item {
+            let result = if is_main_fn && ignore_lang_start_wrapper {
+                // regular main fn, but ignoring #[lang = "start"] as we are running in the jit
+                // FIXME set program arguments somehow
+                let call_inst = bcx.ins().call(main_func_ref, &[]);
+                let call_results = bcx.func.dfg.inst_results(call_inst).to_owned();
+
+                let termination_trait = tcx.require_lang_item(LangItem::Termination, None);
+                let report = tcx
+                    .associated_items(termination_trait)
+                    .find_by_name_and_kind(
+                        tcx,
+                        Ident::from_str("report"),
+                        AssocKind::Fn,
+                        termination_trait,
+                    )
+                    .unwrap();
+                let report = Instance::resolve(
+                    tcx,
+                    ParamEnv::reveal_all(),
+                    report.def_id,
+                    tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()),
+                )
+                .unwrap()
+                .unwrap();
+
+                let report_name = tcx.symbol_name(report).name;
+                let report_sig = get_function_sig(tcx, m.isa().triple(), report);
+                let report_func_id =
+                    m.declare_function(report_name, Linkage::Import, &report_sig).unwrap();
+                let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func);
+
+                // FIXME do proper abi handling instead of expecting the pass mode to be identical
+                // for returns and arguments.
+                let report_call_inst = bcx.ins().call(report_func_ref, &call_results);
+                let res = bcx.func.dfg.inst_results(report_call_inst)[0];
+                match m.target_config().pointer_type() {
+                    types::I32 => res,
+                    types::I64 => bcx.ins().sextend(types::I64, res),
+                    _ => unimplemented!("16bit systems are not yet supported"),
+                }
+            } else if is_main_fn {
                 let start_def_id = tcx.require_lang_item(LangItem::Start, None);
                 let start_instance = Instance::resolve(
                     tcx,
@@ -90,13 +140,14 @@ pub(crate) fn maybe_create_entry_wrapper(
                 let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
 
                 let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
-                bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv])
+                let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]);
+                bcx.inst_results(call_inst)[0]
             } else {
                 // using user-defined start fn
-                bcx.ins().call(main_func_ref, &[arg_argc, arg_argv])
+                let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]);
+                bcx.inst_results(call_inst)[0]
             };
 
-            let result = bcx.inst_results(call_inst)[0];
             bcx.ins().return_(&[result]);
             bcx.seal_all_blocks();
             bcx.finalize();

compiler/rustc_codegen_cranelift/src/metadata.rs

@@ -1,111 +1,20 @@
-//! Reading and writing of the rustc metadata for rlibs and dylibs
+//! Writing of the rustc metadata for dylibs
 
-use std::fs::File;
-use std::path::Path;
-
-use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::memmap::Mmap;
-use rustc_data_structures::owning_ref::OwningRef;
-use rustc_data_structures::rustc_erase_owner;
-use rustc_data_structures::sync::MetadataRef;
-use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config;
-use rustc_target::spec::Target;
 
 use crate::backend::WriteMetadata;
 
-pub(crate) struct CraneliftMetadataLoader;
-
-fn load_metadata_with(
-    path: &Path,
-    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
-) -> Result<MetadataRef, String> {
-    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
-    let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?;
-    let metadata = OwningRef::new(data).try_map(f)?;
-    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
-}
-
-impl MetadataLoader for CraneliftMetadataLoader {
-    fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        load_metadata_with(path, |data| {
-            let archive = object::read::archive::ArchiveFile::parse(&*data)
-                .map_err(|e| format!("{:?}", e))?;
-
-            for entry_result in archive.members() {
-                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
-                if entry.name() == METADATA_FILENAME.as_bytes() {
-                    return Ok(entry.data());
-                }
-            }
-
-            Err("couldn't find metadata entry".to_string())
-        })
-    }
-
-    fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        use object::{Object, ObjectSection};
-
-        load_metadata_with(path, |data| {
-            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
-            file.section_by_name(".rustc")
-                .ok_or("no .rustc section")?
-                .data()
-                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
-        })
-    }
-}
-
 // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112
-pub(crate) fn write_metadata<P: WriteMetadata>(
+pub(crate) fn write_metadata<O: WriteMetadata>(tcx: TyCtxt<'_>, object: &mut O) {
-    tcx: TyCtxt<'_>,
-    product: &mut P,
-) -> EncodedMetadata {
     use snap::write::FrameEncoder;
     use std::io::Write;
 
-    #[derive(PartialEq, Eq, PartialOrd, Ord)]
-    enum MetadataKind {
-        None,
-        Uncompressed,
-        Compressed,
-    }
-
-    let kind = tcx
-        .sess
-        .crate_types()
-        .iter()
-        .map(|ty| match *ty {
-            config::CrateType::Executable
-            | config::CrateType::Staticlib
-            | config::CrateType::Cdylib => MetadataKind::None,
-
-            config::CrateType::Rlib => MetadataKind::Uncompressed,
-
-            config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed,
-        })
-        .max()
-        .unwrap_or(MetadataKind::None);
-
-    if kind == MetadataKind::None {
-        return EncodedMetadata::new();
-    }
-
     let metadata = tcx.encode_metadata();
-    if kind == MetadataKind::Uncompressed {
+    let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
-        return metadata;
-    }
-
-    assert!(kind == MetadataKind::Compressed);
-    let mut compressed = tcx.metadata_encoding_version();
     FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
 
-    product.add_rustc_section(
+    object.add_rustc_section(
         rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx),
         compressed,
-        tcx.sess.target.is_like_osx,
     );
-
-    metadata
 }

compiler/rustc_codegen_cranelift/src/num.rs

@@ -271,14 +271,17 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
                         let val_hi = fx.bcx.ins().umulhi(lhs, rhs);
                         fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0)
                     } else {
+                        // Based on LLVM's instruction sequence for compiling
+                        // a.checked_mul(b).is_some() to riscv64gc:
+                        // mulh    a2, a0, a1
+                        // mul     a0, a0, a1
+                        // srai    a0, a0, 63
+                        // xor     a0, a0, a2
+                        // snez    a0, a0
                         let val_hi = fx.bcx.ins().smulhi(lhs, rhs);
-                        let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0);
+                        let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1));
-                        let not_all_ones = fx.bcx.ins().icmp_imm(
+                        let xor = fx.bcx.ins().bxor(val_hi, val_sign);
-                            IntCC::NotEqual,
+                        fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0)
-                            val_hi,
-                            u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64,
-                        );
-                        fx.bcx.ins().band(not_all_zero, not_all_ones)
                     };
                     (val, has_overflow)
                 }

compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs

@@ -1,34 +0,0 @@
-//! This optimization moves cold code to the end of the function.
-//!
-//! Some code is executed much less often than other code. For example panicking or the
-//! landingpads for unwinding. By moving this cold code to the end of the function the average
-//! amount of jumps is reduced and the code locality is improved.
-//!
-//! # Undefined behaviour
-//!
-//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself.
-
-use crate::prelude::*;
-
-pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block>) {
-    // FIXME Move the block in place instead of remove and append once
-    // bytecodealliance/cranelift#1339 is implemented.
-
-    let mut block_insts = FxHashMap::default();
-    for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
-        let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>();
-        for &inst in &insts {
-            ctx.func.layout.remove_inst(inst);
-        }
-        block_insts.insert(block, insts);
-        ctx.func.layout.remove_block(block);
-    }
-
-    // And then append them at the back again.
-    for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
-        ctx.func.layout.append_block(block);
-        for inst in block_insts.remove(&block).unwrap() {
-            ctx.func.layout.append_inst(inst, block);
-        }
-    }
-}

compiler/rustc_codegen_cranelift/src/optimize/mod.rs

@@ -2,29 +2,16 @@
 
 use crate::prelude::*;
 
-mod code_layout;
 pub(crate) mod peephole;
-mod stack2reg;
 
 pub(crate) fn optimize_function<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
     ctx: &mut Context,
-    cold_blocks: &EntitySet<Block>,
     clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
-    // The code_layout optimization is very cheap.
+    // FIXME classify optimizations over opt levels once we have more
-    self::code_layout::optimize_function(ctx, cold_blocks);
 
-    if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No {
+    crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments);
-        return; // FIXME classify optimizations over opt levels
-    }
-
-    // FIXME(#1142) stack2reg miscompiles lewton
-    if false {
-        self::stack2reg::optimize_function(ctx, clif_comments);
-    }
-
-    crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments);
     crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
 }

compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs

@@ -1,486 +0,0 @@
-//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible.
-//!
-//! # Undefined behaviour
-//!
-//! This optimization is based on the assumption that stack slots which don't have their address
-//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the
-//! function which has the stack slots. This optimization also assumes that stack slot accesses
-//! are never out of bounds. If these assumptions are not correct, then this optimization may remove
-//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value
-//! being loaded by a `stack_load`.
-
-use std::collections::BTreeMap;
-use std::fmt;
-use std::ops::Not;
-
-use rustc_data_structures::fx::FxHashSet;
-
-use cranelift_codegen::cursor::{Cursor, FuncCursor};
-use cranelift_codegen::ir::immediates::Offset32;
-use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef};
-
-use crate::prelude::*;
-
-/// Workaround for `StackSlot` not implementing `Ord`.
-#[derive(Copy, Clone, PartialEq, Eq)]
-struct OrdStackSlot(StackSlot);
-
-impl fmt::Debug for OrdStackSlot {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", self.0)
-    }
-}
-
-impl PartialOrd for OrdStackSlot {
-    fn partial_cmp(&self, rhs: &Self) -> Option<std::cmp::Ordering> {
-        self.0.as_u32().partial_cmp(&rhs.0.as_u32())
-    }
-}
-
-impl Ord for OrdStackSlot {
-    fn cmp(&self, rhs: &Self) -> std::cmp::Ordering {
-        self.0.as_u32().cmp(&rhs.0.as_u32())
-    }
-}
-
-#[derive(Debug, Default)]
-struct StackSlotUsage {
-    stack_addr: FxHashSet<Inst>,
-    stack_load: FxHashSet<Inst>,
-    stack_store: FxHashSet<Inst>,
-}
-
-impl StackSlotUsage {
-    fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec<Inst> {
-        self.stack_store
-            .iter()
-            .cloned()
-            .filter(|&store| {
-                match spatial_overlap(&ctx.func, store, load) {
-                    SpatialOverlap::No => false, // Can never be the source of the loaded value.
-                    SpatialOverlap::Partial | SpatialOverlap::Full => true,
-                }
-            })
-            .filter(|&store| {
-                match temporal_order(ctx, store, load) {
-                    TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value.
-                    TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true,
-                }
-            })
-            .collect::<Vec<Inst>>()
-    }
-
-    fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec<Inst> {
-        self.stack_load
-            .iter()
-            .cloned()
-            .filter(|&load| {
-                match spatial_overlap(&ctx.func, store, load) {
-                    SpatialOverlap::No => false, // Can never be the source of the loaded value.
-                    SpatialOverlap::Partial | SpatialOverlap::Full => true,
-                }
-            })
-            .filter(|&load| {
-                match temporal_order(ctx, store, load) {
-                    TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value.
-                    TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true,
-                }
-            })
-            .collect::<Vec<Inst>>()
-    }
-
-    fn remove_unused_stack_addr(func: &mut Function, inst: Inst) {
-        func.dfg.detach_results(inst);
-        func.dfg.replace(inst).nop();
-    }
-
-    fn remove_unused_load(func: &mut Function, load: Inst) {
-        func.dfg.detach_results(load);
-        func.dfg.replace(load).nop();
-    }
-
-    fn remove_dead_store(&mut self, func: &mut Function, store: Inst) {
-        func.dfg.replace(store).nop();
-        self.stack_store.remove(&store);
-    }
-
-    fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) {
-        let loaded_value = func.dfg.inst_results(load)[0];
-        let loaded_type = func.dfg.value_type(loaded_value);
-
-        if func.dfg.value_type(value) == loaded_type {
-            func.dfg.detach_results(load);
-            func.dfg.replace(load).nop();
-            func.dfg.change_to_alias(loaded_value, value);
-        } else {
-            func.dfg.replace(load).bitcast(loaded_type, value);
-        }
-
-        self.stack_load.remove(&load);
-    }
-}
-
-struct OptimizeContext<'a> {
-    ctx: &'a mut Context,
-    stack_slot_usage_map: BTreeMap<OrdStackSlot, StackSlotUsage>,
-}
-
-impl<'a> OptimizeContext<'a> {
-    fn for_context(ctx: &'a mut Context) -> Self {
-        ctx.flowgraph(); // Compute cfg and domtree.
-
-        // Record all stack_addr, stack_load and stack_store instructions.
-        let mut stack_slot_usage_map = BTreeMap::<OrdStackSlot, StackSlotUsage>::new();
-
-        let mut cursor = FuncCursor::new(&mut ctx.func);
-        while let Some(_block) = cursor.next_block() {
-            while let Some(inst) = cursor.next_inst() {
-                match cursor.func.dfg[inst] {
-                    InstructionData::StackLoad {
-                        opcode: Opcode::StackAddr,
-                        stack_slot,
-                        offset: _,
-                    } => {
-                        stack_slot_usage_map
-                            .entry(OrdStackSlot(stack_slot))
-                            .or_insert_with(StackSlotUsage::default)
-                            .stack_addr
-                            .insert(inst);
-                    }
-                    InstructionData::StackLoad {
-                        opcode: Opcode::StackLoad,
-                        stack_slot,
-                        offset: _,
-                    } => {
-                        stack_slot_usage_map
-                            .entry(OrdStackSlot(stack_slot))
-                            .or_insert_with(StackSlotUsage::default)
-                            .stack_load
-                            .insert(inst);
-                    }
-                    InstructionData::StackStore {
-                        opcode: Opcode::StackStore,
-                        arg: _,
-                        stack_slot,
-                        offset: _,
-                    } => {
-                        stack_slot_usage_map
-                            .entry(OrdStackSlot(stack_slot))
-                            .or_insert_with(StackSlotUsage::default)
-                            .stack_store
-                            .insert(inst);
-                    }
-                    _ => {}
-                }
-            }
-        }
-
-        OptimizeContext { ctx, stack_slot_usage_map }
-    }
-}
-
-pub(super) fn optimize_function(
-    ctx: &mut Context,
-    clif_comments: &mut crate::pretty_clif::CommentWriter,
-) {
-    combine_stack_addr_with_load_store(&mut ctx.func);
-
-    let mut opt_ctx = OptimizeContext::for_context(ctx);
-
-    // FIXME Repeat following instructions until fixpoint.
-
-    remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
-
-    if clif_comments.enabled() {
-        for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
-            clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
-        }
-    }
-
-    for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() {
-        if users.stack_addr.is_empty().not() {
-            // Stack addr leaked; there may be unknown loads and stores.
-            // FIXME use stacked borrows to optimize
-            continue;
-        }
-
-        for load in users.stack_load.clone().into_iter() {
-            let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
-
-            if clif_comments.enabled() {
-                for &store in &potential_stores {
-                    clif_comments.add_comment(
-                        load,
-                        format!(
-                            "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
-                            opt_ctx.ctx.func.dfg.display_inst(store, None),
-                            opt_ctx.ctx.func.dfg.display_inst(load, None),
-                            spatial_overlap(&opt_ctx.ctx.func, store, load),
-                            temporal_order(&opt_ctx.ctx, store, load),
-                        ),
-                    );
-                }
-            }
-
-            match *potential_stores {
-                [] => {
-                    if clif_comments.enabled() {
-                        clif_comments
-                            .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
-                    }
-                }
-                [store]
-                    if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
-                        && temporal_order(&opt_ctx.ctx, store, load)
-                            == TemporalOrder::DefinitivelyBefore =>
-                {
-                    // Only one store could have been the origin of the value.
-                    let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
-
-                    if clif_comments.enabled() {
-                        clif_comments.add_comment(
-                            load,
-                            format!("Store to load forward {} -> {}", store, load),
-                        );
-                    }
-
-                    users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
-                }
-                _ => {} // FIXME implement this
-            }
-        }
-
-        for store in users.stack_store.clone().into_iter() {
-            let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
-
-            if clif_comments.enabled() {
-                for &load in &potential_loads {
-                    clif_comments.add_comment(
-                        store,
-                        format!(
-                            "Potential load from store {} <- {} ({:?}, {:?})",
-                            opt_ctx.ctx.func.dfg.display_inst(load, None),
-                            opt_ctx.ctx.func.dfg.display_inst(store, None),
-                            spatial_overlap(&opt_ctx.ctx.func, store, load),
-                            temporal_order(&opt_ctx.ctx, store, load),
-                        ),
-                    );
-                }
-            }
-
-            if potential_loads.is_empty() {
-                // Never loaded; can safely remove all stores and the stack slot.
-                // FIXME also remove stores when there is always a next store before a load.
-
-                if clif_comments.enabled() {
-                    clif_comments.add_comment(
-                        store,
-                        format!(
-                            "Remove dead stack store {} of {}",
-                            opt_ctx.ctx.func.dfg.display_inst(store, None),
-                            stack_slot.0
-                        ),
-                    );
-                }
-
-                users.remove_dead_store(&mut opt_ctx.ctx.func, store);
-            }
-        }
-
-        if users.stack_store.is_empty() && users.stack_load.is_empty() {
-            opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0;
-        }
-    }
-}
-
-fn combine_stack_addr_with_load_store(func: &mut Function) {
-    // Turn load and store into stack_load and stack_store when possible.
-    let mut cursor = FuncCursor::new(func);
-    while let Some(_block) = cursor.next_block() {
-        while let Some(inst) = cursor.next_inst() {
-            match cursor.func.dfg[inst] {
-                InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => {
-                    if cursor.func.dfg.ctrl_typevar(inst) == types::I128
-                        || cursor.func.dfg.ctrl_typevar(inst).is_vector()
-                    {
-                        continue; // WORKAROUD: stack_load.i128 not yet implemented
-                    }
-                    if let Some((stack_slot, stack_addr_offset)) =
-                        try_get_stack_slot_and_offset_for_addr(cursor.func, addr)
-                    {
-                        if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into())
-                        {
-                            let ty = cursor.func.dfg.ctrl_typevar(inst);
-                            cursor.func.dfg.replace(inst).stack_load(
-                                ty,
-                                stack_slot,
-                                combined_offset,
-                            );
-                        }
-                    }
-                }
-                InstructionData::Store {
-                    opcode: Opcode::Store,
-                    args: [value, addr],
-                    flags: _,
-                    offset,
-                } => {
-                    if cursor.func.dfg.ctrl_typevar(inst) == types::I128
-                        || cursor.func.dfg.ctrl_typevar(inst).is_vector()
-                    {
-                        continue; // WORKAROUND: stack_store.i128 not yet implemented
-                    }
-                    if let Some((stack_slot, stack_addr_offset)) =
-                        try_get_stack_slot_and_offset_for_addr(cursor.func, addr)
-                    {
-                        if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into())
-                        {
-                            cursor.func.dfg.replace(inst).stack_store(
-                                value,
-                                stack_slot,
-                                combined_offset,
-                            );
-                        }
-                    }
-                }
-                _ => {}
-            }
-        }
-    }
-}
-
-fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) {
-    // FIXME incrementally rebuild on each call?
-    let mut stack_addr_load_insts_users = FxHashMap::<Inst, FxHashSet<Inst>>::default();
-
-    let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func);
-    while let Some(_block) = cursor.next_block() {
-        while let Some(inst) = cursor.next_inst() {
-            for &arg in cursor.func.dfg.inst_args(inst) {
-                if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) {
-                    match cursor.func.dfg[arg_origin].opcode() {
-                        Opcode::StackAddr | Opcode::StackLoad => {
-                            stack_addr_load_insts_users
-                                .entry(arg_origin)
-                                .or_insert_with(FxHashSet::default)
-                                .insert(inst);
-                        }
-                        _ => {}
-                    }
-                }
-            }
-        }
-    }
-
-    #[cfg(debug_assertions)]
-    for inst in stack_addr_load_insts_users.keys() {
-        let mut is_recorded_stack_addr_or_stack_load = false;
-        for stack_slot_users in opt_ctx.stack_slot_usage_map.values() {
-            is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst)
-                || stack_slot_users.stack_load.contains(inst);
-        }
-        assert!(is_recorded_stack_addr_or_stack_load);
-    }
-
-    // Replace all unused stack_addr and stack_load instructions with nop.
-    let mut func = &mut opt_ctx.ctx.func;
-
-    for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() {
-        stack_slot_users
-            .stack_addr
-            .drain_filter(|inst| {
-                stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
-            })
-            .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst));
-
-        stack_slot_users
-            .stack_load
-            .drain_filter(|inst| {
-                stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
-            })
-            .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst));
-    }
-}
-
-fn try_get_stack_slot_and_offset_for_addr(
-    func: &Function,
-    addr: Value,
-) -> Option<(StackSlot, Offset32)> {
-    if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) {
-        if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } =
-            func.dfg[addr_inst]
-        {
-            return Some((stack_slot, offset));
-        }
-    }
-    None
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-enum SpatialOverlap {
-    No,
-    Partial,
-    Full,
-}
-
-fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
-    fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) {
-        match func.dfg[inst] {
-            InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset }
-            | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset }
-            | InstructionData::StackStore {
-                opcode: Opcode::StackStore,
-                stack_slot,
-                offset,
-                arg: _,
-            } => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()),
-            _ => unreachable!("{:?}", func.dfg[inst]),
-        }
-    }
-
-    debug_assert_ne!(src, dest);
-
-    let (src_ss, src_offset, src_size) = inst_info(func, src);
-    let (dest_ss, dest_offset, dest_size) = inst_info(func, dest);
-
-    if src_ss != dest_ss {
-        return SpatialOverlap::No;
-    }
-
-    if src_offset == dest_offset && src_size == dest_size {
-        return SpatialOverlap::Full;
-    }
-
-    let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into();
-    let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into();
-    if src_end <= dest_offset.into() || dest_end <= src_offset.into() {
-        return SpatialOverlap::No;
-    }
-
-    SpatialOverlap::Partial
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-enum TemporalOrder {
-    /// `src` will never be executed before `dest`.
-    NeverBefore,
-
-    /// `src` may be executed before `dest`.
-    MaybeBefore,
-
-    /// `src` will always be executed before `dest`.
-    /// There may still be other instructions in between.
-    DefinitivelyBefore,
-}
-
-fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder {
-    debug_assert_ne!(src, dest);
-
-    if ctx.domtree.dominates(src, dest, &ctx.func.layout) {
-        TemporalOrder::DefinitivelyBefore
-    } else if ctx.domtree.dominates(src, dest, &ctx.func.layout) {
-        TemporalOrder::NeverBefore
-    } else {
-        TemporalOrder::MaybeBefore
-    }
-}

compiler/rustc_codegen_cranelift/src/pretty_clif.rs

@@ -207,14 +207,14 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
 
 pub(crate) fn write_ir_file(
     tcx: TyCtxt<'_>,
-    name: &str,
+    name: impl FnOnce() -> String,
     write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
     if !should_write_ir(tcx) {
         return;
     }
 
-    let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif");
+    let clif_output_dir = tcx.output_filenames(()).with_extension("clif");
 
     match std::fs::create_dir(&clif_output_dir) {
         Ok(()) => {}
@@ -222,7 +222,7 @@ pub(crate) fn write_ir_file(
         res @ Err(_) => res.unwrap(),
     }
 
-    let clif_file_name = clif_output_dir.join(name);
+    let clif_file_name = clif_output_dir.join(name());
 
     let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
     if let Err(err) = res {
@@ -238,30 +238,31 @@ pub(crate) fn write_clif_file<'tcx>(
     context: &cranelift_codegen::Context,
     mut clif_comments: &CommentWriter,
 ) {
-    write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| {
+    write_ir_file(
-        let value_ranges =
+        tcx,
-            isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
+        || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
+        |file| {
+            let value_ranges = isa
+                .map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
 
-        let mut clif = String::new();
+            let mut clif = String::new();
-        cranelift_codegen::write::decorate_function(
+            cranelift_codegen::write::decorate_function(
-            &mut clif_comments,
+                &mut clif_comments,
-            &mut clif,
+                &mut clif,
-            &context.func,
+                &context.func,
-            &DisplayFunctionAnnotations {
+                &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() },
-                isa: Some(&*crate::build_isa(tcx.sess)),
+            )
-                value_ranges: value_ranges.as_ref(),
+            .unwrap();
-            },
-        )
-        .unwrap();
 
-        writeln!(file, "test compile")?;
+            writeln!(file, "test compile")?;
-        writeln!(file, "set is_pic")?;
+            writeln!(file, "set is_pic")?;
-        writeln!(file, "set enable_simd")?;
+            writeln!(file, "set enable_simd")?;
-        writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
+            writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
-        writeln!(file)?;
+            writeln!(file)?;
-        file.write_all(clif.as_bytes())?;
+            file.write_all(clif.as_bytes())?;
-        Ok(())
+            Ok(())
-    });
+        },
+    );
 }
 
 impl fmt::Debug for FunctionCx<'_, '_, '_> {

compiler/rustc_codegen_cranelift/src/toolchain.rs

@@ -2,9 +2,8 @@
 
 use std::path::PathBuf;
 
-use rustc_middle::bug;
+use rustc_codegen_ssa::back::link::linker_and_flavor;
 use rustc_session::Session;
-use rustc_target::spec::LinkerFlavor;
 
 /// Tries to infer the path of a binary for the target toolchain from the linker name.
 pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf {
@@ -30,89 +29,3 @@ pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf {
 
     linker
 }
-
-// Adapted from https://github.com/rust-lang/rust/blob/5db778affee7c6600c8e7a177c48282dab3f6292/src/librustc_codegen_ssa/back/link.rs#L848-L931
-fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
-    fn infer_from(
-        sess: &Session,
-        linker: Option<PathBuf>,
-        flavor: Option<LinkerFlavor>,
-    ) -> Option<(PathBuf, LinkerFlavor)> {
-        match (linker, flavor) {
-            (Some(linker), Some(flavor)) => Some((linker, flavor)),
-            // only the linker flavor is known; use the default linker for the selected flavor
-            (None, Some(flavor)) => Some((
-                PathBuf::from(match flavor {
-                    LinkerFlavor::Em => {
-                        if cfg!(windows) {
-                            "emcc.bat"
-                        } else {
-                            "emcc"
-                        }
-                    }
-                    LinkerFlavor::Gcc => {
-                        if cfg!(any(target_os = "solaris", target_os = "illumos")) {
-                            // On historical Solaris systems, "cc" may have
-                            // been Sun Studio, which is not flag-compatible
-                            // with "gcc".  This history casts a long shadow,
-                            // and many modern illumos distributions today
-                            // ship GCC as "gcc" without also making it
-                            // available as "cc".
-                            "gcc"
-                        } else {
-                            "cc"
-                        }
-                    }
-                    LinkerFlavor::Ld => "ld",
-                    LinkerFlavor::Msvc => "link.exe",
-                    LinkerFlavor::Lld(_) => "lld",
-                    LinkerFlavor::PtxLinker => "rust-ptx-linker",
-                }),
-                flavor,
-            )),
-            (Some(linker), None) => {
-                let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| {
-                    sess.fatal("couldn't extract file stem from specified linker")
-                });
-
-                let flavor = if stem == "emcc" {
-                    LinkerFlavor::Em
-                } else if stem == "gcc"
-                    || stem.ends_with("-gcc")
-                    || stem == "clang"
-                    || stem.ends_with("-clang")
-                {
-                    LinkerFlavor::Gcc
-                } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") {
-                    LinkerFlavor::Ld
-                } else if stem == "link" || stem == "lld-link" {
-                    LinkerFlavor::Msvc
-                } else if stem == "lld" || stem == "rust-lld" {
-                    LinkerFlavor::Lld(sess.target.lld_flavor)
-                } else {
-                    // fall back to the value in the target spec
-                    sess.target.linker_flavor
-                };
-
-                Some((linker, flavor))
-            }
-            (None, None) => None,
-        }
-    }
-
-    // linker and linker flavor specified via command line have precedence over what the target
-    // specification specifies
-    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
-        return ret;
-    }
-
-    if let Some(ret) = infer_from(
-        sess,
-        sess.target.linker.clone().map(PathBuf::from),
-        Some(sess.target.linker_flavor),
-    ) {
-        return ret;
-    }
-
-    bug!("Not enough information provided to determine how to invoke the linker");
-}

compiler/rustc_codegen_cranelift/src/trap.rs

@@ -4,7 +4,6 @@ use crate::prelude::*;
 
 fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
     let puts = fx
-        .cx
         .module
         .declare_function(
             "puts",
@@ -16,14 +15,13 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
             },
         )
         .unwrap();
-    let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
+    let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {
         fx.add_comment(puts, "puts");
     }
 
-    let symbol_name = fx.tcx.symbol_name(fx.instance);
+    let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg);
-    let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg);
+    let msg_ptr = fx.anonymous_str(&real_msg);
-    let msg_ptr = fx.anonymous_str("trap", &real_msg);
     fx.bcx.ins().call(puts, &[msg_ptr]);
 }
 

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -554,13 +554,14 @@ impl<'tcx> CPlace<'tcx> {
                 let src_align = src_layout.align.abi.bytes() as u8;
                 let dst_align = dst_layout.align.abi.bytes() as u8;
                 fx.bcx.emit_small_memory_copy(
-                    fx.cx.module.target_config(),
+                    fx.module.target_config(),
                     to_addr,
                     from_addr,
                     size,
                     dst_align,
                     src_align,
                     true,
+                    MemFlags::trusted(),
                 );
             }
             CValueInner::ByRef(_, Some(_)) => todo!(),

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -72,15 +72,15 @@ pub(crate) fn get_vtable<'tcx>(
     layout: TyAndLayout<'tcx>,
     trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
 ) -> Value {
-    let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) {
+    let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) {
         *data_id
     } else {
         let data_id = build_vtable(fx, layout, trait_ref);
-        fx.cx.vtables.insert((layout.ty, trait_ref), data_id);
+        fx.vtables.insert((layout.ty, trait_ref), data_id);
         data_id
     };
 
-    let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+    let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
 }
 
@@ -94,7 +94,7 @@ fn build_vtable<'tcx>(
 
     let drop_in_place_fn = import_function(
         tcx,
-        fx.cx.module,
+        fx.module,
         Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx),
     );
 
@@ -111,7 +111,7 @@ fn build_vtable<'tcx>(
         opt_mth.map(|(def_id, substs)| {
             import_function(
                 tcx,
-                fx.cx.module,
+                fx.module,
                 Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs)
                     .unwrap()
                     .polymorphize(fx.tcx),
@@ -132,34 +132,16 @@ fn build_vtable<'tcx>(
 
     for (i, component) in components.into_iter().enumerate() {
         if let Some(func_id) = component {
-            let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx);
+            let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx);
             data_ctx.write_function_addr((i * usize_size) as u32, func_ref);
         }
     }
 
     data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes());
 
-    let data_id = fx
+    let data_id = fx.module.declare_anonymous_data(false, false).unwrap();
-        .cx
-        .module
-        .declare_data(
-            &format!(
-                "__vtable.{}.for.{:?}.{}",
-                trait_ref
-                    .as_ref()
-                    .map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into())
-                    .unwrap_or(std::borrow::Cow::Borrowed("???")),
-                layout.ty,
-                fx.cx.vtables.len(),
-            ),
-            Linkage::Local,
-            false,
-            false,
-        )
-        .unwrap();
 
-    // FIXME don't duplicate definitions in lazy jit mode
+    fx.module.define_data(data_id, &data_ctx).unwrap();
-    let _ = fx.cx.module.define_data(data_id, &data_ctx);
 
     data_id
 }

compiler/rustc_codegen_llvm/Cargo.toml

@@ -27,6 +27,7 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_incremental = { path = "../rustc_incremental" }
 rustc_index = { path = "../rustc_index" }
 rustc_llvm = { path = "../rustc_llvm" }
+rustc_metadata = { path = "../rustc_metadata" }
 rustc_session = { path = "../rustc_session" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_target = { path = "../rustc_target" }

compiler/rustc_codegen_llvm/src/abi.rs

@@ -1,6 +1,7 @@
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 use crate::llvm::{self, AttributePlace};
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -51,9 +52,15 @@ pub trait ArgAttributesExt {
 }
 
 fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool {
-    // While #84958 has been fixed, mutable noalias is not enabled by default
+    // LLVM prior to version 12 has known miscompiles in the presence of
-    // in Rust 1.53 out of an abundance of caution.
+    // noalias attributes (see #54878). Only enable mutable noalias by
-    cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(false)
+    // default for versions we believe to be safe.
+    cx.tcx
+        .sess
+        .opts
+        .debugging_opts
+        .mutable_noalias
+        .unwrap_or_else(|| llvm_util::get_version() >= (12, 0, 0))
 }
 
 impl ArgAttributesExt for ArgAttributes {

compiler/rustc_codegen_llvm/src/asm.rs

@@ -283,10 +283,12 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                 }
                 InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {}
                 InlineAsmArch::Nvptx64 => {}
+                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
                 InlineAsmArch::Hexagon => {}
                 InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
                 InlineAsmArch::SpirV => {}
                 InlineAsmArch::Wasm32 => {}
+                InlineAsmArch::Bpf => {}
             }
         }
         if !options.contains(InlineAsmOptions::NOMEM) {
@@ -355,10 +357,49 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 }
 
 impl AsmMethods for CodegenCx<'ll, 'tcx> {
-    fn codegen_global_asm(&self, ga: &hir::GlobalAsm) {
+    fn codegen_global_asm(
-        let asm = ga.asm.as_str();
+        &self,
+        template: &[InlineAsmTemplatePiece],
+        operands: &[GlobalAsmOperandRef],
+        options: InlineAsmOptions,
+        _line_spans: &[Span],
+    ) {
+        let asm_arch = self.tcx.sess.asm_arch.unwrap();
+
+        // Default to Intel syntax on x86
+        let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
+            && !options.contains(InlineAsmOptions::ATT_SYNTAX);
+
+        // Build the template string
+        let mut template_str = String::new();
+        if intel_syntax {
+            template_str.push_str(".intel_syntax\n");
+        }
+        for piece in template {
+            match *piece {
+                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
+                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
+                    match operands[operand_idx] {
+                        GlobalAsmOperandRef::Const { ref string } => {
+                            // Const operands get injected directly into the
+                            // template. Note that we don't need to escape $
+                            // here unlike normal inline assembly.
+                            template_str.push_str(string);
+                        }
+                    }
+                }
+            }
+        }
+        if intel_syntax {
+            template_str.push_str("\n.att_syntax\n");
+        }
+
         unsafe {
-            llvm::LLVMRustAppendModuleInlineAsm(self.llmod, asm.as_ptr().cast(), asm.len());
+            llvm::LLVMRustAppendModuleInlineAsm(
+                self.llmod,
+                template_str.as_ptr().cast(),
+                template_str.len(),
+            );
         }
     }
 }
@@ -540,6 +581,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>)
             InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
             InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
             InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
+            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
+            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
@@ -550,6 +594,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>)
             InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
             InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk",
             InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r",
+            InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w",
             InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
                 bug!("LLVM backend does not support SPIR-V")
             }
@@ -590,6 +636,7 @@ fn modifier_to_llvm(
         InlineAsmRegClass::Hexagon(_) => None,
         InlineAsmRegClass::Mips(_) => None,
         InlineAsmRegClass::Nvptx(_) => None,
+        InlineAsmRegClass::PowerPC(_) => None,
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
         | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
         InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
@@ -617,6 +664,7 @@ fn modifier_to_llvm(
         },
         InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
         InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None,
+        InlineAsmRegClass::Bpf(_) => None,
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
         }
@@ -651,6 +699,9 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll
         InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
         InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
         InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
+        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
+        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
+        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
         InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
@@ -661,6 +712,8 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll
         | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
         InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
         InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
+        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
+        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
         }

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -4,12 +4,10 @@ use std::ffi::CString;
 
 use cstr::cstr;
 use rustc_codegen_ssa::traits::*;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::HasTyCtxt;
-use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::OptLevel;
 use rustc_session::Session;
@@ -352,35 +350,6 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
     }
 }
 
-pub fn provide_both(providers: &mut Providers) {
-    providers.wasm_import_module_map = |tcx, cnum| {
-        // Build up a map from DefId to a `NativeLib` structure, where
-        // `NativeLib` internally contains information about
-        // `#[link(wasm_import_module = "...")]` for example.
-        let native_libs = tcx.native_libraries(cnum);
-
-        let def_id_to_native_lib = native_libs
-            .iter()
-            .filter_map(|lib| lib.foreign_module.map(|id| (id, lib)))
-            .collect::<FxHashMap<_, _>>();
-
-        let mut ret = FxHashMap::default();
-        for (def_id, lib) in tcx.foreign_modules(cnum).iter() {
-            let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module);
-            let module = match module {
-                Some(s) => s,
-                None => continue,
-            };
-            ret.extend(lib.foreign_items.iter().map(|id| {
-                assert_eq!(id.krate, cnum);
-                (*id, module.to_string())
-            }));
-        }
-
-        ret
-    };
-}
-
 fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option<CString> {
     tcx.wasm_import_module_map(id.krate).get(&id).map(|s| CString::new(&s[..]).unwrap())
 }

compiler/rustc_codegen_llvm/src/back/archive.rs

@@ -8,9 +8,11 @@ use std::ptr;
 use std::str;
 
 use crate::llvm::archive_ro::{ArchiveRO, Child};
-use crate::llvm::{self, ArchiveKind};
+use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport};
 use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder};
 use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME};
+use rustc_data_structures::temp_dir::MaybeTempDir;
+use rustc_middle::middle::cstore::DllImport;
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 
@@ -61,6 +63,17 @@ fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) ->
     }
 }
 
+/// Map machine type strings to values of LLVM's MachineTypes enum.
+fn llvm_machine_type(cpu: &str) -> LLVMMachineType {
+    match cpu {
+        "x86_64" => LLVMMachineType::AMD64,
+        "x86" => LLVMMachineType::I386,
+        "aarch64" => LLVMMachineType::ARM64,
+        "arm" => LLVMMachineType::ARM,
+        _ => panic!("unsupported cpu type {}", cpu),
+    }
+}
+
 impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
     /// Creates a new static archive, ready for modifying the archive specified
     /// by `config`.
@@ -100,8 +113,9 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
 
     /// Adds all of the contents of a native library to this archive. This will
     /// search in the relevant locations for a library named `name`.
-    fn add_native_library(&mut self, name: Symbol) {
+    fn add_native_library(&mut self, name: Symbol, verbatim: bool) {
-        let location = find_library(name, &self.config.lib_search_paths, self.config.sess);
+        let location =
+            find_library(name, verbatim, &self.config.lib_search_paths, self.config.sess);
         self.add_archive(&location, |_| false).unwrap_or_else(|e| {
             self.config.sess.fatal(&format!(
                 "failed to add native library {}: {}",
@@ -174,6 +188,74 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
             self.config.sess.fatal(&format!("failed to build archive: {}", e));
         }
     }
+
+    fn inject_dll_import_lib(
+        &mut self,
+        lib_name: &str,
+        dll_imports: &[DllImport],
+        tmpdir: &MaybeTempDir,
+    ) {
+        let output_path = {
+            let mut output_path: PathBuf = tmpdir.as_ref().to_path_buf();
+            output_path.push(format!("{}_imports", lib_name));
+            output_path.with_extension("lib")
+        };
+
+        // we've checked for \0 characters in the library name already
+        let dll_name_z = CString::new(lib_name).unwrap();
+        // All import names are Rust identifiers and therefore cannot contain \0 characters.
+        // FIXME: when support for #[link_name] implemented, ensure that import.name values don't
+        // have any \0 characters
+        let import_name_vector: Vec<CString> = dll_imports
+            .iter()
+            .map(if self.config.sess.target.arch == "x86" {
+                |import: &DllImport| CString::new(format!("_{}", import.name.to_string())).unwrap()
+            } else {
+                |import: &DllImport| CString::new(import.name.to_string()).unwrap()
+            })
+            .collect();
+
+        let output_path_z = rustc_fs_util::path_to_c_string(&output_path);
+
+        tracing::trace!("invoking LLVMRustWriteImportLibrary");
+        tracing::trace!("  dll_name {:#?}", dll_name_z);
+        tracing::trace!("  output_path {}", output_path.display());
+        tracing::trace!(
+            "  import names: {}",
+            dll_imports.iter().map(|import| import.name.to_string()).collect::<Vec<_>>().join(", "),
+        );
+
+        let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_vector
+            .iter()
+            .map(|name_z| LLVMRustCOFFShortExport::from_name(name_z.as_ptr()))
+            .collect();
+        let result = unsafe {
+            crate::llvm::LLVMRustWriteImportLibrary(
+                dll_name_z.as_ptr(),
+                output_path_z.as_ptr(),
+                ffi_exports.as_ptr(),
+                ffi_exports.len(),
+                llvm_machine_type(&self.config.sess.target.arch) as u16,
+                !self.config.sess.target.is_like_msvc,
+            )
+        };
+
+        if result == crate::llvm::LLVMRustResult::Failure {
+            self.config.sess.fatal(&format!(
+                "Error creating import library for {}: {}",
+                lib_name,
+                llvm::last_error().unwrap_or("unknown LLVM error".to_string())
+            ));
+        }
+
+        self.add_archive(&output_path, |_| false).unwrap_or_else(|e| {
+            self.config.sess.fatal(&format!(
+                "failed to add native library {}: {}",
+                output_path.display(),
+                e
+            ));
+        });
+    }
 }
 
 impl<'a> LlvmArchiveBuilder<'a> {

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -568,10 +568,11 @@ fn thin_lto(
 
 pub(crate) fn run_pass_manager(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    diag_handler: &Handler,
     module: &ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
     thin: bool,
-) {
+) -> Result<(), FatalError> {
     let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &module.name[..]);
 
     // Now we have one massive module inside of llmod. Time to run the
@@ -584,15 +585,16 @@ pub(crate) fn run_pass_manager(
         if write::should_use_new_llvm_pass_manager(config) {
             let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
             let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
-            // See comment below for why this is necessary.
+            write::optimize_with_new_llvm_pass_manager(
-            let opt_level = if let config::OptLevel::No = opt_level {
+                cgcx,
-                config::OptLevel::Less
+                diag_handler,
-            } else {
+                module,
-                opt_level
+                config,
-            };
+                opt_level,
-            write::optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage);
+                opt_stage,
+            )?;
             debug!("lto done");
-            return;
+            return Ok(());
         }
 
         let pm = llvm::LLVMCreatePassManager();
@@ -603,26 +605,10 @@ pub(crate) fn run_pass_manager(
             llvm::LLVMRustAddPass(pm, pass.unwrap());
         }
 
-        // When optimizing for LTO we don't actually pass in `-O0`, but we force
-        // it to always happen at least with `-O1`.
-        //
-        // With ThinLTO we mess around a lot with symbol visibility in a way
-        // that will actually cause linking failures if we optimize at O0 which
-        // notable is lacking in dead code elimination. To ensure we at least
-        // get some optimizations and correctly link we forcibly switch to `-O1`
-        // to get dead code elimination.
-        //
-        // Note that in general this shouldn't matter too much as you typically
-        // only turn on ThinLTO when you're compiling with optimizations
-        // otherwise.
         let opt_level = config
             .opt_level
             .map(|x| to_llvm_opt_settings(x).0)
             .unwrap_or(llvm::CodeGenOptLevel::None);
-        let opt_level = match opt_level {
-            llvm::CodeGenOptLevel::None => llvm::CodeGenOptLevel::Less,
-            level => level,
-        };
         with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| {
             if thin {
                 llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm);
@@ -650,6 +636,7 @@ pub(crate) fn run_pass_manager(
         llvm::LLVMDisposePassManager(pm);
     }
     debug!("lto done");
+    Ok(())
 }
 
 pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer);
@@ -872,7 +859,7 @@ pub unsafe fn optimize_thin_module(
         {
             info!("running thin lto passes over {}", module.name);
             let config = cgcx.config(module.kind);
-            run_pass_manager(cgcx, &module, config, true);
+            run_pass_manager(cgcx, &diag_handler, &module, config, true)?;
             save_temp_bitcode(cgcx, &module, "thin-lto-after-pm");
         }
     }

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -20,7 +20,6 @@ use rustc_codegen_ssa::{CompiledModule, ModuleCodegen};
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_errors::{FatalError, Handler, Level};
 use rustc_fs_util::{link_or_copy, path_to_c_string};
-use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath};
@@ -92,13 +91,12 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm:
 
 pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine {
     let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
-        tcx.output_filenames(LOCAL_CRATE)
+        tcx.output_filenames(()).split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name))
-            .split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name))
     } else {
         None
     };
     let config = TargetMachineFactoryConfig { split_dwarf_file };
-    target_machine_factory(&tcx.sess, tcx.backend_optimization_level(LOCAL_CRATE))(config)
+    target_machine_factory(&tcx.sess, tcx.backend_optimization_level(()))(config)
         .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise())
 }
 
@@ -410,16 +408,17 @@ fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
 
 pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool {
     // The new pass manager is disabled by default.
-    config.new_llvm_pass_manager
+    config.new_llvm_pass_manager.unwrap_or(false)
 }
 
 pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    diag_handler: &Handler,
     module: &ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
     opt_level: config::OptLevel,
     opt_stage: llvm::OptStage,
-) {
+) -> Result<(), FatalError> {
     let unroll_loops =
         opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
     let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed();
@@ -449,13 +448,12 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
         std::ptr::null_mut()
     };
 
+    let extra_passes = config.passes.join(",");
+
     // FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
     // We would have to add upstream support for this first, before we can support
     // config.inline_threshold and our more aggressive default thresholds.
-    // FIXME: NewPM uses an different and more explicit way to textually represent
+    let result = llvm::LLVMRustOptimizeWithNewPassManager(
-    // pass pipelines. It would probably make sense to expose this, but it would
-    // require a different format than the current -C passes.
-    llvm::LLVMRustOptimizeWithNewPassManager(
         module.module_llvm.llmod(),
         &*module.module_llvm.tm,
         to_pass_builder_opt_level(opt_level),
@@ -472,10 +470,15 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
         sanitizer_options.as_ref(),
         pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
         pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
+        config.instrument_coverage,
+        config.instrument_gcov,
         llvm_selfprofiler,
         selfprofile_before_pass_callback,
         selfprofile_after_pass_callback,
+        extra_passes.as_ptr().cast(),
+        extra_passes.len(),
     );
+    result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes"))
 }
 
 // Unsafe due to LLVM calls.
@@ -484,7 +487,7 @@ pub(crate) unsafe fn optimize(
     diag_handler: &Handler,
     module: &ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
-) {
+) -> Result<(), FatalError> {
     let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &module.name[..]);
 
     let llmod = module.module_llvm.llmod();
@@ -509,8 +512,14 @@ pub(crate) unsafe fn optimize(
                 _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
                 _ => llvm::OptStage::PreLinkNoLTO,
             };
-            optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage);
+            return optimize_with_new_llvm_pass_manager(
-            return;
+                cgcx,
+                diag_handler,
+                module,
+                config,
+                opt_level,
+                opt_stage,
+            );
         }
 
         if cgcx.prof.llvm_recording_enabled() {
@@ -545,15 +554,6 @@ pub(crate) unsafe fn optimize(
                     llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap());
                     continue;
                 }
-                if pass_name == "insert-gcov-profiling" || pass_name == "instrprof" {
-                    // Instrumentation must be inserted before optimization,
-                    // otherwise LLVM may optimize some functions away which
-                    // breaks llvm-cov.
-                    //
-                    // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp.
-                    llvm::LLVMRustAddPass(mpm, find_pass(pass_name).unwrap());
-                    continue;
-                }
 
                 if let Some(pass) = find_pass(pass_name) {
                     extra_passes.push(pass);
@@ -566,6 +566,18 @@ pub(crate) unsafe fn optimize(
                 }
             }
 
+            // Instrumentation must be inserted before optimization,
+            // otherwise LLVM may optimize some functions away which
+            // breaks llvm-cov.
+            //
+            // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp.
+            if config.instrument_gcov {
+                llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap());
+            }
+            if config.instrument_coverage {
+                llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap());
+            }
+
             add_sanitizer_passes(config, &mut extra_passes);
 
             // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need
@@ -642,6 +654,7 @@ pub(crate) unsafe fn optimize(
         llvm::LLVMDisposePassManager(fpm);
         llvm::LLVMDisposePassManager(mpm);
     }
+    Ok(())
 }
 
 unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) {

compiler/rustc_codegen_llvm/src/base.rs

@@ -18,7 +18,6 @@ use crate::builder::Builder;
 use crate::common;
 use crate::context::CodegenCx;
 use crate::llvm;
-use crate::metadata;
 use crate::value::Value;
 
 use rustc_codegen_ssa::base::maybe_create_entry_wrapper;
@@ -47,8 +46,24 @@ pub fn write_compressed_metadata<'tcx>(
     use snap::write::FrameEncoder;
     use std::io::Write;
 
+    // Historical note:
+    //
+    // When using link.exe it was seen that the section name `.note.rustc`
+    // was getting shortened to `.note.ru`, and according to the PE and COFF
+    // specification:
+    //
+    // > Executable images do not use a string table and do not support
+    // > section names longer than 8 characters
+    //
+    // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+    //
+    // As a result, we choose a slightly shorter name! As to why
+    // `.note.rustc` works on MinGW, see
+    // https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197
+    let section_name = if tcx.sess.target.is_like_osx { "__DATA,.rustc" } else { ".rustc" };
+
     let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod());
-    let mut compressed = tcx.metadata_encoding_version();
+    let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
     FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
 
     let llmeta = common::bytes_in_context(metadata_llcx, &compressed);
@@ -59,7 +74,6 @@ pub fn write_compressed_metadata<'tcx>(
         unsafe { llvm::LLVMAddGlobal(metadata_llmod, common::val_ty(llconst), buf.as_ptr()) };
     unsafe {
         llvm::LLVMSetInitializer(llglobal, llconst);
-        let section_name = metadata::metadata_section_name(&tcx.sess.target);
         let name = SmallCStr::new(section_name);
         llvm::LLVMSetSection(llglobal, name.as_ptr());
 

compiler/rustc_codegen_llvm/src/builder.rs

@@ -69,6 +69,7 @@ impl abi::HasDataLayout for Builder<'_, '_, '_> {
 }
 
 impl ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
+    #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.cx.tcx
     }
@@ -81,6 +82,7 @@ impl ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> {
 }
 
 impl HasTargetSpec for Builder<'_, '_, 'tcx> {
+    #[inline]
     fn target_spec(&self) -> &Target {
         &self.cx.target_spec()
     }
@@ -98,6 +100,7 @@ impl abi::LayoutOf for Builder<'_, '_, 'tcx> {
 impl Deref for Builder<'_, 'll, 'tcx> {
     type Target = CodegenCx<'ll, 'tcx>;
 
+    #[inline]
     fn deref(&self) -> &Self::Target {
         self.cx
     }
@@ -118,24 +121,16 @@ macro_rules! builder_methods_for_value_instructions {
 }
 
 impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
-    fn new_block<'b>(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &'b str) -> Self {
+    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
-        let mut bx = Builder::with_cx(cx);
+        let bx = Builder::with_cx(cx);
-        let llbb = unsafe {
+        unsafe {
-            let name = SmallCStr::new(name);
+            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
-            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
+        }
-        };
-        bx.position_at_end(llbb);
         bx
     }
 
-    fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self {
+    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
-        // Create a fresh builder from the crate context.
+        self.cx
-        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) };
-        Builder { llbuilder, cx }
-    }
-
-    fn build_sibling_block(&self, name: &str) -> Self {
-        Builder::new_block(self.cx, self.llfn(), name)
     }
 
     fn llbb(&self) -> &'ll BasicBlock {
@@ -144,12 +139,22 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn set_span(&mut self, _span: Span) {}
 
-    fn position_at_end(&mut self, llbb: &'ll BasicBlock) {
+    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
         unsafe {
-            llvm::LLVMPositionBuilderAtEnd(self.llbuilder, llbb);
+            let name = SmallCStr::new(name);
+            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
         }
     }
 
+    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
+        Self::append_block(self.cx, self.llfn(), name)
+    }
+
+    fn build_sibling_block(&mut self, name: &str) -> Self {
+        let llbb = self.append_sibling_block(name);
+        Self::build(self.cx, llbb)
+    }
+
     fn ret_void(&mut self) {
         unsafe {
             llvm::LLVMBuildRetVoid(self.llbuilder);
@@ -1144,14 +1149,6 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
     }
 
-    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
-        self.cx
-    }
-
-    unsafe fn delete_basic_block(&mut self, bb: &'ll BasicBlock) {
-        llvm::LLVMDeleteBasicBlock(bb);
-    }
-
     fn do_not_inline(&mut self, llret: &'ll Value) {
         llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret);
     }
@@ -1165,6 +1162,12 @@ impl StaticBuilderMethods for Builder<'a, 'll, 'tcx> {
 }
 
 impl Builder<'a, 'll, 'tcx> {
+    fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self {
+        // Create a fresh builder from the crate context.
+        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) };
+        Builder { llbuilder, cx }
+    }
+
     pub fn llfn(&self) -> &'ll Value {
         unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
     }

compiler/rustc_codegen_llvm/src/callee.rs

@@ -14,7 +14,6 @@ use tracing::debug;
 
 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
 use rustc_middle::ty::{self, Instance, TypeFoldable};
-use rustc_target::spec::RelocModel;
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
@@ -181,7 +180,7 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value
                 llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport);
             }
 
-            if cx.tcx.sess.relocation_model() == RelocModel::Static {
+            if cx.should_assume_dso_local(llfn, true) {
                 llvm::LLVMRustSetDSOLocal(llfn, true);
             }
         }

compiler/rustc_codegen_llvm/src/consts.rs

@@ -282,6 +282,12 @@ impl CodegenCx<'ll, 'tcx> {
             }
         }
 
+        unsafe {
+            if self.should_assume_dso_local(g, true) {
+                llvm::LLVMRustSetDSOLocal(g, true);
+            }
+        }
+
         self.instances.borrow_mut().insert(instance, g);
         g
     }
@@ -363,6 +369,10 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> {
             set_global_alignment(&self, g, self.align_of(ty));
             llvm::LLVMSetInitializer(g, v);
 
+            if self.should_assume_dso_local(g, true) {
+                llvm::LLVMRustSetDSOLocal(g, true);
+            }
+
             // As an optimization, all shared statics which do not have interior
             // mutability are placed into read-only memory.
             if !is_mutable && self.type_is_freeze(ty) {

compiler/rustc_codegen_llvm/src/context.rs

@@ -765,18 +765,21 @@ impl<'b, 'tcx> CodegenCx<'b, 'tcx> {
 }
 
 impl HasDataLayout for CodegenCx<'ll, 'tcx> {
+    #[inline]
     fn data_layout(&self) -> &TargetDataLayout {
         &self.tcx.data_layout
     }
 }
 
 impl HasTargetSpec for CodegenCx<'ll, 'tcx> {
+    #[inline]
     fn target_spec(&self) -> &Target {
         &self.tcx.sess.target
     }
 }
 
 impl ty::layout::HasTyCtxt<'tcx> for CodegenCx<'ll, 'tcx> {
+    #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -6,9 +6,8 @@ use llvm::coverageinfo::CounterMappingRegion;
 use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression};
 use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
-use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE};
+use rustc_hir::def_id::{DefId, DefIdSet};
 use rustc_llvm::RustString;
-use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::coverage::CodeRegion;
 use rustc_span::Symbol;
 
@@ -249,7 +248,7 @@ fn save_function_record(
 ///
 /// We can find the unused functions (including generic functions) by the set difference of all MIR
 /// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query
-/// `collect_and_partition_mono_items`).
+/// `codegened_and_inlined_items`).
 ///
 /// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and
 /// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`)
@@ -266,7 +265,7 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
     let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics();
 
     let all_def_ids: DefIdSet = tcx
-        .mir_keys(LOCAL_CRATE)
+        .mir_keys(())
         .iter()
         .filter_map(|local_def_id| {
             let def_id = local_def_id.to_def_id();
@@ -277,15 +276,12 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
         })
         .collect();
 
-    let codegenned_def_ids = tcx.codegened_and_inlined_items(LOCAL_CRATE);
+    let codegenned_def_ids = tcx.codegened_and_inlined_items(());
 
     let mut unused_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default();
     for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) {
-        let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id);
+        // Make sure the non-codegenned (unused) function has at least one MIR
-        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
+        // `Coverage` statement with a code region, and return its file name.
-            continue;
-        }
-        // Make sure the non-codegenned (unused) function has a file_name
         if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) {
             let def_ids =
                 unused_def_ids_by_file.entry(*non_codegenned_file_name).or_insert_with(Vec::new);

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -223,7 +223,8 @@ fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx
 
 fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) {
     let llfn = cx.get_fn(instance);
-    let mut bx = Builder::new_block(cx, llfn, "unused_function");
+    let llbb = Builder::append_block(cx, llfn, "unused_function");
+    let mut bx = Builder::build(cx, llbb);
     let fn_name = bx.get_pgo_func_name_var(instance);
     let hash = bx.const_u64(0);
     let num_counters = bx.const_u32(1);

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -309,6 +309,7 @@ impl RecursiveTypeDescription<'ll, 'tcx> {
                     unfinished_type,
                     member_holding_stub,
                     member_descriptions,
+                    None,
                 );
                 MetadataCreationResult::new(metadata_stub, true)
             }
@@ -759,12 +760,12 @@ fn hex_encode(data: &[u8]) -> String {
 }
 
 pub fn file_metadata(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile {
-    debug!("file_metadata: file_name: {}", source_file.name);
+    debug!("file_metadata: file_name: {:?}", source_file.name);
 
     let hash = Some(&source_file.src_hash);
-    let file_name = Some(source_file.name.to_string());
+    let file_name = Some(source_file.name.prefer_remapped().to_string());
     let directory = if source_file.is_real_file() && !source_file.is_imported() {
-        Some(cx.sess().working_dir.0.to_string_lossy().to_string())
+        Some(cx.sess().working_dir.to_string_lossy(false).to_string())
     } else {
         // If the path comes from an upstream crate we assume it has been made
         // independent of the compiler's working directory one way or another.
@@ -992,11 +993,12 @@ pub fn compile_unit_metadata(
     let producer = format!("clang LLVM ({})", rustc_producer);
 
     let name_in_debuginfo = name_in_debuginfo.to_string_lossy();
-    let work_dir = tcx.sess.working_dir.0.to_string_lossy();
+    let work_dir = tcx.sess.working_dir.to_string_lossy(false);
     let flags = "\0";
-    let out_dir = &tcx.output_filenames(LOCAL_CRATE).out_directory;
+    let output_filenames = tcx.output_filenames(());
+    let out_dir = &output_filenames.out_directory;
     let split_name = if tcx.sess.target_can_use_split_dwarf() {
-        tcx.output_filenames(LOCAL_CRATE)
+        output_filenames
             .split_dwarf_path(tcx.sess.split_debuginfo(), Some(codegen_unit_name))
             .map(|f| out_dir.join(f))
     } else {
@@ -1057,15 +1059,12 @@ pub fn compile_unit_metadata(
         if tcx.sess.opts.debugging_opts.profile {
             let cu_desc_metadata =
                 llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata);
-            let default_gcda_path = &tcx.output_filenames(LOCAL_CRATE).with_extension("gcda");
+            let default_gcda_path = &output_filenames.with_extension("gcda");
             let gcda_path =
                 tcx.sess.opts.debugging_opts.profile_emit.as_ref().unwrap_or(default_gcda_path);
 
             let gcov_cu_info = [
-                path_to_mdstring(
+                path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")),
-                    debug_context.llcontext,
-                    &tcx.output_filenames(LOCAL_CRATE).with_extension("gcno"),
-                ),
                 path_to_mdstring(debug_context.llcontext, &gcda_path),
                 cu_desc_metadata,
             ];
@@ -1458,7 +1457,7 @@ struct EnumMemberDescriptionFactory<'ll, 'tcx> {
     enum_type: Ty<'tcx>,
     layout: TyAndLayout<'tcx>,
     tag_type_metadata: Option<&'ll DIType>,
-    containing_scope: &'ll DIScope,
+    common_members: Vec<Option<&'ll DIType>>,
     span: Span,
 }
 
@@ -1486,17 +1485,9 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
             _ => bug!(),
         };
 
-        // This will always find the metadata in the type map.
         let fallback = use_enum_fallback(cx);
-        let self_metadata = if fallback {
+        // This will always find the metadata in the type map.
-            self.containing_scope
+        let self_metadata = type_metadata(cx, self.enum_type, self.span);
-        } else {
-            type_metadata(cx, self.enum_type, self.span)
-        };
-        let flags = match self.enum_type.kind() {
-            ty::Generator(..) => DIFlags::FlagArtificial,
-            _ => DIFlags::FlagZero,
-        };
 
         match self.layout.variants {
             Variants::Single { index } => {
@@ -1511,7 +1502,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                     cx,
                     self.layout,
                     variant_info,
-                    NoTag,
+                    None,
                     self_metadata,
                     self.span,
                 );
@@ -1523,6 +1514,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                     self.enum_type,
                     variant_type_metadata,
                     member_descriptions,
+                    Some(&self.common_members),
                 );
                 vec![MemberDescription {
                     name: if fallback { String::new() } else { variant_info.variant_name() },
@@ -1530,7 +1522,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                     offset: Size::ZERO,
                     size: self.layout.size,
                     align: self.layout.align.abi,
-                    flags,
+                    flags: DIFlags::FlagZero,
                     discriminant: None,
                     source_info: variant_info.source_info(cx),
                 }]
@@ -1542,13 +1534,26 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                 ..
             } => {
                 let tag_info = if fallback {
-                    RegularTag {
+                    // For MSVC, we generate a union of structs for each variant with an explicit
+                    // discriminant field roughly equivalent to the following C:
+                    // ```c
+                    // union enum$<{name}> {
+                    //   struct {variant 0 name} {
+                    //     tag$ variant$;
+                    //     <variant 0 fields>
+                    //   } variant0;
+                    //   <other variant structs>
+                    // }
+                    // ```
+                    // The natvis in `intrinsic.nativs` then matches on `this.variant0.variant$` to
+                    // determine which variant is active and then displays it.
+                    Some(DirectTag {
                         tag_field: Field::from(tag_field),
                         tag_type_metadata: self.tag_type_metadata.unwrap(),
-                    }
+                    })
                 } else {
                     // This doesn't matter in this case.
-                    NoTag
+                    None
                 };
                 variants
                     .iter_enumerated()
@@ -1572,11 +1577,12 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                             self.enum_type,
                             variant_type_metadata,
                             member_descriptions,
+                            Some(&self.common_members),
                         );
 
                         MemberDescription {
                             name: if fallback {
-                                String::new()
+                                format!("variant{}", i.as_u32())
                             } else {
                                 variant_info.variant_name()
                             },
@@ -1584,7 +1590,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                             offset: Size::ZERO,
                             size: self.layout.size,
                             align: self.layout.align.abi,
-                            flags,
+                            flags: DIFlags::FlagZero,
                             discriminant: Some(
                                 self.layout.ty.discriminant_for_variant(cx.tcx, i).unwrap().val
                                     as u64,
@@ -1601,76 +1607,135 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                 ref variants,
                 tag_field,
             } => {
+                let calculate_niche_value = |i: VariantIdx| {
+                    if i == dataful_variant {
+                        None
+                    } else {
+                        let value = (i.as_u32() as u128)
+                            .wrapping_sub(niche_variants.start().as_u32() as u128)
+                            .wrapping_add(niche_start);
+                        let value = tag.value.size(cx).truncate(value);
+                        // NOTE(eddyb) do *NOT* remove this assert, until
+                        // we pass the full 128-bit value to LLVM, otherwise
+                        // truncation will be silent and remain undetected.
+                        assert_eq!(value as u64 as u128, value);
+                        Some(value as u64)
+                    }
+                };
+
+                // For MSVC, we will generate a union of two fields, one for the dataful variant
+                // and one that just points to the discriminant. We also create an enum that
+                // contains tag values for the non-dataful variants and make the discriminant field
+                // that type. We then use natvis to render the enum type correctly in Windbg/VS.
+                // This will generate debuginfo roughly equivalent to the following C:
+                // ```c
+                // union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> {
+                //   struct <dataful variant name> {
+                //     <fields in dataful variant>
+                //   } dataful_variant;
+                //   enum Discriminant$ {
+                //     <non-dataful variants>
+                //   } discriminant;
+                // }
+                // ```
+                // The natvis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>`
+                // and evaluates `this.discriminant`. If the value is between the min niche and max
+                // niche, then the enum is in the dataful variant and `this.dataful_variant` is
+                // rendered. Otherwise, the enum is in one of the non-dataful variants. In that
+                // case, we just need to render the name of the `this.discriminant` enum.
                 if fallback {
-                    let variant = self.layout.for_variant(cx, dataful_variant);
+                    let dataful_variant_layout = self.layout.for_variant(cx, dataful_variant);
-                    // Create a description of the non-null variant.
+
-                    let (variant_type_metadata, member_description_factory) = describe_enum_variant(
+                    let mut discr_enum_ty = tag.value.to_ty(cx.tcx);
+                    // If the niche is the NULL value of a reference, then `discr_enum_ty` will be a RawPtr.
+                    // CodeView doesn't know what to do with enums whose base type is a pointer so we fix this up
+                    // to just be `usize`.
+                    if let ty::RawPtr(_) = discr_enum_ty.kind() {
+                        discr_enum_ty = cx.tcx.types.usize;
+                    }
+
+                    let tags: Vec<_> = variants
+                        .iter_enumerated()
+                        .filter_map(|(variant_idx, _)| {
+                            calculate_niche_value(variant_idx).map(|tag| {
+                                let variant = variant_info_for(variant_idx);
+                                let name = variant.variant_name();
+
+                                Some(unsafe {
+                                    llvm::LLVMRustDIBuilderCreateEnumerator(
+                                        DIB(cx),
+                                        name.as_ptr().cast(),
+                                        name.len(),
+                                        tag as i64,
+                                        !discr_enum_ty.is_signed(),
+                                    )
+                                })
+                            })
+                        })
+                        .collect();
+
+                    let discr_enum = unsafe {
+                        llvm::LLVMRustDIBuilderCreateEnumerationType(
+                            DIB(cx),
+                            self_metadata,
+                            "Discriminant$".as_ptr().cast(),
+                            "Discriminant$".len(),
+                            unknown_file_metadata(cx),
+                            UNKNOWN_LINE_NUMBER,
+                            tag.value.size(cx).bits(),
+                            tag.value.align(cx).abi.bits() as u32,
+                            create_DIArray(DIB(cx), &tags),
+                            type_metadata(cx, discr_enum_ty, self.span),
+                            true,
+                        )
+                    };
+
+                    let variant_info = variant_info_for(dataful_variant);
+                    let (variant_type_metadata, member_desc_factory) = describe_enum_variant(
                         cx,
-                        variant,
+                        dataful_variant_layout,
-                        variant_info_for(dataful_variant),
+                        variant_info,
-                        OptimizedTag,
+                        Some(NicheTag),
-                        self.containing_scope,
+                        self_metadata,
                         self.span,
                     );
 
-                    let variant_member_descriptions =
+                    let member_descriptions = member_desc_factory.create_member_descriptions(cx);
-                        member_description_factory.create_member_descriptions(cx);
 
                     set_members_of_composite_type(
                         cx,
                         self.enum_type,
                         variant_type_metadata,
-                        variant_member_descriptions,
+                        member_descriptions,
+                        Some(&self.common_members),
                     );
 
-                    // Encode the information about the null variant in the union
+                    let (size, align) =
-                    // member's name.
+                        cx.size_and_align_of(dataful_variant_layout.field(cx, tag_field).ty);
-                    let mut name = String::from("RUST$ENCODED$ENUM$");
-                    // Right now it's not even going to work for `niche_start > 0`,
-                    // and for multiple niche variants it only supports the first.
-                    fn compute_field_path<'a, 'tcx>(
-                        cx: &CodegenCx<'a, 'tcx>,
-                        name: &mut String,
-                        layout: TyAndLayout<'tcx>,
-                        offset: Size,
-                        size: Size,
-                    ) {
-                        for i in 0..layout.fields.count() {
-                            let field_offset = layout.fields.offset(i);
-                            if field_offset > offset {
-                                continue;
-                            }
-                            let inner_offset = offset - field_offset;
-                            let field = layout.field(cx, i);
-                            if inner_offset + size <= field.size {
-                                write!(name, "{}$", i).unwrap();
-                                compute_field_path(cx, name, field, inner_offset, size);
-                            }
-                        }
-                    }
-                    compute_field_path(
-                        cx,
-                        &mut name,
-                        self.layout,
-                        self.layout.fields.offset(tag_field),
-                        self.layout.field(cx, tag_field).size,
-                    );
-                    let variant_info = variant_info_for(*niche_variants.start());
-                    variant_info.map_struct_name(|variant_name| {
-                        name.push_str(variant_name);
-                    });
 
-                    // Create the (singleton) list of descriptions of union members.
+                    vec![
-                    vec![MemberDescription {
+                        MemberDescription {
-                        name,
+                            // Name the dataful variant so that we can identify it for natvis
-                        type_metadata: variant_type_metadata,
+                            name: "dataful_variant".to_string(),
-                        offset: Size::ZERO,
+                            type_metadata: variant_type_metadata,
-                        size: variant.size,
+                            offset: Size::ZERO,
-                        align: variant.align.abi,
+                            size: self.layout.size,
-                        flags,
+                            align: self.layout.align.abi,
-                        discriminant: None,
+                            flags: DIFlags::FlagZero,
-                        source_info: variant_info.source_info(cx),
+                            discriminant: None,
-                    }]
+                            source_info: variant_info.source_info(cx),
+                        },
+                        MemberDescription {
+                            name: "discriminant".into(),
+                            type_metadata: discr_enum,
+                            offset: dataful_variant_layout.fields.offset(tag_field),
+                            size,
+                            align,
+                            flags: DIFlags::FlagZero,
+                            discriminant: None,
+                            source_info: None,
+                        },
+                    ]
                 } else {
                     variants
                         .iter_enumerated()
@@ -1682,7 +1747,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                                     cx,
                                     variant,
                                     variant_info,
-                                    OptimizedTag,
+                                    Some(NicheTag),
                                     self_metadata,
                                     self.span,
                                 );
@@ -1695,21 +1760,10 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                                 self.enum_type,
                                 variant_type_metadata,
                                 member_descriptions,
+                                Some(&self.common_members),
                             );
 
-                            let niche_value = if i == dataful_variant {
+                            let niche_value = calculate_niche_value(i);
-                                None
-                            } else {
-                                let value = (i.as_u32() as u128)
-                                    .wrapping_sub(niche_variants.start().as_u32() as u128)
-                                    .wrapping_add(niche_start);
-                                let value = tag.value.size(cx).truncate(value);
-                                // NOTE(eddyb) do *NOT* remove this assert, until
-                                // we pass the full 128-bit value to LLVM, otherwise
-                                // truncation will be silent and remain undetected.
-                                assert_eq!(value as u64 as u128, value);
-                                Some(value as u64)
-                            };
 
                             MemberDescription {
                                 name: variant_info.variant_name(),
@@ -1717,7 +1771,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> {
                                 offset: Size::ZERO,
                                 size: self.layout.size,
                                 align: self.layout.align.abi,
-                                flags,
+                                flags: DIFlags::FlagZero,
                                 discriminant: niche_value,
                                 source_info: variant_info.source_info(cx),
                             }
@@ -1771,14 +1825,10 @@ impl VariantMemberDescriptionFactory<'ll, 'tcx> {
     }
 }
 
-// FIXME: terminology here should be aligned with `abi::TagEncoding`.
-// `OptimizedTag` is `TagEncoding::Niche`, `RegularTag` is `TagEncoding::Direct`.
-// `NoTag` should be removed; users should use `Option<EnumTagInfo>` instead.
 #[derive(Copy, Clone)]
 enum EnumTagInfo<'ll> {
-    RegularTag { tag_field: Field, tag_type_metadata: &'ll DIType },
+    DirectTag { tag_field: Field, tag_type_metadata: &'ll DIType },
-    OptimizedTag,
+    NicheTag,
-    NoTag,
 }
 
 #[derive(Copy, Clone)]
@@ -1849,13 +1899,6 @@ impl<'tcx> VariantInfo<'_, 'tcx> {
         }
         None
     }
-
-    fn is_artificial(&self) -> bool {
-        match self {
-            VariantInfo::Generator { .. } => true,
-            VariantInfo::Adt(..) => false,
-        }
-    }
 }
 
 /// Returns a tuple of (1) `type_metadata_stub` of the variant, (2) a
@@ -1866,7 +1909,7 @@ fn describe_enum_variant(
     cx: &CodegenCx<'ll, 'tcx>,
     layout: layout::TyAndLayout<'tcx>,
     variant: VariantInfo<'_, 'tcx>,
-    discriminant_info: EnumTagInfo<'ll>,
+    discriminant_info: Option<EnumTagInfo<'ll>>,
     containing_scope: &'ll DIScope,
     span: Span,
 ) -> (&'ll DICompositeType, MemberDescriptionFactory<'ll, 'tcx>) {
@@ -1881,8 +1924,7 @@ fn describe_enum_variant(
             &variant_name,
             unique_type_id,
             Some(containing_scope),
-            // FIXME(tmandry): This doesn't seem to have any effect.
+            DIFlags::FlagZero,
-            if variant.is_artificial() { DIFlags::FlagArtificial } else { DIFlags::FlagZero },
         )
     });
 
@@ -1890,12 +1932,11 @@ fn describe_enum_variant(
     let (offsets, args) = if use_enum_fallback(cx) {
         // If this is not a univariant enum, there is also the discriminant field.
         let (discr_offset, discr_arg) = match discriminant_info {
-            RegularTag { tag_field, .. } => {
+            Some(DirectTag { tag_field, .. }) => {
                 // We have the layout of an enum variant, we need the layout of the outer enum
                 let enum_layout = cx.layout_of(layout.ty);
                 let offset = enum_layout.fields.offset(tag_field.as_usize());
-                let args =
+                let args = ("variant$".to_owned(), enum_layout.field(cx, tag_field.as_usize()).ty);
-                    ("RUST$ENUM$DISR".to_owned(), enum_layout.field(cx, tag_field.as_usize()).ty);
                 (Some(offset), Some(args))
             }
             _ => (None, None),
@@ -1926,7 +1967,7 @@ fn describe_enum_variant(
         offsets,
         args,
         tag_type_metadata: match discriminant_info {
-            RegularTag { tag_type_metadata, .. } => Some(tag_type_metadata),
+            Some(DirectTag { tag_type_metadata, .. }) => Some(tag_type_metadata),
             _ => None,
         },
         span,
@@ -1945,11 +1986,6 @@ fn prepare_enum_metadata(
 ) -> RecursiveTypeDescription<'ll, 'tcx> {
     let tcx = cx.tcx;
     let enum_name = compute_debuginfo_type_name(tcx, enum_type, false);
-    // FIXME(tmandry): This doesn't seem to have any effect.
-    let enum_flags = match enum_type.kind() {
-        ty::Generator(..) => DIFlags::FlagArtificial,
-        _ => DIFlags::FlagZero,
-    };
 
     let containing_scope = get_namespace_for_item(cx, enum_def_id);
     // FIXME: This should emit actual file metadata for the enum, but we
@@ -2061,9 +2097,9 @@ fn prepare_enum_metadata(
 
     if use_enum_fallback(cx) {
         let discriminant_type_metadata = match layout.variants {
-            Variants::Single { .. }
+            Variants::Single { .. } => None,
-            | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => None,
+            Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, ref tag, .. }
-            Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => {
+            | Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => {
                 Some(discriminant_type_metadata(tag.value))
             }
         };
@@ -2075,14 +2111,14 @@ fn prepare_enum_metadata(
             unsafe {
                 llvm::LLVMRustDIBuilderCreateUnionType(
                     DIB(cx),
-                    containing_scope,
+                    None,
                     enum_name.as_ptr().cast(),
                     enum_name.len(),
                     file_metadata,
                     UNKNOWN_LINE_NUMBER,
                     layout.size.bits(),
                     layout.align.abi.bits() as u32,
-                    enum_flags,
+                    DIFlags::FlagZero,
                     None,
                     0, // RuntimeLang
                     unique_type_id_str.as_ptr().cast(),
@@ -2101,7 +2137,7 @@ fn prepare_enum_metadata(
                 enum_type,
                 layout,
                 tag_type_metadata: discriminant_type_metadata,
-                containing_scope,
+                common_members: vec![],
                 span,
             }),
         );
@@ -2171,7 +2207,7 @@ fn prepare_enum_metadata(
         }
     };
 
-    let mut outer_fields = match layout.variants {
+    let outer_fields = match layout.variants {
         Variants::Single { .. } => vec![],
         Variants::Multiple { .. } => {
             let tuple_mdf = TupleMemberDescriptionFactory {
@@ -2203,18 +2239,21 @@ fn prepare_enum_metadata(
             UNKNOWN_LINE_NUMBER,
             layout.size.bits(),
             layout.align.abi.bits() as u32,
-            enum_flags,
+            DIFlags::FlagZero,
             discriminator_metadata,
             empty_array,
             variant_part_unique_type_id_str.as_ptr().cast(),
             variant_part_unique_type_id_str.len(),
         )
     };
-    outer_fields.push(Some(variant_part));
 
     let struct_wrapper = {
         // The variant part must be wrapped in a struct according to DWARF.
-        let type_array = create_DIArray(DIB(cx), &outer_fields);
+        // All fields except the discriminant (including `outer_fields`)
+        // should be put into structures inside the variant part, which gives
+        // an equivalent layout but offers us much better integration with
+        // debuggers.
+        let type_array = create_DIArray(DIB(cx), &[Some(variant_part)]);
 
         let type_map = debug_context(cx).type_map.borrow();
         let unique_type_id_str = type_map.get_unique_type_id_as_string(unique_type_id);
@@ -2229,7 +2268,7 @@ fn prepare_enum_metadata(
                 UNKNOWN_LINE_NUMBER,
                 layout.size.bits(),
                 layout.align.abi.bits() as u32,
-                enum_flags,
+                DIFlags::FlagZero,
                 None,
                 type_array,
                 0,
@@ -2250,7 +2289,7 @@ fn prepare_enum_metadata(
             enum_type,
             layout,
             tag_type_metadata: None,
-            containing_scope,
+            common_members: outer_fields,
             span,
         }),
     )
@@ -2283,7 +2322,13 @@ fn composite_type_metadata(
         DIFlags::FlagZero,
     );
     // ... and immediately create and add the member descriptions.
-    set_members_of_composite_type(cx, composite_type, composite_type_metadata, member_descriptions);
+    set_members_of_composite_type(
+        cx,
+        composite_type,
+        composite_type_metadata,
+        member_descriptions,
+        None,
+    );
 
     composite_type_metadata
 }
@@ -2293,6 +2338,7 @@ fn set_members_of_composite_type(
     composite_type: Ty<'tcx>,
     composite_type_metadata: &'ll DICompositeType,
     member_descriptions: Vec<MemberDescription<'ll>>,
+    common_members: Option<&Vec<Option<&'ll DIType>>>,
 ) {
     // In some rare cases LLVM metadata uniquing would lead to an existing type
     // description being used instead of a new one created in
@@ -2311,10 +2357,13 @@ fn set_members_of_composite_type(
         }
     }
 
-    let member_metadata: Vec<_> = member_descriptions
+    let mut member_metadata: Vec<_> = member_descriptions
         .into_iter()
         .map(|desc| Some(desc.into_metadata(cx, composite_type_metadata)))
         .collect();
+    if let Some(other_members) = common_members {
+        member_metadata.extend(other_members.iter());
+    }
 
     let type_params = compute_type_parameters(cx, composite_type);
     unsafe {
@@ -2435,7 +2484,7 @@ fn create_union_stub(
 
         llvm::LLVMRustDIBuilderCreateUnionType(
             DIB(cx),
-            containing_scope,
+            Some(containing_scope),
             union_type_name.as_ptr().cast(),
             union_type_name.len(),
             unknown_file_metadata(cx),

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -23,7 +23,7 @@ use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, Variab
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
-use rustc_hir::def_id::{DefId, DefIdMap, LOCAL_CRATE};
+use rustc_hir::def_id::{DefId, DefIdMap};
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir;
 use rustc_middle::ty::layout::HasTyCtxt;
@@ -343,7 +343,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         if self.sess().opts.optimize != config::OptLevel::No {
             spflags |= DISPFlags::SPFlagOptimized;
         }
-        if let Some((id, _)) = self.tcx.entry_fn(LOCAL_CRATE) {
+        if let Some((id, _)) = self.tcx.entry_fn(()) {
             if id == def_id {
                 spflags |= DISPFlags::SPFlagMainSubprogram;
             }

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -678,7 +678,8 @@ fn gen_fn<'ll, 'tcx>(
     cx.apply_target_cpu_attr(llfn);
     // FIXME(eddyb) find a nicer way to do this.
     unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) };
-    let bx = Builder::new_block(cx, llfn, "entry-block");
+    let llbb = Builder::append_block(cx, llfn, "entry-block");
+    let bx = Builder::build(cx, llbb);
     codegen(bx);
     llfn
 }

compiler/rustc_codegen_llvm/src/lib.rs

@@ -8,12 +8,11 @@
 #![feature(bool_to_option)]
 #![feature(const_cstr_unchecked)]
 #![feature(crate_visibility_modifier)]
-#![feature(extended_key_value_attributes)]
+#![cfg_attr(bootstrap, feature(extended_key_value_attributes))]
 #![feature(extern_types)]
 #![feature(in_band_lifetimes)]
 #![feature(iter_zip)]
 #![feature(nll)]
-#![cfg_attr(bootstrap, feature(or_patterns))]
 #![recursion_limit = "256"]
 
 use back::write::{create_informational_target_machine, create_target_machine};
@@ -30,8 +29,8 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{ErrorReported, FatalError, Handler};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
-use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoaderDyn};
+use rustc_middle::middle::cstore::EncodedMetadata;
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
@@ -69,7 +68,6 @@ pub mod llvm {
 }
 
 mod llvm_util;
-mod metadata;
 mod mono_item;
 mod type_;
 mod type_of;
@@ -162,7 +160,7 @@ impl WriteBackendMethods for LlvmCodegenBackend {
         module: &ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<(), FatalError> {
-        Ok(back::write::optimize(cgcx, diag_handler, module, config))
+        back::write::optimize(cgcx, diag_handler, module, config)
     }
     unsafe fn optimize_thin(
         cgcx: &CodegenContext<Self>,
@@ -189,8 +187,9 @@ impl WriteBackendMethods for LlvmCodegenBackend {
         module: &ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
         thin: bool,
-    ) {
+    ) -> Result<(), FatalError> {
-        back::lto::run_pass_manager(cgcx, module, config, thin)
+        let diag_handler = cgcx.create_diag_handler();
+        back::lto::run_pass_manager(cgcx, &diag_handler, module, config, thin)
     }
 }
 
@@ -249,18 +248,6 @@ impl CodegenBackend for LlvmCodegenBackend {
         target_features(sess)
     }
 
-    fn metadata_loader(&self) -> Box<MetadataLoaderDyn> {
-        Box::new(metadata::LlvmMetadataLoader)
-    }
-
-    fn provide(&self, providers: &mut ty::query::Providers) {
-        attributes::provide_both(providers);
-    }
-
-    fn provide_extern(&self, providers: &mut ty::query::Providers) {
-        attributes::provide_both(providers);
-    }
-
     fn codegen_crate<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -270,6 +257,7 @@ impl CodegenBackend for LlvmCodegenBackend {
         Box::new(rustc_codegen_ssa::base::codegen_crate(
             LlvmCodegenBackend(()),
             tcx,
+            crate::llvm_util::target_cpu(tcx.sess).to_string(),
             metadata,
             need_metadata_module,
         ))
@@ -305,13 +293,11 @@ impl CodegenBackend for LlvmCodegenBackend {
 
         // Run the linker on any artifacts that resulted from the LLVM run.
         // This should produce either a finished executable or library.
-        let target_cpu = crate::llvm_util::target_cpu(sess);
         link_binary::<LlvmArchiveBuilder<'_>>(
             sess,
             &codegen_results,
             outputs,
-            &codegen_results.crate_name.as_str(),
+            &codegen_results.crate_info.local_crate_name.as_str(),
-            target_cpu,
         );
 
         Ok(())

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -29,6 +29,31 @@ pub enum LLVMRustResult {
     Success,
     Failure,
 }
+
+// Rust version of the C struct with the same name in rustc_llvm/llvm-wrapper/RustWrapper.cpp.
+#[repr(C)]
+pub struct LLVMRustCOFFShortExport {
+    pub name: *const c_char,
+}
+
+impl LLVMRustCOFFShortExport {
+    pub fn from_name(name: *const c_char) -> LLVMRustCOFFShortExport {
+        LLVMRustCOFFShortExport { name }
+    }
+}
+
+/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h.
+///
+/// We include only architectures supported on Windows.
+#[derive(Copy, Clone, PartialEq)]
+#[repr(C)]
+pub enum LLVMMachineType {
+    AMD64 = 0x8664,
+    I386 = 0x14c,
+    ARM64 = 0xaa64,
+    ARM = 0x01c0,
+}
+
 // Consts for the LLVM CallConv type, pre-cast to usize.
 
 /// LLVM CallingConv::ID. Should we wrap this?
@@ -54,7 +79,7 @@ pub enum CallConv {
 }
 
 /// LLVMRustLinkage
-#[derive(PartialEq)]
+#[derive(Copy, Clone, PartialEq)]
 #[repr(C)]
 pub enum Linkage {
     ExternalLinkage = 0,
@@ -72,6 +97,7 @@ pub enum Linkage {
 
 // LLVMRustVisibility
 #[repr(C)]
+#[derive(Copy, Clone, PartialEq)]
 pub enum Visibility {
     Default = 0,
     Hidden = 1,
@@ -581,11 +607,6 @@ pub struct PassManager<'a>(InvariantOpaque<'a>);
 extern "C" {
     pub type PassManagerBuilder;
 }
-extern "C" {
-    pub type ObjectFile;
-}
-#[repr(C)]
-pub struct SectionIterator<'a>(InvariantOpaque<'a>);
 extern "C" {
     pub type Pass;
 }
@@ -1034,6 +1055,7 @@ extern "C" {
     pub fn LLVMDeleteGlobal(GlobalVar: &Value);
     pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>;
     pub fn LLVMSetInitializer(GlobalVar: &'a Value, ConstantVal: &'a Value);
+    pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool;
     pub fn LLVMSetThreadLocal(GlobalVar: &Value, IsThreadLocal: Bool);
     pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode);
     pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool;
@@ -1079,7 +1101,6 @@ extern "C" {
         Fn: &'a Value,
         Name: *const c_char,
     ) -> &'a BasicBlock;
-    pub fn LLVMDeleteBasicBlock(BB: &BasicBlock);
 
     // Operations on instructions
     pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>;
@@ -1702,35 +1723,6 @@ extern "C" {
 
     pub fn LLVMDisposeMessage(message: *mut c_char);
 
-    // Stuff that's in llvm-wrapper/ because it's not upstream yet.
-
-    /// Opens an object file.
-    pub fn LLVMCreateObjectFile(
-        MemBuf: &'static mut MemoryBuffer,
-    ) -> Option<&'static mut ObjectFile>;
-    /// Closes an object file.
-    pub fn LLVMDisposeObjectFile(ObjFile: &'static mut ObjectFile);
-
-    /// Enumerates the sections in an object file.
-    pub fn LLVMGetSections(ObjFile: &'a ObjectFile) -> &'a mut SectionIterator<'a>;
-    /// Destroys a section iterator.
-    pub fn LLVMDisposeSectionIterator(SI: &'a mut SectionIterator<'a>);
-    /// Returns `true` if the section iterator is at the end of the section
-    /// list:
-    pub fn LLVMIsSectionIteratorAtEnd(ObjFile: &'a ObjectFile, SI: &SectionIterator<'a>) -> Bool;
-    /// Moves the section iterator to point to the next section.
-    pub fn LLVMMoveToNextSection(SI: &SectionIterator<'_>);
-    /// Returns the current section size.
-    pub fn LLVMGetSectionSize(SI: &SectionIterator<'_>) -> c_ulonglong;
-    /// Returns the current section contents as a string buffer.
-    pub fn LLVMGetSectionContents(SI: &SectionIterator<'_>) -> *const c_char;
-
-    /// Reads the given file and returns it as a memory buffer. Use
-    /// LLVMDisposeMemoryBuffer() to get rid of it.
-    pub fn LLVMRustCreateMemoryBufferWithContentsOfFile(
-        Path: *const c_char,
-    ) -> Option<&'static mut MemoryBuffer>;
-
     pub fn LLVMStartMultithreaded() -> Bool;
 
     /// Returns a string describing the last error caused by an LLVMRust* call.
@@ -2037,7 +2029,7 @@ extern "C" {
 
     pub fn LLVMRustDIBuilderCreateUnionType(
         Builder: &DIBuilder<'a>,
-        Scope: &'a DIScope,
+        Scope: Option<&'a DIScope>,
         Name: *const c_char,
         NameLen: size_t,
         File: &'a DIFile,
@@ -2203,10 +2195,14 @@ extern "C" {
         SanitizerOptions: Option<&SanitizerOptions>,
         PGOGenPath: *const c_char,
         PGOUsePath: *const c_char,
+        InstrumentCoverage: bool,
+        InstrumentGCOV: bool,
         llvm_selfprofiler: *mut c_void,
         begin_callback: SelfProfileBeforePassCallback,
         end_callback: SelfProfileAfterPassCallback,
-    );
+        ExtraPasses: *const c_char,
+        ExtraPassesLen: size_t,
+    ) -> LLVMRustResult;
     pub fn LLVMRustPrintModule(
         M: &'a Module,
         Output: *const c_char,
@@ -2231,12 +2227,6 @@ extern "C" {
     pub fn LLVMRustArchiveIteratorFree(AIR: &'a mut ArchiveIterator<'a>);
     pub fn LLVMRustDestroyArchive(AR: &'static mut Archive);
 
-    #[allow(improper_ctypes)]
-    pub fn LLVMRustGetSectionName(
-        SI: &SectionIterator<'_>,
-        data: &mut Option<std::ptr::NonNull<c_char>>,
-    ) -> size_t;
-
     #[allow(improper_ctypes)]
     pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString);
 
@@ -2300,6 +2290,15 @@ extern "C" {
     ) -> &'a mut RustArchiveMember<'a>;
     pub fn LLVMRustArchiveMemberFree(Member: &'a mut RustArchiveMember<'a>);
 
+    pub fn LLVMRustWriteImportLibrary(
+        ImportName: *const c_char,
+        Path: *const c_char,
+        Exports: *const LLVMRustCOFFShortExport,
+        NumExports: usize,
+        Machine: u16,
+        MinGW: bool,
+    ) -> LLVMRustResult;
+
     pub fn LLVMRustSetDataLayoutFromTargetMachine(M: &'a Module, TM: &'a TargetMachine);
 
     pub fn LLVMRustBuildOperandBundleDef(

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -150,50 +150,6 @@ impl Attribute {
     }
 }
 
-// Memory-managed interface to object files.
-
-pub struct ObjectFile {
-    pub llof: &'static mut ffi::ObjectFile,
-}
-
-unsafe impl Send for ObjectFile {}
-
-impl ObjectFile {
-    // This will take ownership of llmb
-    pub fn new(llmb: &'static mut MemoryBuffer) -> Option<ObjectFile> {
-        unsafe {
-            let llof = LLVMCreateObjectFile(llmb)?;
-            Some(ObjectFile { llof })
-        }
-    }
-}
-
-impl Drop for ObjectFile {
-    fn drop(&mut self) {
-        unsafe {
-            LLVMDisposeObjectFile(&mut *(self.llof as *mut _));
-        }
-    }
-}
-
-// Memory-managed interface to section iterators.
-
-pub struct SectionIter<'a> {
-    pub llsi: &'a mut SectionIterator<'a>,
-}
-
-impl Drop for SectionIter<'a> {
-    fn drop(&mut self) {
-        unsafe {
-            LLVMDisposeSectionIterator(&mut *(self.llsi as *mut _));
-        }
-    }
-}
-
-pub fn mk_section_iter(llof: &ffi::ObjectFile) -> SectionIter<'_> {
-    unsafe { SectionIter { llsi: LLVMGetSections(llof) } }
-}
-
 pub fn set_section(llglobal: &Value, section_name: &str) {
     let section_name_cstr = CString::new(section_name).expect("unexpected CString error");
     unsafe {

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -163,6 +163,12 @@ pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str {
         ("x86", "avx512vpclmulqdq") => "vpclmulqdq",
         ("aarch64", "fp") => "fp-armv8",
         ("aarch64", "fp16") => "fullfp16",
+        ("aarch64", "fhm") => "fp16fml",
+        ("aarch64", "rcpc2") => "rcpc-immo",
+        ("aarch64", "dpb") => "ccpp",
+        ("aarch64", "dpb2") => "ccdp",
+        ("aarch64", "frintts") => "fptoint",
+        ("aarch64", "fcma") => "complxnum",
         (_, s) => s,
     }
 }
@@ -350,24 +356,32 @@ pub fn llvm_global_features(sess: &Session) -> Vec<String> {
         Some(_) | None => {}
     };
 
+    let filter = |s: &str| {
+        if s.is_empty() {
+            return None;
+        }
+        let feature = if s.starts_with("+") || s.starts_with("-") {
+            &s[1..]
+        } else {
+            return Some(s.to_string());
+        };
+        // Rustc-specific feature requests like `+crt-static` or `-crt-static`
+        // are not passed down to LLVM.
+        if RUSTC_SPECIFIC_FEATURES.contains(&feature) {
+            return None;
+        }
+        // ... otherwise though we run through `to_llvm_feature` feature when
+        // passing requests down to LLVM. This means that all in-language
+        // features also work on the command line instead of having two
+        // different names when the LLVM name and the Rust name differ.
+        Some(format!("{}{}", &s[..1], to_llvm_feature(sess, feature)))
+    };
+
     // Features implied by an implicit or explicit `--target`.
-    features.extend(
+    features.extend(sess.target.features.split(',').filter_map(&filter));
-        sess.target
-            .features
-            .split(',')
-            .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s)))
-            .map(String::from),
-    );
 
     // -Ctarget-features
-    features.extend(
+    features.extend(sess.opts.cg.target_feature.split(',').filter_map(&filter));
-        sess.opts
-            .cg
-            .target_feature
-            .split(',')
-            .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s)))
-            .map(String::from),
-    );
 
     features
 }

compiler/rustc_codegen_llvm/src/metadata.rs

@@ -1,112 +0,0 @@
-use crate::llvm;
-use crate::llvm::archive_ro::ArchiveRO;
-use crate::llvm::{mk_section_iter, False, ObjectFile};
-use rustc_middle::middle::cstore::MetadataLoader;
-use rustc_target::spec::Target;
-
-use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::owning_ref::OwningRef;
-use rustc_data_structures::rustc_erase_owner;
-use tracing::debug;
-
-use rustc_fs_util::path_to_c_string;
-use std::path::Path;
-use std::slice;
-
-pub use rustc_data_structures::sync::MetadataRef;
-
-pub struct LlvmMetadataLoader;
-
-impl MetadataLoader for LlvmMetadataLoader {
-    fn get_rlib_metadata(&self, _: &Target, filename: &Path) -> Result<MetadataRef, String> {
-        // Use ArchiveRO for speed here, it's backed by LLVM and uses mmap
-        // internally to read the file. We also avoid even using a memcpy by
-        // just keeping the archive along while the metadata is in use.
-        let archive =
-            ArchiveRO::open(filename).map(|ar| OwningRef::new(Box::new(ar))).map_err(|e| {
-                debug!("llvm didn't like `{}`: {}", filename.display(), e);
-                format!("failed to read rlib metadata in '{}': {}", filename.display(), e)
-            })?;
-        let buf: OwningRef<_, [u8]> = archive.try_map(|ar| {
-            ar.iter()
-                .filter_map(|s| s.ok())
-                .find(|sect| sect.name() == Some(METADATA_FILENAME))
-                .map(|s| s.data())
-                .ok_or_else(|| {
-                    debug!("didn't find '{}' in the archive", METADATA_FILENAME);
-                    format!("failed to read rlib metadata: '{}'", filename.display())
-                })
-        })?;
-        Ok(rustc_erase_owner!(buf))
-    }
-
-    fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result<MetadataRef, String> {
-        unsafe {
-            let buf = path_to_c_string(filename);
-            let mb = llvm::LLVMRustCreateMemoryBufferWithContentsOfFile(buf.as_ptr())
-                .ok_or_else(|| format!("error reading library: '{}'", filename.display()))?;
-            let of =
-                ObjectFile::new(mb).map(|of| OwningRef::new(Box::new(of))).ok_or_else(|| {
-                    format!("provided path not an object file: '{}'", filename.display())
-                })?;
-            let buf = of.try_map(|of| search_meta_section(of, target, filename))?;
-            Ok(rustc_erase_owner!(buf))
-        }
-    }
-}
-
-fn search_meta_section<'a>(
-    of: &'a ObjectFile,
-    target: &Target,
-    filename: &Path,
-) -> Result<&'a [u8], String> {
-    unsafe {
-        let si = mk_section_iter(of.llof);
-        while llvm::LLVMIsSectionIteratorAtEnd(of.llof, si.llsi) == False {
-            let mut name_buf = None;
-            let name_len = llvm::LLVMRustGetSectionName(si.llsi, &mut name_buf);
-            let name = name_buf.map_or_else(
-                String::new, // We got a NULL ptr, ignore `name_len`.
-                |buf| {
-                    String::from_utf8(
-                        slice::from_raw_parts(buf.as_ptr() as *const u8, name_len as usize)
-                            .to_vec(),
-                    )
-                    .unwrap()
-                },
-            );
-            debug!("get_metadata_section: name {}", name);
-            if read_metadata_section_name(target) == name {
-                let cbuf = llvm::LLVMGetSectionContents(si.llsi);
-                let csz = llvm::LLVMGetSectionSize(si.llsi) as usize;
-                // The buffer is valid while the object file is around
-                let buf: &'a [u8] = slice::from_raw_parts(cbuf as *const u8, csz);
-                return Ok(buf);
-            }
-            llvm::LLVMMoveToNextSection(si.llsi);
-        }
-    }
-    Err(format!("metadata not found: '{}'", filename.display()))
-}
-
-pub fn metadata_section_name(target: &Target) -> &'static str {
-    // Historical note:
-    //
-    // When using link.exe it was seen that the section name `.note.rustc`
-    // was getting shortened to `.note.ru`, and according to the PE and COFF
-    // specification:
-    //
-    // > Executable images do not use a string table and do not support
-    // > section names longer than 8 characters
-    //
-    // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
-    //
-    // As a result, we choose a slightly shorter name! As to why
-    // `.note.rustc` works on MinGW, that's another good question...
-
-    if target.is_like_osx { "__DATA,.rustc" } else { ".rustc" }
-}
-
-fn read_metadata_section_name(_target: &Target) -> &'static str {
-    ".rustc"
-}

compiler/rustc_codegen_llvm/src/mono_item.rs

@@ -37,7 +37,7 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe {
             llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage));
             llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility));
-            if self.should_assume_dso_local(linkage, visibility) {
+            if self.should_assume_dso_local(g, false) {
                 llvm::LLVMRustSetDSOLocal(g, true);
             }
         }
@@ -85,7 +85,7 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         attributes::from_fn_attrs(self, lldecl, instance);
 
         unsafe {
-            if self.should_assume_dso_local(linkage, visibility) {
+            if self.should_assume_dso_local(lldecl, false) {
                 llvm::LLVMRustSetDSOLocal(lldecl, true);
             }
         }
@@ -95,31 +95,51 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 }
 
 impl CodegenCx<'ll, 'tcx> {
-    /// Whether a definition (NB: not declaration!) can be assumed to be local to a group of
+    /// Whether a definition or declaration can be assumed to be local to a group of
     /// libraries that form a single DSO or executable.
     pub(crate) unsafe fn should_assume_dso_local(
         &self,
-        linkage: Linkage,
+        llval: &llvm::Value,
-        visibility: Visibility,
+        is_declaration: bool,
     ) -> bool {
-        if matches!(linkage, Linkage::Internal | Linkage::Private) {
+        let linkage = llvm::LLVMRustGetLinkage(llval);
+        let visibility = llvm::LLVMRustGetVisibility(llval);
+
+        if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) {
             return true;
         }
 
-        if visibility != Visibility::Default && linkage != Linkage::ExternalWeak {
+        if visibility != llvm::Visibility::Default && linkage != llvm::Linkage::ExternalWeakLinkage
+        {
             return true;
         }
 
+        // Symbols from executables can't really be imported any further.
+        let all_exe = self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable);
+        let is_declaration_for_linker =
+            is_declaration || linkage == llvm::Linkage::AvailableExternallyLinkage;
+        if all_exe && !is_declaration_for_linker {
+            return true;
+        }
+
+        // PowerPC64 prefers TOC indirection to avoid copy relocations.
+        if matches!(&*self.tcx.sess.target.arch, "powerpc64" | "powerpc64le") {
+            return false;
+        }
+
+        // Thread-local variables generally don't support copy relocations.
+        let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval)
+            .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True)
+            .unwrap_or(false);
+        if is_thread_local_var {
+            return false;
+        }
+
         // Static relocation model should force copy relocations everywhere.
         if self.tcx.sess.relocation_model() == RelocModel::Static {
             return true;
         }
 
-        // Symbols from executables can't really be imported any further.
-        if self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) {
-            return true;
-        }
-
         return false;
     }
 }