Rollup merge of #63525 - matklad:centraliza-file-loading, r=petrochenkov

Make sure that all file loading happens via SourceMap

That way, callers don't need to repeat "let's add this to sm manually
for tracking dependencies" trick.

It should make it easier to switch to using `FileLoader` for binary
files in the future as well

cc #62948

r? @petrochenkov

src/libsyntax/ext/expand.rs

@@ -25,7 +25,6 @@ use syntax_pos::{Span, DUMMY_SP, FileName};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
-use std::fs;
 use std::io::ErrorKind;
 use std::{iter, mem};
 use std::ops::DerefMut;
@@ -1241,13 +1240,11 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                     }
 
                     let filename = self.cx.resolve_path(&*file.as_str(), it.span());
-                    match fs::read_to_string(&filename) {
+                    match self.cx.source_map().load_file(&filename) {
-                        Ok(src) => {
+                        Ok(source_file) => {
-                            let src_interned = Symbol::intern(&src);
+                            let src = source_file.src.as_ref()
-
+                                .expect("freshly loaded file should have a source");
-                            // Add this input file to the code map to make it available as
+                            let src_interned = Symbol::intern(src.as_str());
-                            // dependency information
-                            self.cx.source_map().new_source_file(filename.into(), src);
 
                             let include_info = vec![
                                 ast::NestedMetaItem::MetaItem(

src/libsyntax/source_map.rs

@@ -171,6 +171,26 @@ impl SourceMap {
         Ok(self.new_source_file(filename, src))
     }
 
+    /// Loads source file as a binary blob.
+    ///
+    /// Unlike `load_file`, guarantees that no normalization like BOM-removal
+    /// takes place.
+    pub fn load_binary_file(&self, path: &Path) -> io::Result<Vec<u8>> {
+        // Ideally, this should use `self.file_loader`, but it can't
+        // deal with binary files yet.
+        let bytes = fs::read(path)?;
+
+        // We need to add file to the `SourceMap`, so that it is present
+        // in dep-info. There's also an edge case that file might be both
+        // loaded as a binary via `include_bytes!` and as proper `SourceFile`
+        // via `mod`, so we try to use real file contents and not just an
+        // empty string.
+        let text = std::str::from_utf8(&bytes).unwrap_or("")
+            .to_string();
+        self.new_source_file(path.to_owned().into(), text);
+        Ok(bytes)
+    }
+
     pub fn files(&self) -> MappedLockGuard<'_, Vec<Lrc<SourceFile>>> {
         LockGuard::map(self.files.borrow(), |files| &mut files.source_files)
     }

src/libsyntax_ext/source_util.rs

@@ -9,8 +9,6 @@ use syntax::tokenstream;
 use smallvec::SmallVec;
 use syntax_pos::{self, Pos, Span};
 
-use std::fs;
-use std::io::ErrorKind;
 use rustc_data_structures::sync::Lrc;
 
 // These macros all relate to the file system; they either return
@@ -114,20 +112,17 @@ pub fn expand_include_str(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::To
         None => return DummyResult::any(sp)
     };
     let file = cx.resolve_path(file, sp);
-    match fs::read_to_string(&file) {
+    match cx.source_map().load_binary_file(&file) {
-        Ok(src) => {
+        Ok(bytes) => match std::str::from_utf8(&bytes) {
-            let interned_src = Symbol::intern(&src);
+            Ok(src) => {
-
+                let interned_src = Symbol::intern(&src);
-            // Add this input file to the code map to make it available as
+                base::MacEager::expr(cx.expr_str(sp, interned_src))
-            // dependency information
+            }
-            cx.source_map().new_source_file(file.into(), src);
+            Err(_) => {
-
+                cx.span_err(sp, &format!("{} wasn't a utf-8 file", file.display()));
-            base::MacEager::expr(cx.expr_str(sp, interned_src))
+                DummyResult::any(sp)
+            }
         },
-        Err(ref e) if e.kind() == ErrorKind::InvalidData => {
-            cx.span_err(sp, &format!("{} wasn't a utf-8 file", file.display()));
-            DummyResult::any(sp)
-        }
         Err(e) => {
             cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e));
             DummyResult::any(sp)
@@ -142,18 +137,8 @@ pub fn expand_include_bytes(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::
         None => return DummyResult::any(sp)
     };
     let file = cx.resolve_path(file, sp);
-    match fs::read(&file) {
+    match cx.source_map().load_binary_file(&file) {
         Ok(bytes) => {
-            // Add the contents to the source map if it contains UTF-8.
-            let (contents, bytes) = match String::from_utf8(bytes) {
-                Ok(s) => {
-                    let bytes = s.as_bytes().to_owned();
-                    (s, bytes)
-                },
-                Err(e) => (String::new(), e.into_bytes()),
-            };
-            cx.source_map().new_source_file(file.into(), contents);
-
             base::MacEager::expr(cx.expr_lit(sp, ast::LitKind::ByteStr(Lrc::new(bytes))))
         },
         Err(e) => {

src/test/ui/.gitattributes

@@ -1,2 +1,3 @@
 lexer-crlf-line-endings-string-literal-doc-comment.rs -text
 trailing-carriage-return-in-string.rs -text
+*.bin -text

src/test/ui/include-macros/data.bin

@@ -0,0 +1,2 @@
+This file starts with BOM.
+Lines are separated by \r\n.

src/test/ui/include-macros/normalization.rs

@@ -0,0 +1,12 @@
+// run-pass
+
+fn main() {
+    assert_eq!(
+        &include_bytes!("data.bin")[..],
+        &b"\xEF\xBB\xBFThis file starts with BOM.\r\nLines are separated by \\r\\n.\r\n"[..],
+    );
+    assert_eq!(
+        include_str!("data.bin"),
+        "\u{FEFF}This file starts with BOM.\r\nLines are separated by \\r\\n.\r\n",
+    );
+}