Skip to content

Commit edf3b27

Browse files
abhina-sreeabhina-sree
authored
[SystemZ][z/OS] Propagate IsText parameter to open text files as text (#107906)
This patch adds an IsText parameter to the following functions openFileForRead, getBufferForFile, getBufferForFileImpl and determines whether a file is text by querying the file tag on z/OS. The default is set to OF_Text instead of OF_None, this change in value does not affect any other platforms other than z/OS.
1 parent f3f3883 commit edf3b27

File tree

22 files changed

+105
-55
lines changed

22 files changed

+105
-55
lines changed

‎clang-tools-extra/clangd/FS.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ PreambleFileStatusCache::getProducingFS(
6464
: ProxyFileSystem(std::move(FS)), StatCache(StatCache) {}
6565

6666
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
67-
openFileForRead(const llvm::Twine &Path) override {
68-
auto File = getUnderlyingFS().openFileForRead(Path);
67+
openFileForRead(const llvm::Twine &Path, bool IsText = true) override {
68+
auto File = getUnderlyingFS().openFileForRead(Path, IsText);
6969
if (!File || !*File)
7070
return File;
7171
// Eagerly stat opened file, as the followup `status` call on the file

‎clang-tools-extra/clangd/Preamble.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -479,9 +479,9 @@ class TimerFS : public llvm::vfs::ProxyFileSystem {
479479
: ProxyFileSystem(std::move(FS)) {}
480480

481481
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
482-
openFileForRead(const llvm::Twine &Path) override {
482+
openFileForRead(const llvm::Twine &Path, bool IsText = true) override {
483483
WallTimerRegion T(Timer);
484-
auto FileOr = getUnderlyingFS().openFileForRead(Path);
484+
auto FileOr = getUnderlyingFS().openFileForRead(Path, IsText);
485485
if (!FileOr)
486486
return FileOr;
487487
return std::make_unique<TimerFile>(Timer, std::move(FileOr.get()));

‎clang-tools-extra/clangd/support/ThreadsafeFS.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class VolatileFileSystem : public llvm::vfs::ProxyFileSystem {
2929
: ProxyFileSystem(std::move(FS)) {}
3030

3131
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
32-
openFileForRead(const llvm::Twine &InPath) override {
32+
openFileForRead(const llvm::Twine &InPath, bool IsText = true) override {
3333
llvm::SmallString<128> Path;
3434
InPath.toVector(Path);
3535

‎clang-tools-extra/clangd/unittests/ClangdTests.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1010,7 +1010,7 @@ TEST(ClangdTests, PreambleVFSStatCache) {
10101010
: ProxyFileSystem(std::move(FS)), CountStats(CountStats) {}
10111011

10121012
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
1013-
openFileForRead(const Twine &Path) override {
1013+
openFileForRead(const Twine &Path, bool IsText = true) override {
10141014
++CountStats[llvm::sys::path::filename(Path.str())];
10151015
return ProxyFileSystem::openFileForRead(Path);
10161016
}

‎clang/include/clang/Basic/FileManager.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -286,21 +286,21 @@ class FileManager : public RefCountedBase<FileManager> {
286286
/// MemoryBuffer if successful, otherwise returning null.
287287
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
288288
getBufferForFile(FileEntryRef Entry, bool isVolatile = false,
289-
bool RequiresNullTerminator = true,
289+
bool RequiresNullTerminator = true, bool IsText = true,
290290
std::optional<int64_t> MaybeLimit = std::nullopt);
291291
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
292292
getBufferForFile(StringRef Filename, bool isVolatile = false,
293-
bool RequiresNullTerminator = true,
293+
bool RequiresNullTerminator = true, bool IsText = true,
294294
std::optional<int64_t> MaybeLimit = std::nullopt) const {
295295
return getBufferForFileImpl(Filename,
296296
/*FileSize=*/(MaybeLimit ? *MaybeLimit : -1),
297-
isVolatile, RequiresNullTerminator);
297+
isVolatile, RequiresNullTerminator, IsText);
298298
}
299299

300300
private:
301301
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
302302
getBufferForFileImpl(StringRef Filename, int64_t FileSize, bool isVolatile,
303-
bool RequiresNullTerminator) const;
303+
bool RequiresNullTerminator, bool IsText) const;
304304

305305
DirectoryEntry *&getRealDirEntry(const llvm::vfs::Status &Status);
306306

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ class DependencyScanningWorkerFilesystem
346346

347347
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
348348
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
349-
openFileForRead(const Twine &Path) override;
349+
openFileForRead(const Twine &Path, bool IsText = true) override;
350350

351351
std::error_code getRealPath(const Twine &Path,
352352
SmallVectorImpl<char> &Output) override;

‎clang/lib/Basic/FileManager.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -530,7 +530,7 @@ void FileManager::fillRealPathName(FileEntry *UFE, llvm::StringRef FileName) {
530530

531531
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
532532
FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
533-
bool RequiresNullTerminator,
533+
bool RequiresNullTerminator, bool IsText,
534534
std::optional<int64_t> MaybeLimit) {
535535
const FileEntry *Entry = &FE.getFileEntry();
536536
// If the content is living on the file entry, return a reference to it.
@@ -558,21 +558,21 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
558558

559559
// Otherwise, open the file.
560560
return getBufferForFileImpl(Filename, FileSize, isVolatile,
561-
RequiresNullTerminator);
561+
RequiresNullTerminator, IsText);
562562
}
563563

564564
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
565565
FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize,
566-
bool isVolatile,
567-
bool RequiresNullTerminator) const {
566+
bool isVolatile, bool RequiresNullTerminator,
567+
bool IsText) const {
568568
if (FileSystemOpts.WorkingDir.empty())
569569
return FS->getBufferForFile(Filename, FileSize, RequiresNullTerminator,
570-
isVolatile);
570+
isVolatile, IsText);
571571

572572
SmallString<128> FilePath(Filename);
573573
FixupRelativePath(FilePath);
574574
return FS->getBufferForFile(FilePath, FileSize, RequiresNullTerminator,
575-
isVolatile);
575+
isVolatile, IsText);
576576
}
577577

578578
/// getStatValue - Get the 'stat' information for the specified path,

‎clang/lib/Basic/SourceManager.cpp

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,18 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
121121
// Start with the assumption that the buffer is invalid to simplify early
122122
// return paths.
123123
IsBufferInvalid = true;
124-
125-
auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile);
124+
bool IsText = false;
125+
126+
#ifdef __MVS__
127+
// If the file is tagged with a text ccsid, it may require autoconversion.
128+
llvm::ErrorOr<bool> IsFileText =
129+
llvm::iszOSTextFile(ContentsEntry->getName().data());
130+
if (IsFileText)
131+
IsText = *IsFileText;
132+
#endif
133+
134+
auto BufferOrError = FM.getBufferForFile(
135+
*ContentsEntry, IsFileVolatile, /*RequiresNullTerminator=*/true, IsText);
126136

127137
// If we were unable to open the file, then we are in an inconsistent
128138
// situation where the content cache referenced a file which no longer

‎clang/lib/Serialization/ASTReader.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5317,7 +5317,8 @@ std::string ASTReader::getOriginalSourceFile(
53175317
const PCHContainerReader &PCHContainerRdr, DiagnosticsEngine &Diags) {
53185318
// Open the AST file.
53195319
auto Buffer = FileMgr.getBufferForFile(ASTFileName, /*IsVolatile=*/false,
5320-
/*RequiresNullTerminator=*/false);
5320+
/*RequiresNullTerminator=*/false,
5321+
/*IsText=*/true);
53215322
if (!Buffer) {
53225323
Diags.Report(diag::err_fe_unable_to_read_pch_file)
53235324
<< ASTFileName << Buffer.getError().message();

‎clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -353,12 +353,13 @@ DepScanFile::create(EntryRef Entry) {
353353
}
354354

355355
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
356-
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
356+
DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path,
357+
bool IsText) {
357358
SmallString<256> OwnedFilename;
358359
StringRef Filename = Path.toStringRef(OwnedFilename);
359360

360361
if (shouldBypass(Filename))
361-
return getUnderlyingFS().openFileForRead(Path);
362+
return getUnderlyingFS().openFileForRead(Path, IsText);
362363

363364
llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
364365
if (!Result)

‎clang/unittests/Driver/DistroTest.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,9 +352,9 @@ TEST(DistroTest, DetectWindowsAndCrossCompile) {
352352
}
353353

354354
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
355-
openFileForRead(const llvm::Twine &Path) override {
355+
openFileForRead(const llvm::Twine &Path, bool IsText = true) override {
356356
++Count;
357-
return llvm::vfs::ProxyFileSystem::openFileForRead(Path);
357+
return llvm::vfs::ProxyFileSystem::openFileForRead(Path, IsText);
358358
}
359359

360360
unsigned Count{};

‎clang/unittests/Driver/ToolChainTest.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -662,7 +662,7 @@ struct FileSystemWithError : public llvm::vfs::FileSystem {
662662
return std::make_error_code(std::errc::no_such_file_or_directory);
663663
}
664664
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
665-
openFileForRead(const Twine &Path) override {
665+
openFileForRead(const Twine &Path, bool IsText = true) override {
666666
return std::make_error_code(std::errc::permission_denied);
667667
}
668668
llvm::vfs::directory_iterator dir_begin(const Twine &Dir,

‎clang/unittests/Frontend/PCHPreambleTest.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ class ReadCountingInMemoryFileSystem : public vfs::InMemoryFileSystem
3636
std::map<std::string, unsigned> ReadCounts;
3737

3838
public:
39-
ErrorOr<std::unique_ptr<vfs::File>> openFileForRead(const Twine &Path) override
40-
{
39+
ErrorOr<std::unique_ptr<vfs::File>>
40+
openFileForRead(const Twine &Path, bool IsText = true) override {
4141
++ReadCounts[Canonicalize(Path)];
42-
return InMemoryFileSystem::openFileForRead(Path);
42+
return InMemoryFileSystem::openFileForRead(Path, IsText);
4343
}
4444

4545
unsigned GetReadCount(const Twine &Path) const

‎clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,9 +277,9 @@ TEST(DependencyScanner, ScanDepsWithModuleLookup) {
277277
}
278278

279279
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
280-
openFileForRead(const Twine &Path) override {
280+
openFileForRead(const Twine &Path, bool IsText = true) override {
281281
ReadFiles.push_back(Path.str());
282-
return ProxyFileSystem::openFileForRead(Path);
282+
return ProxyFileSystem::openFileForRead(Path, IsText);
283283
}
284284
};
285285

‎lldb/unittests/Host/FileSystemTest.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ class DummyFileSystem : public vfs::FileSystem {
5959
return I->second;
6060
}
6161
ErrorOr<std::unique_ptr<vfs::File>>
62-
openFileForRead(const Twine &Path) override {
62+
openFileForRead(const Twine &Path, bool Text = true) override {
6363
auto S = status(Path);
6464
if (S)
6565
return std::unique_ptr<vfs::File>(new DummyFile{*S});

‎lldb/unittests/Utility/MockSymlinkFileSystem.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class MockSymlinkFileSystem : public llvm::vfs::FileSystem {
4343
return llvm::errc::operation_not_permitted;
4444
}
4545
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
46-
openFileForRead(const llvm::Twine &Path) override {
46+
openFileForRead(const llvm::Twine &Path, bool IsText = true) override {
4747
return llvm::errc::operation_not_permitted;
4848
}
4949
llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,

‎llvm/include/llvm/Support/AutoConvert.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ std::error_code restorezOSStdHandleAutoConversion(int FD);
5252
/// \brief Set the tag information for a file descriptor.
5353
std::error_code setzOSFileTag(int FD, int CCSID, bool Text);
5454

55+
// Get the the tag ccsid for a file name or a file descriptor.
56+
ErrorOr<__ccsid_t> getzOSFileTag(const char *FileName, const int FD = -1);
57+
58+
// Query the file tag to determine if the file is a text file.
59+
ErrorOr<bool> iszOSTextFile(const char *Filename, const int FD = -1);
60+
5561
} // namespace llvm
5662
#endif // __cplusplus
5763

‎llvm/include/llvm/Support/VirtualFileSystem.h

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -273,13 +273,14 @@ class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
273273

274274
/// Get a \p File object for the file at \p Path, if one exists.
275275
virtual llvm::ErrorOr<std::unique_ptr<File>>
276-
openFileForRead(const Twine &Path) = 0;
276+
openFileForRead(const Twine &Path, bool IsText = true) = 0;
277277

278278
/// This is a convenience method that opens a file, gets its content and then
279279
/// closes the file.
280280
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
281281
getBufferForFile(const Twine &Name, int64_t FileSize = -1,
282-
bool RequiresNullTerminator = true, bool IsVolatile = false);
282+
bool RequiresNullTerminator = true, bool IsVolatile = false,
283+
bool IsText = true);
283284

284285
/// Get a directory_iterator for \p Dir.
285286
/// \note The 'end' iterator is directory_iterator().
@@ -392,7 +393,7 @@ class OverlayFileSystem : public RTTIExtends<OverlayFileSystem, FileSystem> {
392393
llvm::ErrorOr<Status> status(const Twine &Path) override;
393394
bool exists(const Twine &Path) override;
394395
llvm::ErrorOr<std::unique_ptr<File>>
395-
openFileForRead(const Twine &Path) override;
396+
openFileForRead(const Twine &Path, bool IsText = true) override;
396397
directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
397398
llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
398399
std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
@@ -446,8 +447,8 @@ class ProxyFileSystem : public RTTIExtends<ProxyFileSystem, FileSystem> {
446447
}
447448
bool exists(const Twine &Path) override { return FS->exists(Path); }
448449
llvm::ErrorOr<std::unique_ptr<File>>
449-
openFileForRead(const Twine &Path) override {
450-
return FS->openFileForRead(Path);
450+
openFileForRead(const Twine &Path, bool IsText = true) override {
451+
return FS->openFileForRead(Path, IsText);
451452
}
452453
directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override {
453454
return FS->dir_begin(Dir, EC);
@@ -615,7 +616,7 @@ class InMemoryFileSystem : public RTTIExtends<InMemoryFileSystem, FileSystem> {
615616

616617
llvm::ErrorOr<Status> status(const Twine &Path) override;
617618
llvm::ErrorOr<std::unique_ptr<File>>
618-
openFileForRead(const Twine &Path) override;
619+
openFileForRead(const Twine &Path, bool IsText = true) override;
619620
directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
620621

621622
llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
@@ -1051,7 +1052,8 @@ class RedirectingFileSystem
10511052

10521053
ErrorOr<Status> status(const Twine &Path) override;
10531054
bool exists(const Twine &Path) override;
1054-
ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
1055+
ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path,
1056+
bool IsText = true) override;
10551057

10561058
std::error_code getRealPath(const Twine &Path,
10571059
SmallVectorImpl<char> &Output) override;
@@ -1155,9 +1157,10 @@ class TracingFileSystem
11551157
return ProxyFileSystem::status(Path);
11561158
}
11571159

1158-
ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override {
1160+
ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path,
1161+
bool IsText = true) override {
11591162
++NumOpenFileForReadCalls;
1160-
return ProxyFileSystem::openFileForRead(Path);
1163+
return ProxyFileSystem::openFileForRead(Path, IsText);
11611164
}
11621165

11631166
directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override {

‎llvm/lib/Support/AutoConvert.cpp

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,4 +116,30 @@ std::error_code llvm::setzOSFileTag(int FD, int CCSID, bool Text) {
116116
return std::error_code();
117117
}
118118

119+
ErrorOr<__ccsid_t> llvm::getzOSFileTag(const char *FileName, const int FD) {
120+
// If we have a file descriptor, use it to find out file tagging. Otherwise we
121+
// need to use stat() with the file path.
122+
if (FD != -1) {
123+
struct f_cnvrt Query = {
124+
QUERYCVT, // cvtcmd
125+
0, // pccsid
126+
0, // fccsid
127+
};
128+
if (fcntl(FD, F_CONTROL_CVT, &Query) == -1)
129+
return std::error_code(errno, std::generic_category());
130+
return Query.fccsid;
131+
}
132+
struct stat Attr;
133+
if (stat(FileName, &Attr) == -1)
134+
return std::error_code(errno, std::generic_category());
135+
return Attr.st_tag.ft_ccsid;
136+
}
137+
138+
ErrorOr<bool> llvm::iszOSTextFile(const char *Filename, const int FD) {
139+
ErrorOr<__ccsid_t> Ccsid = getzOSFileTag(Filename, FD);
140+
if (std::error_code EC = Ccsid.getError())
141+
return EC;
142+
return *Ccsid != FT_BINARY;
143+
}
144+
119145
#endif // __MVS__

‎llvm/lib/Support/FileCollector.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,8 @@ class FileCollectorFileSystem : public vfs::FileSystem {
268268
}
269269

270270
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
271-
openFileForRead(const Twine &Path) override {
272-
auto Result = FS->openFileForRead(Path);
271+
openFileForRead(const Twine &Path, bool IsText) override {
272+
auto Result = FS->openFileForRead(Path, IsText);
273273
if (Result && *Result)
274274
Collector->addFile(Path);
275275
return Result;

0 commit comments

Comments
 (0)