[libc++] Refactors fstream open. (llvm#76617)

This moves the duplicated code to one new function.

This is a preparation to fix
llvm#60509

Author: mordante

libcxx/include/fstream

@@ -279,6 +279,9 @@ public:
 #  endif //  _LIBCPP_STD_VER >= 26
 
   _LIBCPP_HIDE_FROM_ABI inline static const char* __make_mdstring(ios_base::openmode __mode) _NOEXCEPT;
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
+  _LIBCPP_HIDE_FROM_ABI inline static const wchar_t* __make_mdwstring(ios_base::openmode __mode) _NOEXCEPT;
+#  endif
 
 protected:
   // 27.9.1.5 Overridden virtual functions:
@@ -314,6 +317,26 @@ private:
   void __write_mode();
 
   _LIBCPP_EXPORTED_FROM_ABI friend FILE* __get_ostream_file(ostream&);
+
+  // There are multiple (__)open function, they use different C-API open
+  // function. After that call these functions behave the same. This function
+  // does that part and determines the final return value.
+  _LIBCPP_HIDE_FROM_ABI basic_filebuf* __do_open(FILE* __file, ios_base::openmode __mode) {
+    __file_ = __file;
+    if (!__file_)
+      return nullptr;
+
+    __om_ = __mode;
+    if (__mode & ios_base::ate) {
+      if (fseek(__file_, 0, SEEK_END)) {
+        fclose(__file_);
+        __file_ = nullptr;
+        return nullptr;
+      }
+    }
+
+    return this;
+  }
 };
 
 template <class _CharT, class _Traits>
@@ -548,140 +571,93 @@ const char* basic_filebuf<_CharT, _Traits>::__make_mdstring(ios_base::openmode _
   __libcpp_unreachable();
 }
 
+#  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
 template <class _CharT, class _Traits>
-basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
-  basic_filebuf<_CharT, _Traits>* __rt = nullptr;
-  if (__file_ == nullptr) {
-    if (const char* __mdstr = __make_mdstring(__mode)) {
-      __rt    = this;
-      __file_ = fopen(__s, __mdstr);
-      if (__file_) {
-        __om_ = __mode;
-        if (__mode & ios_base::ate) {
-          if (fseek(__file_, 0, SEEK_END)) {
-            fclose(__file_);
-            __file_ = nullptr;
-            __rt    = nullptr;
-          }
-        }
-      } else
-        __rt = nullptr;
-    }
+const wchar_t* basic_filebuf<_CharT, _Traits>::__make_mdwstring(ios_base::openmode __mode) _NOEXCEPT {
+  switch (__mode & ~ios_base::ate) {
+  case ios_base::out:
+  case ios_base::out | ios_base::trunc:
+    return L"w";
+  case ios_base::out | ios_base::app:
+  case ios_base::app:
+    return L"a";
+  case ios_base::in:
+    return L"r";
+  case ios_base::in | ios_base::out:
+    return L"r+";
+  case ios_base::in | ios_base::out | ios_base::trunc:
+    return L"w+";
+  case ios_base::in | ios_base::out | ios_base::app:
+  case ios_base::in | ios_base::app:
+    return L"a+";
+  case ios_base::out | ios_base::binary:
+  case ios_base::out | ios_base::trunc | ios_base::binary:
+    return L"wb";
+  case ios_base::out | ios_base::app | ios_base::binary:
+  case ios_base::app | ios_base::binary:
+    return L"ab";
+  case ios_base::in | ios_base::binary:
+    return L"rb";
+  case ios_base::in | ios_base::out | ios_base::binary:
+    return L"r+b";
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary:
+    return L"w+b";
+  case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
+  case ios_base::in | ios_base::app | ios_base::binary:
+    return L"a+b";
+#    if _LIBCPP_STD_VER >= 23
+  case ios_base::out | ios_base::noreplace:
+  case ios_base::out | ios_base::trunc | ios_base::noreplace:
+    return L"wx";
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::noreplace:
+    return L"w+x";
+  case ios_base::out | ios_base::binary | ios_base::noreplace:
+  case ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
+    return L"wbx";
+  case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
+    return L"w+bx";
+#    endif // _LIBCPP_STD_VER >= 23
+  default:
+    return nullptr;
   }
-  return __rt;
+  __libcpp_unreachable();
+}
+#  endif
+
+template <class _CharT, class _Traits>
+basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const char* __s, ios_base::openmode __mode) {
+  if (__file_)
+    return nullptr;
+  const char* __mdstr = __make_mdstring(__mode);
+  if (!__mdstr)
+    return nullptr;
+
+  return __do_open(fopen(__s, __mdstr), __mode);
 }
 
 template <class _CharT, class _Traits>
 inline basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::__open(int __fd, ios_base::openmode __mode) {
-  basic_filebuf<_CharT, _Traits>* __rt = nullptr;
-  if (__file_ == nullptr) {
-    if (const char* __mdstr = __make_mdstring(__mode)) {
-      __rt    = this;
-      __file_ = fdopen(__fd, __mdstr);
-      if (__file_) {
-        __om_ = __mode;
-        if (__mode & ios_base::ate) {
-          if (fseek(__file_, 0, SEEK_END)) {
-            fclose(__file_);
-            __file_ = nullptr;
-            __rt    = nullptr;
-          }
-        }
-      } else
-        __rt = nullptr;
-    }
-  }
-  return __rt;
+  if (__file_)
+    return nullptr;
+  const char* __mdstr = __make_mdstring(__mode);
+  if (!__mdstr)
+    return nullptr;
+
+  return __do_open(fdopen(__fd, __mdstr), __mode);
 }
 
 #  ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR
 // This is basically the same as the char* overload except that it uses _wfopen
 // and long mode strings.
 template <class _CharT, class _Traits>
 basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::open(const wchar_t* __s, ios_base::openmode __mode) {
-  basic_filebuf<_CharT, _Traits>* __rt = nullptr;
-  if (__file_ == nullptr) {
-    __rt = this;
-    const wchar_t* __mdstr;
-    switch (__mode & ~ios_base::ate) {
-    case ios_base::out:
-    case ios_base::out | ios_base::trunc:
-      __mdstr = L"w";
-      break;
-    case ios_base::out | ios_base::app:
-    case ios_base::app:
-      __mdstr = L"a";
-      break;
-    case ios_base::in:
-      __mdstr = L"r";
-      break;
-    case ios_base::in | ios_base::out:
-      __mdstr = L"r+";
-      break;
-    case ios_base::in | ios_base::out | ios_base::trunc:
-      __mdstr = L"w+";
-      break;
-    case ios_base::in | ios_base::out | ios_base::app:
-    case ios_base::in | ios_base::app:
-      __mdstr = L"a+";
-      break;
-    case ios_base::out | ios_base::binary:
-    case ios_base::out | ios_base::trunc | ios_base::binary:
-      __mdstr = L"wb";
-      break;
-    case ios_base::out | ios_base::app | ios_base::binary:
-    case ios_base::app | ios_base::binary:
-      __mdstr = L"ab";
-      break;
-    case ios_base::in | ios_base::binary:
-      __mdstr = L"rb";
-      break;
-    case ios_base::in | ios_base::out | ios_base::binary:
-      __mdstr = L"r+b";
-      break;
-    case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary:
-      __mdstr = L"w+b";
-      break;
-    case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
-    case ios_base::in | ios_base::app | ios_base::binary:
-      __mdstr = L"a+b";
-      break;
-#    if _LIBCPP_STD_VER >= 23
-    case ios_base::out | ios_base::noreplace:
-    case ios_base::out | ios_base::trunc | ios_base::noreplace:
-      __mdstr = L"wx";
-      break;
-    case ios_base::in | ios_base::out | ios_base::trunc | ios_base::noreplace:
-      __mdstr = L"w+x";
-      break;
-    case ios_base::out | ios_base::binary | ios_base::noreplace:
-    case ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
-      __mdstr = L"wbx";
-      break;
-    case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary | ios_base::noreplace:
-      __mdstr = L"w+bx";
-      break;
-#    endif // _LIBCPP_STD_VER >= 23
-    default:
-      __rt = nullptr;
-      break;
-    }
-    if (__rt) {
-      __file_ = _wfopen(__s, __mdstr);
-      if (__file_) {
-        __om_ = __mode;
-        if (__mode & ios_base::ate) {
-          if (fseek(__file_, 0, SEEK_END)) {
-            fclose(__file_);
-            __file_ = nullptr;
-            __rt    = nullptr;
-          }
-        }
-      } else
-        __rt = nullptr;
-    }
-  }
-  return __rt;
+  if (__file_)
+    return nullptr;
+  const wchar_t* __mdstr = __make_mdwstring(__mode);
+  if (!__mdstr)
+    return nullptr;
+
+  return __do_open(_wfopen(__s, __mdstr), __mode);
 }
 #  endif