Merge branch 'main' into sphinx-tv-tutorial

Author: vfdev-5

.github/scripts/docathon-label-sync.py

@@ -25,11 +25,11 @@ def main():
     issue_number = int(re.findall(r'#(\d{1,5})', pull_request_body)[0])
     issue = repo.get_issue(issue_number)
     issue_labels = issue.labels
-    docathon_label_present = any(label.name == 'docathon-h1-2023' for label in issue_labels)
+    docathon_label_present = any(label.name == 'docathon-h2-2023' for label in issue_labels)
 
     # if the issue has a docathon label, add all labels from the issue to the PR.
     if not docathon_label_present:
-        print("The 'docathon-h1-2023' label is not present in the issue.")
+        print("The 'docathon-h2-2023' label is not present in the issue.")
         return    
     pull_request_labels = pull_request.get_labels()
     issue_label_names = [label.name for label in issue_labels]

.github/workflows/docathon-assign.yml

@@ -9,15 +9,8 @@ jobs:
   assign:
     runs-on: ubuntu-latest
     steps:
-      - name: Install Dependencies
-        uses: actions/setup-node@v3
-        with:
-          node-version: '18'
-      - name: Install @octokit/core
-        run: |
-          npm i @octokit/core @octokit/rest
       - name: Check for "/assigntome" in comment
-        uses: actions/github-script@v4
+        uses: actions/github-script@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
@@ -27,39 +20,39 @@ jobs:
             if (assignRegex.test(issueComment)) {
               const assignee = context.payload.comment.user.login;
               const issueNumber = context.payload.issue.number;
-              const { Octokit } = require("@octokit/rest");
-              const octokit = new Octokit({
-                auth: process.env.GITHUB_TOKEN,
-              });
-              const { data: issue } = await octokit.issues.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: issueNumber
-              });
-              const hasLabel = issue.labels.some(label => label.name === 'docathon-h1-2023');
+              try {
+                const { data: issue } = await github.rest.issues.get({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: issueNumber
+                });
+              const hasLabel = issue.labels.some(label => label.name === 'docathon-h2-2023');
               if (hasLabel) {
                 if (issue.assignee !== null) {
-                  await octokit.issues.createComment({
+                  await github.rest.issues.createComment({
                     owner: context.repo.owner,
                     repo: context.repo.repo,
                     issue_number: issueNumber,
-                    body: "The issue is already assigned. Please pick an opened and unnasigned issue with the [docathon-h1-2023 label](https://github.com/pytorch/tutorials/issues?q=is%3Aopen+is%3Aissue+label%3Adocathon-h1-2023)."
+                    body: "The issue is already assigned. Please pick an opened and unnasigned issue with the [docathon-h2-2023 label](https://github.com/pytorch/pytorch/issues?q=is%3Aopen+is%3Aissue+label%3Adocathon-h2-2023)."
                   });
                 } else {
-                  octokit.issues.addAssignees({
+                  await github.rest.issues.addAssignees({
                     owner: context.repo.owner,
                     repo: context.repo.repo,
                     issue_number: issueNumber,
                     assignees: [assignee]
                   });
                 }
               } else {
-                const commmentMessage = "This issue does not have the correct label. Please pick an opened and unnasigned issue with the [docathon-h1-2023 label](https://github.com/pytorch/tutorials/issues?q=is%3Aopen+is%3Aissue+label%3Adocathon-h1-2023)."
-                await octokit.issues.createComment({
+                const commmentMessage = "This issue does not have the correct label. Please pick an opened and unnasigned issue with the [docathon-h2-2023 label](https://github.com/pytorch/pytorch/issues?q=is%3Aopen+is%3Aissue+label%3Adocathon-h2-2023)."
+                await github.rest.issues.createComment({
                   owner: context.repo.owner,
                   repo: context.repo.repo,
                   issue_number: issueNumber,
                   body: commmentMessage
                 });
+               }
+              } catch (error) {
+                console.error(error);
               }
             }

.jenkins/validate_tutorials_built.py

@@ -26,6 +26,7 @@
     "intermediate_source/parametrizations",
     "intermediate_source/mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
     "intermediate_source/fx_conv_bn_fuser",
+    "intermediate_source/_torch_export_nightly_tutorial",  # does not work on release
     "advanced_source/super_resolution_with_onnxruntime",
     "advanced_source/ddp_pipeline",  # requires 4 gpus
     "prototype_source/fx_graph_mode_ptq_dynamic",

README.md

@@ -5,7 +5,16 @@ All the tutorials are now presented as sphinx style documentation at:
 
 ## [https://pytorch.org/tutorials](https://pytorch.org/tutorials)
 
+# Asking a question
 
+If you have a question about a tutorial, post in https://dev-discuss.pytorch.org/ rather than creating an issue in this repo. Your question will be answered much faster on the dev-discuss forum.
+
+# Submitting an issue
+
+You can submit the following types of issues:
+
+* Feature request - request a new tutorial to be added. Please explain why this tutorial is needed and how it demonstrates PyTorch value.
+* Bug report - report a failiure or outdated information in an existing tutorial. When submitting a bug report, please run: `python3 -m torch.utils.collect_env` to get information about your environment and add the output to the bug report.
 
 # Contributing
 

advanced_source/static_quantization_tutorial.rst

@@ -59,7 +59,7 @@ to enable quantization:
 - Replace ReLU6 with ReLU 
 
 Note: this code is taken from 
-`here <https://github.com/pytorch/vision/blob/master/torchvision/models/mobilenet.py>`_.  
+`here <https://github.com/pytorch/vision/blob/main/torchvision/models/mobilenetv2.py>`_.
 
 .. code:: python
 

advanced_source/super_resolution_with_onnxruntime.py

@@ -26,7 +26,7 @@
 .. code-block:: bash
 
    %%bash
-   pip install onnxruntime
+   pip install onnx onnxruntime
 
 ONNX Runtime recommends using the latest stable runtime for PyTorch.
 

beginner_source/blitz/autograd_tutorial.py

@@ -321,3 +321,4 @@
 #
 # -  `In-place operations & Multithreaded Autograd <https://pytorch.org/docs/stable/notes/autograd.html>`__
 # -  `Example implementation of reverse-mode autodiff <https://colab.research.google.com/drive/1VpeE6UvEPRz9HmsHh1KS0XxXjYu533EC>`__
+# -  `Video: PyTorch Autograd Explained - In-depth Tutorial <https://www.youtube.com/watch?v=MswxJw-8PvE>`__

beginner_source/introyt/tensors_deeper_tutorial.py

@@ -378,7 +378,7 @@
 #
 # -  The multiplication operation that created ``b`` was 
 #    broadcast over every “layer” of ``a``.
-# -  For ``c``, the operation was broadcast over ever layer and row of
+# -  For ``c``, the operation was broadcast over every layer and row of
 #    ``a`` - every 3-element column is identical. 
 # -  For ``d``, we switched it around - now every *row* is identical,
 #    across layers and columns.

conf.py

@@ -106,7 +106,8 @@ def reset_seeds(gallery_conf, fname):
     'first_notebook_cell': ("# For tips on running notebooks in Google Colab, see\n"
                             "# https://pytorch.org/tutorials/beginner/colab\n"
                             "%matplotlib inline"),
-    'reset_modules': (reset_seeds)
+    'reset_modules': (reset_seeds),
+    'ignore_pattern': r'_torch_export_nightly_tutorial.py'
 }
 
 if os.getenv('GALLERY_PATTERN'):

en-wordlist.txt

@@ -55,6 +55,7 @@ CPUs
 CPython
 CUDA
 Caffe
+callable's
 Captum
 Captum's
 CartPole
@@ -75,6 +76,8 @@ DQN
 DataPipe
 DataPipes
 DataLoaders
+Decompositions
+decompositions
 DeepMind
 DeiT
 DenseNet
@@ -164,6 +167,8 @@ ONNX Runtime
 ONNX Script
 OpenAI
 OpenMP
+Opset
+opset
 Ornstein
 OU
 PIL

intermediate_source/FSDP_adavnced_tutorial.rst

@@ -74,8 +74,8 @@ summarization using UncycloHow dataset.  The main focus of this tutorial is to
 highlight different available features in FSDP that are helpful for training
 large scale model above 3B parameters. Also, we cover specific features for
 Transformer based models. The code for this tutorial is available in  `Pytorch
-Examples
-<https://github.com/HamidShojanazeri/examples/tree/FSDP_example/distributed/FSDP/>`__.
+examples
+<https://github.com/pytorch/examples/tree/main/distributed/FSDP/>`__.
 
 
 *Setup*
@@ -97,13 +97,13 @@ Please create a `data` folder, download the UncycloHow dataset from `wikihowAll.csv
 `wikihowSep.cs <https://ucsb.app.box.com/s/7yq601ijl1lzvlfu4rjdbbxforzd2oag>`__,
 and place them in the `data` folder.  We will use the wikihow dataset from
 `summarization_dataset
-<https://github.com/HamidShojanazeri/examples/blob/FSDP_example/distributed/FSDP/summarization_dataset.py>`__.
+<https://github.com/pytorch/examples/blob/main/distributed/FSDP/summarization_dataset.py>`__.
 
 Next, we add the following code snippets to a Python script “T5_training.py”.
 
 .. note::
    The full source code for this tutorial is available in `PyTorch examples
-   <https://github.com/HamidShojanazeri/examples/tree/FSDP_example/distributed/FSDP>`__.
+   <https://github.com/pytorch/examples/tree/main/distributed/FSDP/>`__.
 
 1.3  Import necessary packages:
 

intermediate_source/FSDP_tutorial.rst

@@ -8,7 +8,7 @@ Getting Started with Fully Sharded Data Parallel(FSDP)
 
 Training AI models at a large scale is a challenging task that requires a lot of compute power and resources. 
 It also comes with considerable engineering complexity to handle the training of these very large models.
-`Pytorch FSDP <https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api/>`__, released in PyTorch 1.11 makes this easier.
+`PyTorch FSDP <https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api/>`__, released in PyTorch 1.11 makes this easier.
 
 In this tutorial, we show how to use `FSDP APIs <https://pytorch.org/docs/1.11/fsdp.html>`__, for simple MNIST models that can be extended to other larger models such as `HuggingFace BERT models <https://huggingface.co/blog/zero-deepspeed-fairscale>`__, 
 `GPT 3 models up to 1T parameters <https://pytorch.medium.com/training-a-1-trillion-parameter-model-with-pytorch-fully-sharded-data-parallel-on-aws-3ac13aa96cff>`__ . The sample DDP MNIST code has been borrowed from `here <https://github.com/yqhu/mnist_examples>`__. 
@@ -18,7 +18,7 @@ How FSDP works
 --------------
 In `DistributedDataParallel <https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html>`__, (DDP) training, each process/ worker owns a replica of the model and processes a batch of data, finally it uses all-reduce to sum up gradients over different workers. In DDP the model weights and optimizer states are replicated across all workers. FSDP is a type of data parallelism that shards model parameters, optimizer states and gradients across DDP ranks. 
 
-FSDP GPU memory footprint would be smaller than DDP across all workers. This makes the training of some very large models feasible and helps to fit larger models or batch sizes for our training job. This would come with the cost of increased communication volume. The communication overhead is reduced by internal optimizations like communication and computation overlapping.
+When training with FSDP, the GPU memory footprint is smaller than when training with DDP across all workers. This makes the training of some very large models feasible by allowing larger models or batch sizes to fit on device. This comes with the cost of increased communication volume. The communication overhead is reduced by internal optimizations like overlapping communication and computation.
 
 .. figure:: /_static/img/distributed/fsdp_workflow.png
    :width: 100%
@@ -27,7 +27,7 @@ FSDP GPU memory footprint would be smaller than DDP across all workers. This mak
 
    FSDP Workflow
 
-At high level FSDP works as follow:
+At a high level FSDP works as follow:
 
 *In constructor*
 
@@ -48,11 +48,11 @@ At high level FSDP works as follow:
 
 How to use FSDP
 --------------
-Here we use a toy model to run training on MNIST dataset for demonstration purposes. Similarly the APIs and logic can be applied to larger models for training. 
+Here we use a toy model to run training on the MNIST dataset for demonstration purposes. The APIs and logic can be applied to training larger models as well. 
 
 *Setup*
 
-1.1 Install Pytorch along with Torchvision
+1.1 Install PyTorch along with Torchvision
 
 .. code-block:: bash 
 
@@ -139,7 +139,7 @@ We add the following code snippets to a python script “FSDP_mnist.py”.
             output = F.log_softmax(x, dim=1)
             return output
 
-2.2 define a train function 
+2.2 Define a train function 
 
 .. code-block:: python
 
@@ -189,7 +189,7 @@ We add the following code snippets to a python script “FSDP_mnist.py”.
 
 2.4 Define a distributed train function that wraps the model in FSDP
 
-**Note: to save the FSDP model, we need to call the state_dict on each rank then on Rank 0 save the overall states. This is only available in Pytorch nightlies, current Pytorch release is 1.11 at the moment.**
+**Note: to save the FSDP model, we need to call the state_dict on each rank then on Rank 0 save the overall states.**
 
 .. code-block:: python
 
@@ -250,7 +250,6 @@ We add the following code snippets to a python script “FSDP_mnist.py”.
         if args.save_model:
             # use a barrier to make sure training is done on all ranks
             dist.barrier()
-            # state_dict for FSDP model is only available on Nightlies for now
             states = model.state_dict()
             if rank == 0:
                 torch.save(states, "mnist_cnn.pt")
@@ -259,7 +258,7 @@ We add the following code snippets to a python script “FSDP_mnist.py”.
 
 
 
-2.5 Finally parsing the arguments and setting the main function
+2.5 Finally parse the arguments and set the main function
 
 .. code-block:: python
 
@@ -319,7 +318,7 @@ Alternatively, we will look at adding the fsdp_auto_wrap_policy next and will di
     )
  )
 
-Following is the peak memory usage from FSDP MNIST training on g4dn.12.xlarge AWS EC2 instance with 4 gpus captured from Pytorch Profiler. 
+The following is the peak memory usage from FSDP MNIST training on g4dn.12.xlarge AWS EC2 instance with 4 GPUs captured from PyTorch Profiler. 
 
 
 .. figure:: /_static/img/distributed/FSDP_memory.gif
@@ -329,7 +328,7 @@ Following is the peak memory usage from FSDP MNIST training on g4dn.12.xlarge AW
 
    FSDP Peak Memory Usage
 
-*Applying fsdp_auto_wrap_policy* in FSDP otherwise, FSDP will put the entire model in one FSDP unit, which will reduce computation efficiency and memory efficiency. 
+Applying *fsdp_auto_wrap_policy* in FSDP otherwise, FSDP will put the entire model in one FSDP unit, which will reduce computation efficiency and memory efficiency. 
 The way it works is that, suppose your model contains 100 Linear layers. If you do FSDP(model), there will only be one FSDP unit which wraps the entire model. 
 In that case, the allgather would collect the full parameters for all 100 linear layers, and hence won't save CUDA memory for parameter sharding.
 Also, there is only one blocking allgather call for the all 100 linear layers, there will not be communication and computation overlapping between layers. 
@@ -354,7 +353,7 @@ Finding an optimal auto wrap policy is challenging, PyTorch will add auto tuning
     model = FSDP(model,
         fsdp_auto_wrap_policy=my_auto_wrap_policy)
 
-Applying the FSDP_auto_wrap_policy, the model would be as follows:
+Applying the fsdp_auto_wrap_policy, the model would be as follows:
 
 .. code-block:: bash
 
@@ -381,7 +380,7 @@ Applying the FSDP_auto_wrap_policy, the model would be as follows:
 
     CUDA event elapsed time on training loop 41.89130859375sec
 
-Following is the peak memory usage from FSDP with auto_wrap policy of MNIST training on g4dn.12.xlarge AWS EC2 instance with 4 gpus captured from Pytorch Profiler. 
+The following is the peak memory usage from FSDP with auto_wrap policy of MNIST training on a g4dn.12.xlarge AWS EC2 instance with 4 GPUs captured from PyTorch Profiler. 
 It can be observed that the peak memory usage on each device is smaller compared to FSDP without auto wrap policy applied, from ~75 MB to 66 MB.
 
 .. figure:: /_static/img/distributed/FSDP_autowrap.gif
@@ -391,11 +390,11 @@ It can be observed that the peak memory usage on each device is smaller compared
 
    FSDP Peak Memory Usage using Auto_wrap policy
 
-*CPU Off-loading*: In case the model is very large that even with FSDP wouldn't fit into gpus, then CPU offload can be helpful here. 
+*CPU Off-loading*: In case the model is very large that even with FSDP wouldn't fit into GPUs, then CPU offload can be helpful here. 
 
 Currently, only parameter and gradient CPU offload is supported. It can be enabled via passing in cpu_offload=CPUOffload(offload_params=True).
 
-Note that this currently implicitly enables gradient offloading to CPU in order for params and grads to be on the same device to work with the optimizer. This API is subject to change. Default is None in which case there will be no offloading.
+Note that this currently implicitly enables gradient offloading to CPU in order for params and grads to be on the same device to work with the optimizer. This API is subject to change. The default is None in which case there will be no offloading.
 
 Using this feature may slow down the training considerably, due to frequent copying of tensors from host to device, but it could help improve memory efficiency and train larger scale models. 
 
@@ -409,7 +408,7 @@ In 2.4 we just add it to the FSDP wrapper
         cpu_offload=CPUOffload(offload_params=True))
 
 
-Compare it with DDP, if in 2.4 we just normally wrap the model in ddp, saving the changes in “DDP_mnist.py”.
+Compare it with DDP, if in 2.4 we just normally wrap the model in DPP, saving the changes in “DDP_mnist.py”.
 
 .. code-block:: python
 
@@ -423,7 +422,7 @@ Compare it with DDP, if in 2.4 we just normally wrap the model in ddp, saving th
 
     CUDA event elapsed time on training loop 39.77766015625sec
 
-Following is the peak memory usage from DDP MNIST training on g4dn.12.xlarge AWS EC2 instance with 4 gpus captured from Pytorch profiler. 
+The following is the peak memory usage from DDP MNIST training on g4dn.12.xlarge AWS EC2 instance with 4 GPUs captured from PyTorch profiler. 
 
 .. figure:: /_static/img/distributed/DDP_memory.gif
    :width: 100%
@@ -434,8 +433,8 @@ Following is the peak memory usage from DDP MNIST training on g4dn.12.xlarge AWS
 
 
 Considering the toy example and tiny MNIST model we defined here, we can observe the difference between peak memory usage of DDP and FSDP. 
-In DDP each process holds a replica of the model, so the memory footprint is higher compared to FSDP that shards the model parameter, optimizer states and gradients over DDP ranks.
+In DDP each process holds a replica of the model, so the memory footprint is higher compared to FSDP which shards the model parameters, optimizer states and gradients over DDP ranks.
 The peak memory usage using FSDP with auto_wrap policy is the lowest followed by FSDP and DDP. 
 
-Also, looking at timings, considering the small model and running the training on a single machine, FSDP with/out auto_wrap policy performed almost as fast as DDP.
+Also, looking at timings, considering the small model and running the training on a single machine, FSDP with and without auto_wrap policy performed almost as fast as DDP.
 This example does not represent most of the real applications, for detailed analysis and comparison between DDP and FSDP please refer to this `blog post  <https://pytorch.medium.com/6c8da2be180d>`__ .