executable README (#632)

* executable README

* fix title of CI workflow

* markup commands in markdown

* extend the markup-markdown language

* improvements

* update readme from doc branch

* tab/spc

* fix errors in updown language

* fix errors in updown language, and [skip]: begin/end

* fix errors in updown language, and [skip]: begin/end

* a storied run

* stories run on readme instructions does not need HF token

* increase timeout

* check for hang un hf_login

* skip hf login in case it hangs

* skip checkout

* don't skip login

Author: mikekgfb

.github/workflows/run-readme.yml

@@ -0,0 +1,46 @@
+name: Run the README instructions periodically to ensure they work
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs daily at midnight UTC
+  push:
+    tags:
+      - ciflow/periodic/*
+  workflow_dispatch:
+
+jobs:
+  test-readme:
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    secrets: inherit
+    with:
+      runner: linux.g5.4xlarge.nvidia.gpu
+      secrets-env: "HF_TOKEN_PERIODIC"
+      gpu-arch-type: cuda
+      gpu-arch-version: "12.1"
+      script: |
+        echo "::group::Print machine info"
+        uname -a
+        echo "::endgroup::"
+
+        # echo "::group::Install newer objcopy that supports --set-section-alignment"
+        # yum install -y  devtoolset-10-binutils
+        # export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
+        # echo "::endgroup::"
+
+        # echo "::group::get_llama"
+        # (
+        #     set +x
+        #     HF_TOKEN="${SECRET_HF_TOKEN_PERIODIC}" bash .ci/scripts/download_llama.sh
+        # )
+        # echo "::endgroup::"
+
+        echo "::group::Create script"
+        python3 scripts/process-readme.py > ./we-run-this.sh
+        echo "::endgroup::"
+
+        echo "::group::Run This"
+        bash -x ./we-run-this.sh
+        
+        echo "tests complete"
+        echo "******************************************"
+        echo "::endgroup::"

.github/workflows/run-readme2.yml

@@ -0,0 +1,41 @@
+name: Run the README instructions - with stories - to ensure they work
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test-readme:
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      runner: linux.g5.4xlarge.nvidia.gpu
+      gpu-arch-type: cuda
+      gpu-arch-version: "12.1"
+      timeout: 60
+      script: |
+        echo "::group::Print machine info"
+        uname -a
+        echo "::endgroup::"
+
+        # echo "::group::Install newer objcopy that supports --set-section-alignment"
+        # yum install -y  devtoolset-10-binutils
+        # export PATH=/opt/rh/devtoolset-10/root/usr/bin/:$PATH
+        # echo "::endgroup::"
+
+        echo "::group::Create script"
+        python3 scripts/process-readme.py > ./readme-commands.sh
+        echo "::endgroup::"
+
+        echo "::group::Run This"
+        grep -v login ./readme-commands.sh | sed  -e '1,$s/llama3/stories15M/g' > ./we-run-this.sh
+        echo "******************************************"
+        cat ./we-run-this.sh
+        echo "******************************************"
+        bash -x ./we-run-this.sh
+        
+        echo "tests complete"
+        echo "******************************************"
+        echo "::endgroup::"