Skip to content

Use rsync for remote-run. #41125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 3, 2022
Merged

Use rsync for remote-run. #41125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 10 additions & 16 deletions test/remote-run/custom-options.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,24 @@ RUN: %remote-run -n --remote-dir /xyz-REMOTE -o FIRST_OPT -o SECOND_OPT --output
CHECK: /usr/bin/ssh -n
CHECK-DAG: -p 12345
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/rm' '-rf' '{{.+}}-REMOTE/output'
CHECK-SAME: some_user@some_host -- '/usr/bin/env'
CHECK-SAME: '/bin/rm' '-rf' '{{.+}}-REMOTE/output'

CHECK-NEXT: /usr/bin/ssh -n
CHECK-DAG: -p 12345
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'

CHECK-NEXT: /usr/bin/sftp
CHECK-DAG: -P 12345
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
CHECK-SAME: some_user@some_host
CHECK-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
CHECK-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
CHECK-SAME: some_user@some_host -- '/usr/bin/env'
CHECK-SAME: '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'

CHECK: /usr/bin/ssh -n
CHECK-DAG: -p 12345
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
CHECK-SAME: some_user@some_host -- '/usr/bin/env' 'cp'
CHECK-SAME: some_user@some_host -- '/usr/bin/env'
CHECK-SAME: 'cp'

CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}

CHECK-NEXT: /usr/bin/sftp
CHECK-DAG: -P 12345
CHECK-DAG: -o FIRST_OPT -o SECOND_OPT
CHECK-NEXT: /usr/bin/rsync
CHECK-DAG: '-p' '12345'
CHECK-DAG: '-o' 'FIRST_OPT' '-o' 'SECOND_OPT'
CHECK-SAME: some_user@some_host
CHECK-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
CHECK-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'
25 changes: 11 additions & 14 deletions test/remote-run/dry-run-remote.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,22 @@
RUN: %remote-run -n --remote-dir /xyz-REMOTE --input-prefix %S/Inputs/ some_user@some_host ls %S/Inputs/upload/1.txt %S/Inputs/upload/2.txt 2>&1 >/dev/null | %FileCheck -check-prefix CHECK-INPUT %s

CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/input/upload'
CHECK-INPUT-NEXT: /usr/bin/sftp
CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/input'
CHECK-INPUT-NEXT: /usr/bin/rsync
CHECK-INPUT-SAME: some_user@some_host
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/1.txt' '/xyz-REMOTE/input/upload/1.txt'
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/2.txt' '/xyz-REMOTE/input/upload/2.txt'
CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' 'ls'

CHECK-INPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' {{.*}}'ls'

RUN: %empty-directory(%t)
RUN: %empty-directory(%t/nested)
RUN: touch %t/nested/input %t/nested/BAD
RUN: %remote-run -n --remote-dir /xyz-REMOTE --output-prefix %/t some_user@some_host cp %/t/nested/input %/t/nested/output 2>&1 >/dev/null | %FileCheck -check-prefix CHECK-OUTPUT %s

CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
CHECK-OUTPUT-NEXT: /usr/bin/sftp
CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output'
CHECK-OUTPUT-NEXT: /usr/bin/rsync
CHECK-OUTPUT-SAME: some_user@some_host
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' 'cp'
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
CHECK-OUTPUT-NEXT: /usr/bin/sftp

CHECK-OUTPUT: /usr/bin/ssh -n some_user@some_host -- '/usr/bin/env' {{.*}}'cp'
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}
CHECK-OUTPUT-NEXT: /usr/bin/rsync
CHECK-OUTPUT-SAME: some_user@some_host
CHECK-OUTPUT-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
CHECK-OUTPUT-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'

21 changes: 8 additions & 13 deletions test/remote-run/dry-run.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,9 @@ RUN: %debug-remote-run -n --input-prefix %S/Inputs/ ls %S/Inputs/upload/1.txt %S
RUN: test -z "`ls %t`"
RUN: test ! -e %t-REMOTE

CHECK-INPUT: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input/upload
CHECK-INPUT-NEXT: /usr/bin/sftp
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/1.txt' '{{.+}}-REMOTE/input/upload/1.txt'
CHECK-INPUT-DAG: -put '{{.+}}/Inputs/upload/2.txt' '{{.+}}-REMOTE/input/upload/2.txt'
CHECK-INPUT: /usr/bin/env ls
CHECK-INPUT: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input
CHECK-INPUT-NEXT: /usr/bin/rsync
CHECK-INPUT: /usr/bin/env {{.*}}ls

RUN: %empty-directory(%t)
RUN: %empty-directory(%t/nested)
Expand All @@ -18,11 +16,8 @@ RUN: %debug-remote-run -n --output-prefix %t cp %t/nested/input %t/nested/output
RUN: test ! -e %t-REMOTE

CHECK-OUTPUT: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/output/nested
CHECK-OUTPUT-NEXT: /usr/bin/sftp
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/output' '{{.+}}-REMOTE/output/nested/output'
CHECK-OUTPUT-DAG: -put '{{.+}}/nested/input' '{{.+}}-REMOTE/output/nested/input'
CHECK-OUTPUT: /usr/bin/env cp
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
CHECK-OUTPUT-NEXT: /usr/bin/sftp
CHECK-OUTPUT-DAG: -get '{{.+}}-REMOTE/output/nested/output' '{{.+}}/nested/output'
CHECK-OUTPUT-DAG: -get '{{.+}}-REMOTE/output/nested/input' '{{.+}}/nested/input'
CHECK-OUTPUT: /usr/bin/rsync
CHECK-OUTPUT: /usr/bin/env {{.*}}cp
CHECK-OUTPUT-NEXT: {{^}}/bin/mkdir -p {{.+}}
CHECK-OUTPUT-NEXT: /usr/bin/rsync

2 changes: 1 addition & 1 deletion test/remote-run/env.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ RUN: env REMOTE_RUN_CHILD_FOO=foo REMOTE_RUN_CHILD_BAR=bar %debug-remote-run -v

CHECK: {{^:foo: :bar:$}}

VERBOSE: /usr/bin/env BAR=bar FOO=foo sh -c echo ":${FOO}:" ":${BAR}:"
VERBOSE: /usr/bin/env BAR=bar FOO=foo {{.*}}sh -c echo ":${FOO}:" ":${BAR}:"
31 changes: 13 additions & 18 deletions test/remote-run/identity.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,36 +8,31 @@ CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/rm' '-rf' '{{.+}}-REMOTE
CHECK-NEXT: /usr/bin/ssh -n
CHECK-DAG: -p 12345
CHECK-DAG: -i spiderman
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'

CHECK-NEXT: /usr/bin/sftp
CHECK-DAG: -P 12345
CHECK-DAG: -i spiderman
CHECK-SAME: some_user@some_host
CHECK-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
CHECK-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
CHECK-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}'

CHECK: /usr/bin/ssh -n
CHECK-DAG: -p 12345
CHECK-DAG: -i spiderman
CHECK-SAME: some_user@some_host -- '/usr/bin/env' 'cp'
CHECK-SAME: some_user@some_host -- '/usr/bin/env' {{.*}}'cp'

CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}

CHECK-NEXT: /usr/bin/sftp
CHECK-DAG: -P 12345
CHECK-DAG: -i spiderman
CHECK-NEXT: /usr/bin/rsync
CHECK-DAG: '-p' '12345'
CHECK-DAG: '-i' 'spiderman'
CHECK-SAME: some_user@some_host
CHECK-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
CHECK-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'

# Make sure things work without a port.
RUN: %remote-run -n --remote-dir /xyz-REMOTE -i spiderman --output-prefix %/t some_user@some_host cp %/t/nested/input %/t/nested/output 2>&1 >/dev/null | %FileCheck -check-prefix CHECK-PORTLESS %s

CHECK-PORTLESS: /usr/bin/sftp
CHECK-PORTLESS: /usr/bin/ssh -n
CHECK-PORTLESS-SAME: -i spiderman
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' '/bin/rm' '-rf' '{{.+}}'

CHECK-PORTLESS: /usr/bin/ssh -n
CHECK-PORTLESS-SAME: -i spiderman
CHECK-PORTLESS-SAME: some_user@some_host
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}'

CHECK-PORTLESS: /usr/bin/ssh -n
CHECK-PORTLESS-SAME: -i spiderman
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' 'cp'
CHECK-PORTLESS-SAME: some_user@some_host -- '/usr/bin/env' {{.*}}'cp'
2 changes: 1 addition & 1 deletion test/remote-run/lit.local.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
config.substitutions = list(config.substitutions)

config.substitutions.insert(0, ('%debug-remote-run',
r'%r %%utils/remote-run --debug-as-local %%sftp-server --remote-dir %%t-REMOTE' % (sys.executable,)))
r'%r %%utils/remote-run --debug-as-local --remote-dir %%t-REMOTE' % (sys.executable,)))
config.substitutions.insert(0, ('%remote-run',
r'%r %%utils/remote-run' % (sys.executable,)))
15 changes: 4 additions & 11 deletions test/remote-run/port.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,8 @@
RUN: %remote-run -n --remote-dir /xyz-REMOTE --output-prefix %/t some_user@some_host:12345 cp %/t/nested/input %/t/nested/output 2>&1 >/dev/null | %FileCheck %s

CHECK: /usr/bin/ssh -n -p 12345 some_user@some_host -- '/usr/bin/env' '/bin/mkdir' '-p' '{{.+}}-REMOTE/output/nested'
CHECK-NEXT: /usr/bin/sftp
CHECK-SAME: -P 12345
CHECK: /usr/bin/ssh -n -p 12345 some_user@some_host -- '/usr/bin/env' {{.*}}'cp'
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}
CHECK-NEXT: /usr/bin/rsync
CHECK-SAME: '-p' '12345'
CHECK-SAME: some_user@some_host
CHECK-DAG: -put '{{.+}}/nested/output' '/xyz-REMOTE/output/nested/output'
CHECK-DAG: -put '{{.+}}/nested/input' '/xyz-REMOTE/output/nested/input'
CHECK: /usr/bin/ssh -n -p 12345 some_user@some_host -- '/usr/bin/env' 'cp'
CHECK-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
CHECK-NEXT: /usr/bin/sftp
CHECK-SAME: -P 12345
CHECK-SAME: some_user@some_host
CHECK-DAG: -get '/xyz-REMOTE/output/nested/output' '{{.+}}/nested/output'
CHECK-DAG: -get '/xyz-REMOTE/output/nested/input' '{{.+}}/nested/input'
2 changes: 1 addition & 1 deletion test/remote-run/run-only.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ RUN: %debug-remote-run -v echo hello 2>&1 >/dev/null | %FileCheck -check-prefix

CHECK: {{^hello$}}

VERBOSE: /usr/bin/env echo hello
VERBOSE: /usr/bin/env {{.*}}echo hello
18 changes: 7 additions & 11 deletions test/remote-run/upload-and-download.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,26 +27,22 @@ RUN: ls %t-REMOTE/output/nested/ | %FileCheck -check-prefix CHECK-REMOTE %s
RUN: %debug-remote-run -v --output-prefix %t cp %t/nested/input %t/nested/output 2>&1 >/dev/null | %FileCheck -check-prefix VERBOSE %s

VERBOSE: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/output/nested
VERBOSE-NEXT: /usr/bin/sftp
VERBOSE-DAG: -put '{{.+}}/nested/output' '{{.+}}-REMOTE/output/nested/output'
VERBOSE-DAG: -put '{{.+}}/nested/input' '{{.+}}-REMOTE/output/nested/input'
VERBOSE: /usr/bin/env cp
VERBOSE-NEXT: {{^}}/bin/mkdir -p {{.+}}/nested
VERBOSE-NEXT: /usr/bin/sftp
VERBOSE-DAG: -get '{{.+}}-REMOTE/output/nested/output' '{{.+}}/nested/output'
VERBOSE-DAG: -get '{{.+}}-REMOTE/output/nested/input' '{{.+}}/nested/input'
VERBOSE: /usr/bin/rsync
VERBOSE: /usr/bin/env {{.*}}cp
VERBOSE-NEXT: {{^}}/bin/mkdir -p {{.+}}
VERBOSE-NEXT: /usr/bin/rsync

RUN: %empty-directory(%t)
RUN: touch %t/xyz-1before
RUN: %debug-remote-run --output-prefix %t/xyz cp %t/xyz-1before %t/xyz-2after
RUN: ls %t | %FileCheck -check-prefix CHECK-PREFIXED %s
RUN: ls %t-REMOTE/ | %FileCheck -check-prefix CHECK-PREFIXED-REMOTE %s
RUN: ls %t-REMOTE/output | %FileCheck -check-prefix CHECK-PREFIXED-REMOTE %s

CHECK-PREFIXED: {{^xyz-1before$}}
CHECK-PREFIXED: {{^xyz-2after$}}

CHECK-PREFIXED-REMOTE: {{^output-1before$}}
CHECK-PREFIXED-REMOTE: {{^output-2after$}}
CHECK-PREFIXED-REMOTE: {{^xyz-1before$}}
CHECK-PREFIXED-REMOTE: {{^xyz-2after$}}

RUN: %empty-directory(%t)
RUN: %empty-directory(%t-REMOTE)
Expand Down
2 changes: 1 addition & 1 deletion test/remote-run/upload-stderr.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ REQUIRES: sftp_server
RUN: %empty-directory(%t)
RUN: %empty-directory(%t/REMOTE/input)
RUN: chmod a-w %t/REMOTE/input
RUN: %debug-remote-run --remote-dir %t/REMOTE --input-prefix %S/Inputs/upload/ true -- %S/Inputs/upload/1.txt 2>&1 | %FileCheck %s
RUN: %debug-remote-run --ignore-rsync-failure --remote-dir %t/REMOTE --input-prefix %S/Inputs/upload/ true -- %S/Inputs/upload/1.txt 2>&1 | %FileCheck %s

CHECK: Permission denied
12 changes: 5 additions & 7 deletions test/remote-run/upload.test-sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ RUN: %debug-remote-run -v --input-prefix %S/Inputs/ ls %S/Inputs/upload/1.txt %S

RUN: %empty-directory(%t-REMOTE)
RUN: %debug-remote-run --input-prefix %S/Inputs/upload/1 ls %S/Inputs/upload/1.txt | %FileCheck -check-prefix CHECK-REMOTE-SINGLE-FILE %s
RUN: test -f %t-REMOTE/input.txt
RUN: test -f %t-REMOTE/input/1.txt

RUN: %empty-directory(%t-REMOTE)
RUN: %debug-remote-run --input-prefix %S/Inputs/upload/ --remote-input-prefix custom-input ls %S/Inputs/upload/1.txt %S/Inputs/upload/2.txt | %FileCheck -check-prefix CHECK-REMOTE-CUSTOM %s
Expand All @@ -22,7 +22,7 @@ CHECK-REMOTE-NEXT: {{-REMOTE/input/2.txt$}}
CHECK-REMOTE-NESTED: {{-REMOTE/input/upload/1.txt$}}
CHECK-REMOTE-NESTED-NEXT: {{-REMOTE/input/upload/2.txt$}}

CHECK-REMOTE-SINGLE-FILE: {{-REMOTE/input.txt$}}
CHECK-REMOTE-SINGLE-FILE: {{-REMOTE/input/1.txt$}}

CHECK-REMOTE-CUSTOM: {{-REMOTE/custom-input/1.txt$}}
CHECK-REMOTE-CUSTOM-NEXT: {{-REMOTE/custom-input/2.txt$}}
Expand All @@ -32,8 +32,6 @@ CHECK: {{^1.txt$}}
CHECK-NEXT: {{^2.txt$}}
CHECK-NOT: BAD

VERBOSE-NESTED: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input/upload
VERBOSE-NESTED-NEXT: /usr/bin/sftp
VERBOSE-NESTED-DAG: -put '{{.+}}/Inputs/upload/1.txt' '{{.+}}-REMOTE/input/upload/1.txt'
VERBOSE-NESTED-DAG: -put '{{.+}}/Inputs/upload/2.txt' '{{.+}}-REMOTE/input/upload/2.txt'
VERBOSE-NESTED: /usr/bin/env ls
VERBOSE-NESTED: /usr/bin/env /bin/mkdir -p {{.+}}-REMOTE/input
VERBOSE-NESTED-NEXT: /usr/bin/rsync
VERBOSE-NESTED: /usr/bin/env {{.*}}ls
Loading