Skip to content

Commit 775121e

Browse files
bk2204bk2204
committed
WIP: tq: implement a pure SSH-based protocol for transfers
A pure SSH-based protocol has been a long time request from many users, so let's implement one. Implement basic upload and download support, plus batch requests, and support pkt-line tracing for text packets to make debugging easier. Note that locking is not yet supported; that will come in a future patch.
1 parent 6ab1d19 commit 775121e

File tree

6 files changed

+512
-42
lines changed

6 files changed

+512
-42
lines changed

‎ssh/connection.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package ssh
2+
3+
import (
4+
"github.com/git-lfs/git-lfs/config"
5+
"github.com/git-lfs/git-lfs/subprocess"
6+
"github.com/git-lfs/pktline"
7+
)
8+
9+
type SSHTransfer struct {
10+
conn *PktlineConnection
11+
}
12+
13+
func NewSSHTransfer(osEnv config.Environment, gitEnv config.Environment, meta *SSHMetadata, operation string) (*SSHTransfer, error) {
14+
exe, args := GetLFSExeAndArgs(osEnv, gitEnv, meta, "git-lfs-transfer", operation)
15+
cmd := subprocess.ExecCommand(exe, args...)
16+
r, err := cmd.StdoutPipe()
17+
if err != nil {
18+
return nil, err
19+
}
20+
w, err := cmd.StdinPipe()
21+
if err != nil {
22+
return nil, err
23+
}
24+
err = cmd.Start()
25+
if err != nil {
26+
return nil, err
27+
}
28+
29+
var pl Pktline
30+
if osEnv.Bool("GIT_TRACE_PACKET", false) {
31+
pl = &TraceablePktline{pl: pktline.NewPktline(r, w)}
32+
} else {
33+
pl = pktline.NewPktline(r, w)
34+
}
35+
conn := &PktlineConnection{
36+
cmd: cmd,
37+
pl: pl,
38+
}
39+
err = conn.Start()
40+
if err != nil {
41+
return nil, err
42+
}
43+
return &SSHTransfer{
44+
conn: conn,
45+
}, nil
46+
}
47+
48+
func (tr *SSHTransfer) Connection() *PktlineConnection {
49+
return tr.conn
50+
}

‎t/t-env.sh‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ PruneRemoteName=origin
5151
LfsStorageDir=%s
5252
AccessDownload=none
5353
AccessUpload=none
54-
DownloadTransfers=basic,lfs-standalone-file
55-
UploadTransfers=basic,lfs-standalone-file
54+
DownloadTransfers=basic,lfs-standalone-file,ssh
55+
UploadTransfers=basic,lfs-standalone-file,ssh
5656
%s
5757
%s
5858
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -104,8 +104,8 @@ PruneRemoteName=origin
104104
LfsStorageDir=%s
105105
AccessDownload=none
106106
AccessUpload=none
107-
DownloadTransfers=basic,lfs-standalone-file
108-
UploadTransfers=basic,lfs-standalone-file
107+
DownloadTransfers=basic,lfs-standalone-file,ssh
108+
UploadTransfers=basic,lfs-standalone-file,ssh
109109
%s
110110
%s
111111
' "$(git lfs version)" "$(git version)" "$endpoint" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -164,8 +164,8 @@ PruneRemoteName=origin
164164
LfsStorageDir=%s
165165
AccessDownload=none
166166
AccessUpload=none
167-
DownloadTransfers=basic,lfs-standalone-file
168-
UploadTransfers=basic,lfs-standalone-file
167+
DownloadTransfers=basic,lfs-standalone-file,ssh
168+
UploadTransfers=basic,lfs-standalone-file,ssh
169169
%s
170170
%s
171171
' "$(git lfs version)" "$(git version)" "$endpoint" "$endpoint2" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -222,8 +222,8 @@ PruneRemoteName=origin
222222
LfsStorageDir=%s
223223
AccessDownload=none
224224
AccessUpload=none
225-
DownloadTransfers=basic,lfs-standalone-file
226-
UploadTransfers=basic,lfs-standalone-file
225+
DownloadTransfers=basic,lfs-standalone-file,ssh
226+
UploadTransfers=basic,lfs-standalone-file,ssh
227227
%s
228228
%s
229229
' "$(git lfs version)" "$(git version)" "$endpoint" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -281,8 +281,8 @@ PruneRemoteName=origin
281281
LfsStorageDir=%s
282282
AccessDownload=none
283283
AccessUpload=none
284-
DownloadTransfers=basic,lfs-standalone-file
285-
UploadTransfers=basic,lfs-standalone-file
284+
DownloadTransfers=basic,lfs-standalone-file,ssh
285+
UploadTransfers=basic,lfs-standalone-file,ssh
286286
%s
287287
%s
288288
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -342,8 +342,8 @@ PruneRemoteName=origin
342342
LfsStorageDir=%s
343343
AccessDownload=none
344344
AccessUpload=none
345-
DownloadTransfers=basic,lfs-standalone-file
346-
UploadTransfers=basic,lfs-standalone-file
345+
DownloadTransfers=basic,lfs-standalone-file,ssh
346+
UploadTransfers=basic,lfs-standalone-file,ssh
347347
%s
348348
%s
349349
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -403,8 +403,8 @@ PruneRemoteName=origin
403403
LfsStorageDir=%s
404404
AccessDownload=none
405405
AccessUpload=none
406-
DownloadTransfers=basic,lfs-standalone-file
407-
UploadTransfers=basic,lfs-standalone-file
406+
DownloadTransfers=basic,lfs-standalone-file,ssh
407+
UploadTransfers=basic,lfs-standalone-file,ssh
408408
%s
409409
%s
410410
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -472,8 +472,8 @@ PruneRemoteName=origin
472472
LfsStorageDir=%s
473473
AccessDownload=none
474474
AccessUpload=none
475-
DownloadTransfers=basic,lfs-standalone-file
476-
UploadTransfers=basic,lfs-standalone-file
475+
DownloadTransfers=basic,lfs-standalone-file,ssh
476+
UploadTransfers=basic,lfs-standalone-file,ssh
477477
%s
478478
%s
479479
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -528,8 +528,8 @@ PruneRemoteName=origin
528528
LfsStorageDir=%s
529529
AccessDownload=none
530530
AccessUpload=none
531-
DownloadTransfers=basic,lfs-standalone-file
532-
UploadTransfers=basic,lfs-standalone-file
531+
DownloadTransfers=basic,lfs-standalone-file,ssh
532+
UploadTransfers=basic,lfs-standalone-file,ssh
533533
%s
534534
%s
535535
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -582,8 +582,8 @@ PruneRemoteName=origin
582582
LfsStorageDir=lfs
583583
AccessDownload=none
584584
AccessUpload=none
585-
DownloadTransfers=basic,lfs-standalone-file
586-
UploadTransfers=basic,lfs-standalone-file
585+
DownloadTransfers=basic,lfs-standalone-file,ssh
586+
UploadTransfers=basic,lfs-standalone-file,ssh
587587
%s
588588
git config filter.lfs.process = ""
589589
git config filter.lfs.smudge = ""
@@ -618,8 +618,8 @@ PruneRemoteName=origin
618618
LfsStorageDir=%s
619619
AccessDownload=none
620620
AccessUpload=none
621-
DownloadTransfers=basic,lfs-standalone-file
622-
UploadTransfers=basic,lfs-standalone-file
621+
DownloadTransfers=basic,lfs-standalone-file,ssh
622+
UploadTransfers=basic,lfs-standalone-file,ssh
623623
%s
624624
%s
625625
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -651,8 +651,8 @@ PruneRemoteName=origin
651651
LfsStorageDir=%s
652652
AccessDownload=none
653653
AccessUpload=none
654-
DownloadTransfers=basic,lfs-standalone-file
655-
UploadTransfers=basic,lfs-standalone-file
654+
DownloadTransfers=basic,lfs-standalone-file,ssh
655+
UploadTransfers=basic,lfs-standalone-file,ssh
656656
%s
657657
%s
658658
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -697,8 +697,8 @@ PruneRemoteName=origin
697697
LfsStorageDir=%s
698698
AccessDownload=none
699699
AccessUpload=none
700-
DownloadTransfers=basic,lfs-standalone-file
701-
UploadTransfers=basic,lfs-standalone-file
700+
DownloadTransfers=basic,lfs-standalone-file,ssh
701+
UploadTransfers=basic,lfs-standalone-file,ssh
702702
%s
703703
%s
704704
" "$(git lfs version)" "$(git version)" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -777,8 +777,8 @@ PruneRemoteName=origin
777777
LfsStorageDir=%s
778778
AccessDownload=none
779779
AccessUpload=none
780-
DownloadTransfers=basic,lfs-standalone-file
781-
UploadTransfers=basic,lfs-standalone-file
780+
DownloadTransfers=basic,lfs-standalone-file,ssh
781+
UploadTransfers=basic,lfs-standalone-file,ssh
782782
%s
783783
%s
784784
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -810,8 +810,8 @@ PruneRemoteName=origin
810810
LfsStorageDir=%s
811811
AccessDownload=none
812812
AccessUpload=none
813-
DownloadTransfers=basic,lfs-standalone-file
814-
UploadTransfers=basic,lfs-standalone-file
813+
DownloadTransfers=basic,lfs-standalone-file,ssh
814+
UploadTransfers=basic,lfs-standalone-file,ssh
815815
%s
816816
%s
817817
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -843,8 +843,8 @@ PruneRemoteName=origin
843843
LfsStorageDir=%s
844844
AccessDownload=none
845845
AccessUpload=none
846-
DownloadTransfers=basic,lfs-standalone-file
847-
UploadTransfers=basic,lfs-standalone-file
846+
DownloadTransfers=basic,lfs-standalone-file,ssh
847+
UploadTransfers=basic,lfs-standalone-file,ssh
848848
%s
849849
%s
850850
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVarsEnabled" "$envInitConfig")
@@ -906,8 +906,8 @@ PruneRemoteName=origin
906906
LfsStorageDir=%s
907907
AccessDownload=none
908908
AccessUpload=none
909-
DownloadTransfers=basic,lfs-standalone-file,supertransfer
910-
UploadTransfers=basic,lfs-standalone-file,supertransfer,tus
909+
DownloadTransfers=basic,lfs-standalone-file,ssh,supertransfer
910+
UploadTransfers=basic,lfs-standalone-file,ssh,supertransfer,tus
911911
%s
912912
%s
913913
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -966,8 +966,8 @@ PruneRemoteName=origin
966966
LfsStorageDir=%s
967967
AccessDownload=none
968968
AccessUpload=none
969-
DownloadTransfers=basic,lfs-standalone-file
970-
UploadTransfers=basic,lfs-standalone-file
969+
DownloadTransfers=basic,lfs-standalone-file,ssh
970+
UploadTransfers=basic,lfs-standalone-file,ssh
971971
%s
972972
%s
973973
' "$(git lfs version)" "$(git version)" "$endpoint" "$endpoint2" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")
@@ -1031,8 +1031,8 @@ PruneRemoteName=origin
10311031
LfsStorageDir=%s
10321032
AccessDownload=none
10331033
AccessUpload=none
1034-
DownloadTransfers=basic,lfs-standalone-file
1035-
UploadTransfers=basic,lfs-standalone-file
1034+
DownloadTransfers=basic,lfs-standalone-file,ssh
1035+
UploadTransfers=basic,lfs-standalone-file,ssh
10361036
%s
10371037
%s
10381038
' "$(git lfs version)" "$(git version)" "$endpoint" "$endpoint2" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig")

‎t/t-worktree.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ PruneRemoteName=origin
4747
LfsStorageDir=$(canonical_path_escaped "$TRASHDIR/$reponame/.git/lfs")
4848
AccessDownload=none
4949
AccessUpload=none
50-
DownloadTransfers=basic,lfs-standalone-file
51-
UploadTransfers=basic,lfs-standalone-file
50+
DownloadTransfers=basic,lfs-standalone-file,ssh
51+
UploadTransfers=basic,lfs-standalone-file,ssh
5252
$(escape_path "$(env | grep "^GIT")")
5353
%s
5454
" "$(git lfs version)" "$(git version)" "$envInitConfig")
@@ -83,8 +83,8 @@ PruneRemoteName=origin
8383
LfsStorageDir=$(canonical_path_escaped "$TRASHDIR/$reponame/.git/lfs")
8484
AccessDownload=none
8585
AccessUpload=none
86-
DownloadTransfers=basic,lfs-standalone-file
87-
UploadTransfers=basic,lfs-standalone-file
86+
DownloadTransfers=basic,lfs-standalone-file,ssh
87+
UploadTransfers=basic,lfs-standalone-file,ssh
8888
$(escape_path "$(env | grep "^GIT")")
8989
%s
9090
" "$(git lfs version)" "$(git version)" "$envInitConfig")

‎tq/manifest.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/git-lfs/git-lfs/config"
88
"github.com/git-lfs/git-lfs/fs"
99
"github.com/git-lfs/git-lfs/lfsapi"
10+
"github.com/git-lfs/git-lfs/ssh"
1011
"github.com/rubyist/tracerx"
1112
)
1213

@@ -30,6 +31,7 @@ type Manifest struct {
3031
uploadAdapterFuncs map[string]NewAdapterFunc
3132
fs *fs.Filesystem
3233
apiClient *lfsapi.Client
34+
sshTransfer *ssh.SSHTransfer
3335
batchClientAdapter BatchTransferAdapter
3436
mu sync.Mutex
3537
}
@@ -71,12 +73,25 @@ func NewManifest(f *fs.Filesystem, apiClient *lfsapi.Client, operation, remote s
7173
apiClient = cli
7274
}
7375

76+
var sshTransfer *ssh.SSHTransfer
77+
var err error
78+
endpoint := apiClient.Endpoints.RemoteEndpoint(operation, remote)
79+
if len(endpoint.SSHMetadata.UserAndHost) > 0 {
80+
ctx := apiClient.Context()
81+
tracerx.Printf("attempting pure SSH protocol connection")
82+
sshTransfer, err = ssh.NewSSHTransfer(ctx.OSEnv(), ctx.GitEnv(), &endpoint.SSHMetadata, operation)
83+
if err != nil {
84+
tracerx.Printf("pure SSH protocol connection failed: %s", err)
85+
}
86+
}
87+
7488
m := &Manifest{
7589
fs: f,
7690
apiClient: apiClient,
7791
batchClientAdapter: &tqClient{Client: apiClient},
7892
downloadAdapterFuncs: make(map[string]NewAdapterFunc),
7993
uploadAdapterFuncs: make(map[string]NewAdapterFunc),
94+
sshTransfer: sshTransfer,
8095
}
8196

8297
var tusAllowed bool
@@ -109,11 +124,21 @@ func NewManifest(f *fs.Filesystem, apiClient *lfsapi.Client, operation, remote s
109124
m.concurrentTransfers = defaultConcurrentTransfers
110125
}
111126

127+
if sshTransfer != nil {
128+
// Multiple concurrent transfers are not yet supported.
129+
m.concurrentTransfers = 1
130+
m.batchClientAdapter = &SSHBatchTransferAdapter{
131+
maxRetries: m.maxRetries,
132+
transfer: sshTransfer,
133+
}
134+
}
135+
112136
configureBasicDownloadAdapter(m)
113137
configureBasicUploadAdapter(m)
114138
if tusAllowed {
115139
configureTusAdapter(m)
116140
}
141+
configureSSHAdapter(m)
117142
return m
118143
}
119144

0 commit comments

Comments
 (0)