From a7adb4853fe4344c78983e77ee7167f1b5994a1a Mon Sep 17 00:00:00 2001 From: Nicolas Hillegeer Date: Fri, 7 Apr 2017 22:40:54 +0200 Subject: [PATCH 1/2] Revert "Revert "*: add a 1 minute HTTP inactivity timeout"" This reverts commit 26d8f4f6ce25c2f6a5789c421e320bdcb9e55895. --- github/github.go | 4 +-- github/http.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 github/http.go diff --git a/github/github.go b/github/github.go index 6e3b2de..fe9b758 100644 --- a/github/github.go +++ b/github/github.go @@ -29,7 +29,7 @@ func DoAuthRequest(method, url, mime, token string, headers map[string]string, b return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { return nil, err } @@ -149,7 +149,7 @@ func (c Client) getPaginated(uri string) (io.ReadCloser, error) { v.Set("access_token", c.Token) } u.RawQuery = v.Encode() - resp, err := http.Get(u.String()) + resp, err := client.Get(u.String()) if err != nil { return nil, err } diff --git a/github/http.go b/github/http.go new file mode 100644 index 0000000..b474ed8 --- /dev/null +++ b/github/http.go @@ -0,0 +1,76 @@ +package github + +import ( + "context" + "net" + "net/http" + "time" +) + +// Github's HTTP endpoint for asset uploads is flaky. Often it will drop a +// TCP connection *. The HTTP POST will just hang indefinitely. To solve +// this without using absolute deadlines (which would be hard to predict as +// it depends on the size of the asset and the speed of the users' +// connection), we use a modified http.Client which resets a timeout every +// time the kernel accepts a read/write on the socket. Doing so basically +// creates a sort of "inactivity watcher". +// +// We can't use the http.Client.Timeout field, as that's an absolute timeout +// which doesn't get reset whenever there's some activity. +// +// * At least that's what I think is happening, I have never observed this +// myself since I never upload big assets, see issue +// http://github.com/aktau/github-release/issues/26. + +// HTTPReadWriteTimeout is the read/write timeout after which connections +// will be closed. +const HTTPReadWriteTimeout = 1 * time.Minute + +// dialer for use by the transport below, initialized like the net/http +// dialer. +var dialer = &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, +} + +// transport for use by the http.Client below. +// +// TODO: enable HTTP/2 transport as documented in net/http. +var transport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: func(ctx context.Context, netw, addr string) (net.Conn, error) { + return newWatchdogConn(dialer.DialContext(ctx, netw, addr)) + }, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: 1 * time.Minute, +} + +// client is an HTTP client suited for use with GitHub. It includes watchdog +// functionality that will break of an upload/download attempt after +// HTTPReadWriteTimeout. +var client = &http.Client{Transport: transport} + +// watchdogConn wraps a net.Conn, on every read and write it sets a timeout. +// If no bytes are read or written in that time, the connection is closed. +type watchdogConn struct { + net.Conn + timeout time.Duration // The amount of time to wait between reads/writes on the connection before cancelling it. +} + +func newWatchdogConn(conn net.Conn, err error) (net.Conn, error) { + return &watchdogConn{Conn: conn, timeout: HTTPReadWriteTimeout}, err +} + +func (c *watchdogConn) Read(b []byte) (n int, err error) { + c.SetReadDeadline(time.Now().Add(c.timeout)) + return c.Conn.Read(b) +} + +func (c *watchdogConn) Write(b []byte) (n int, err error) { + c.SetWriteDeadline(time.Now().Add(c.timeout)) + return c.Conn.Write(b) +} From dcac27dad7a8653cc7337380590a7cac778121eb Mon Sep 17 00:00:00 2001 From: Nicolas Hillegeer Date: Fri, 7 Apr 2017 22:41:02 +0200 Subject: [PATCH 2/2] Revert "Revert "*: use watchdogConn also for paginated requests"" This reverts commit 27cc45adb4f0da87a83083b6cd262993ac8eaed8. --- github/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github/github.go b/github/github.go index fe9b758..beecd67 100644 --- a/github/github.go +++ b/github/github.go @@ -182,7 +182,7 @@ func (c Client) getPaginated(uri string) (io.ReadCloser, error) { return // We're done. } - resp, err := http.Get(URL) + resp, err := client.Get(URL) links = linkheader.Parse(resp.Header.Get("Link")) if err != nil { w.CloseWithError(err)