shithub: hugo

Info  •  Files  •  Log  •  Branches

deploy: compute MD5 by reading if List didn't provide one

--- a/deploy/deploy.go

+++ b/deploy/deploy.go

@@ -19,7 +19,6 @@

 	"context"

 	"crypto/md5"

 	"fmt"

-	"golang.org/x/text/unicode/norm"

 	"io"

 	"mime"

 	"os"

@@ -33,6 +32,7 @@

 	"github.com/pkg/errors"

 	"github.com/spf13/afero"

 	jww "github.com/spf13/jwalterweatherman"

+	"golang.org/x/text/unicode/norm"

 	"gocloud.dev/blob"

 	_ "gocloud.dev/blob/azureblob" // import

@@ -418,6 +418,23 @@

 		if err != nil {

 			return nil, err

 		}

+		// If the remote didn't give us an MD5, compute one.

+		// This can happen for some providers (e.g., fileblob, which uses the

+		// local filesystem), but not for the most common Cloud providers

+		// (S3, GCS, Azure). Although, it can happen for S3 if the blob was uploaded

+		// via a multi-part upload.

+		// Although it's unfortunate to have to read the file, it's likely better

+		// than assuming a delta and re-uploading it.

+		if len(obj.MD5) == 0 {

+			r, err := bucket.NewReader(ctx, obj.Key, nil)

+			if err == nil {

+				h := md5.New()

+				if _, err := io.Copy(h, r); err == nil {

+					obj.MD5 = h.Sum(nil)

+				}

+				r.Close()

+			}

+		}

 		retval[obj.Key] = obj

 	}

 	return retval, nil

@@ -494,15 +511,9 @@

 				upload = true

 				reason = reasonSize

 			} else if len(remoteFile.MD5) == 0 {

-				// TODO: This can happen if the remote provider doesn't return an MD5

-				// hash for the blob from their "list" command. This is common for

-				// some providers (e.g., fileblob, which uses the local filesystem),

-				// but not for the biggest Cloud providers (S3, GCS, Azure). Although,

-				// it can happen for S3 if the blob was originally uploaded as a

-				// multi-part upload (shouldn't happen when using "hugo deploy").

-				// For now, we treat this as an MD5 mismatch and re-upload. An

-				// alternative would be to read entire the remote blob and compute the

-				// MD5 hash.

+				// This shouldn't happen unless the remote didn't give us an MD5 hash

+				// from List, AND we failed to compute one by reading the remote file.

+				// Default to considering the files different.

 				upload = true

 				reason = reasonMD5Missing

 			} else if !bytes.Equal(lf.MD5(), remoteFile.MD5) {

--- a/go.mod

+++ b/go.mod

@@ -19,6 +19,7 @@

 	github.com/fortytw2/leaktest v1.2.0

 	github.com/fsnotify/fsnotify v1.4.7

 	github.com/gobwas/glob v0.2.3

+	github.com/google/go-cmp v0.2.0

 	github.com/gorilla/websocket v1.4.0

 	github.com/hashicorp/go-immutable-radix v1.0.0

 	github.com/jdkato/prose v1.1.0

--- a/go.sum

+++ b/go.sum

@@ -211,6 +211,7 @@

 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=

 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=

 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=

+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=

 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=

 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=

 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=