shithub: hugo

Download patch

ref: 08c0de5cc37cd4e512268b8f72ec5a6c68cd5754
parent: 1cf2f3dc4fa81503485a73db21bfda6e965dee15
author: Cameron Moore <[email protected]>
date: Mon May 1 18:41:08 EDT 2017 

tpl/data: Clean up data namespace

- Move the main GetCSV and GetJSON into data.go.
- Add error returns to GetCSV and GetJSON.
- Add http client to Namespace for test mocking.
- Send accept headers on remote requests. Fixes #3395
- Return an error on non-2XX HTTP response codes and don't retry.
- Move cache tests to cache_test.go.


--- /dev/null
+++ b/tpl/data/cache_test.go
@@ -1,0 +1,63 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCache(t *testing.T) {
+	t.Parallel()
+
+	fs := new(afero.MemMapFs)
+
+	for i, test := range []struct {
+		path    string
+		content []byte
+		ignore  bool
+	}{
+		{"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false},
+		{"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false},
+		{"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false},
+		{"трям/трям", []byte(`T€st трям/трям Content 123`), false},
+		{"은행", []byte(`T€st C은행ontent 123`), false},
+		{"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false},
+		{"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true},
+	} {
+		msg := fmt.Sprintf("Test #%d: %v", i, test)
+
+		cfg := viper.New()
+
+		c, err := getCache(test.path, fs, cfg, test.ignore)
+		assert.NoError(t, err, msg)
+		assert.Nil(t, c, msg)
+
+		err = writeCache(test.path, test.content, fs, cfg, test.ignore)
+		assert.NoError(t, err, msg)
+
+		c, err = getCache(test.path, fs, cfg, test.ignore)
+		assert.NoError(t, err, msg)
+
+		if test.ignore {
+			assert.Nil(t, c, msg)
+		} else {
+			assert.Equal(t, string(test.content), string(c))
+		}
+	}
+}
--- a/tpl/data/data.go
+++ b/tpl/data/data.go
@@ -13,12 +13,24 @@
 
 package data
 
-import "github.com/spf13/hugo/deps"
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"time"
 
+	"github.com/spf13/hugo/deps"
+	jww "github.com/spf13/jwalterweatherman"
+)
+
 // New returns a new instance of the data-namespaced template functions.
 func New(deps *deps.Deps) *Namespace {
 	return &Namespace{
-		deps: deps,
+		deps:   deps,
+		client: http.DefaultClient,
 	}
 }
 
@@ -25,4 +37,102 @@
 // Namespace provides template functions for the "data" namespace.
 type Namespace struct {
 	deps *deps.Deps
+
+	client *http.Client
+}
+
+// GetCSV expects a data separator and one or n-parts of a URL to a resource which
+// can either be a local or a remote one.
+// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
+// If you provide multiple parts for the URL they will be joined together to the final URL.
+// GetCSV returns nil or a slice slice to use in a short code.
+func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err error) {
+	url := strings.Join(urlParts, "")
+
+	var clearCacheSleep = func(i int, u string) {
+		jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
+		time.Sleep(resSleep)
+		deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
+	}
+
+	for i := 0; i <= resRetries; i++ {
+		var req *http.Request
+		req, err = http.NewRequest("GET", url, nil)
+		if err != nil {
+			jww.ERROR.Printf("Failed to create request for getJSON: %s", err)
+			return nil, err
+		}
+
+		req.Header.Add("Accept", "text/csv")
+		req.Header.Add("Accept", "text/plain")
+
+		var c []byte
+		c, err = ns.getResource(req)
+		if err != nil {
+			jww.ERROR.Printf("Failed to read csv resource %q with error message %s", url, err)
+			return nil, err
+		}
+
+		if !bytes.Contains(c, []byte(sep)) {
+			err = errors.New("Cannot find separator " + sep + " in CSV.")
+			return
+		}
+
+		if d, err = parseCSV(c, sep); err != nil {
+			jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
+			clearCacheSleep(i, url)
+			continue
+		}
+		break
+	}
+	return
+}
+
+// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
+// If you provide multiple parts they will be joined together to the final URL.
+// GetJSON returns nil or parsed JSON to use in a short code.
+func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) {
+	url := strings.Join(urlParts, "")
+
+	for i := 0; i <= resRetries; i++ {
+		var req *http.Request
+		req, err = http.NewRequest("GET", url, nil)
+		if err != nil {
+			jww.ERROR.Printf("Failed to create request for getJSON: %s", err)
+			return nil, err
+		}
+
+		req.Header.Add("Accept", "application/json")
+
+		var c []byte
+		c, err = ns.getResource(req)
+		if err != nil {
+			jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
+			return nil, err
+		}
+
+		err = json.Unmarshal(c, &v)
+		if err != nil {
+			jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
+			jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
+			time.Sleep(resSleep)
+			deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
+			continue
+		}
+		break
+	}
+	return
+}
+
+// parseCSV parses bytes of CSV data into a slice slice string or an error
+func parseCSV(c []byte, sep string) ([][]string, error) {
+	if len(sep) != 1 {
+		return nil, errors.New("Incorrect length of csv separator: " + sep)
+	}
+	b := bytes.NewReader(c)
+	r := csv.NewReader(b)
+	rSep := []rune(sep)
+	r.Comma = rSep[0]
+	r.FieldsPerRecord = 0
+	return r.ReadAll()
 }
--- /dev/null
+++ b/tpl/data/data_test.go
@@ -1,0 +1,251 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package data
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetCSV(t *testing.T) {
+	t.Parallel()
+
+	ns := New(newDeps(viper.New()))
+
+	for i, test := range []struct {
+		sep     string
+		url     string
+		content string
+		expect  interface{}
+	}{
+		// Remotes
+		{
+			",",
+			`http://success/`,
+			"gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm\n",
+			[][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}},
+		},
+		{
+			",",
+			`http://error.extra.field/`,
+			"gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm,EXTRA\n",
+			false,
+		},
+		{
+			",",
+			`http://error.no.sep/`,
+			"gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
+			false,
+		},
+		{
+			",",
+			`http://nofound/404`,
+			``,
+			false,
+		},
+
+		// Locals
+		{
+			";",
+			"pass/semi",
+			"gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
+			[][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}},
+		},
+		{
+			";",
+			"fail/no-file",
+			"",
+			false,
+		},
+	} {
+		msg := fmt.Sprintf("Test %d", i)
+
+		// Setup HTTP test server
+		var srv *httptest.Server
+		srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+			if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") {
+				http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+				return
+			}
+
+			if r.URL.Path == "/404" {
+				http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+				return
+			}
+
+			w.Header().Add("Content-type", "text/csv")
+
+			w.Write([]byte(test.content))
+		})
+		defer func() { srv.Close() }()
+
+		// Setup local test file for schema-less URLs
+		if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
+			f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
+			require.NoError(t, err, msg)
+			f.WriteString(test.content)
+			f.Close()
+		}
+
+		// Get on with it
+		got, err := ns.GetCSV(test.sep, test.url)
+
+		if _, ok := test.expect.(bool); ok {
+			assert.Error(t, err, msg)
+			continue
+		}
+		require.NoError(t, err, msg)
+		require.NotNil(t, got, msg)
+
+		assert.EqualValues(t, test.expect, got, msg)
+	}
+}
+
+func TestGetJSON(t *testing.T) {
+	t.Parallel()
+
+	ns := New(newDeps(viper.New()))
+
+	for i, test := range []struct {
+		url     string
+		content string
+		expect  interface{}
+	}{
+		{
+			`http://success/`,
+			`{"gomeetup":["Sydney","San Francisco","Stockholm"]}`,
+			map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}},
+		},
+		{
+			`http://malformed/`,
+			`{gomeetup:["Sydney","San Francisco","Stockholm"]}`,
+			false,
+		},
+		{
+			`http://nofound/404`,
+			``,
+			false,
+		},
+		// Locals
+		{
+			"pass/semi",
+			`{"gomeetup":["Sydney","San Francisco","Stockholm"]}`,
+			map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}},
+		},
+		{
+			"fail/no-file",
+			"",
+			false,
+		},
+	} {
+		msg := fmt.Sprintf("Test %d", i)
+
+		// Setup HTTP test server
+		var srv *httptest.Server
+		srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
+			if !haveHeader(r.Header, "Accept", "application/json") {
+				http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+				return
+			}
+
+			if r.URL.Path == "/404" {
+				http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+				return
+			}
+
+			w.Header().Add("Content-type", "application/json")
+
+			w.Write([]byte(test.content))
+		})
+		defer func() { srv.Close() }()
+
+		// Setup local test file for schema-less URLs
+		if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
+			f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
+			require.NoError(t, err, msg)
+			f.WriteString(test.content)
+			f.Close()
+		}
+
+		// Get on with it
+		got, err := ns.GetJSON(test.url)
+
+		if _, ok := test.expect.(bool); ok {
+			assert.Error(t, err, msg)
+			continue
+		}
+		require.NoError(t, err, msg)
+		require.NotNil(t, got, msg)
+
+		assert.EqualValues(t, test.expect, got, msg)
+	}
+}
+
+func TestParseCSV(t *testing.T) {
+	t.Parallel()
+
+	for i, test := range []struct {
+		csv []byte
+		sep string
+		exp string
+		err bool
+	}{
+		{[]byte("a,b,c\nd,e,f\n"), "", "", true},
+		{[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
+		{[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
+		{[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
+		{[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
+		{[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
+	} {
+		msg := fmt.Sprintf("Test %d: %v", i, test)
+
+		csv, err := parseCSV(test.csv, test.sep)
+		if test.err {
+			assert.Error(t, err, msg)
+			continue
+		}
+		require.NoError(t, err, msg)
+
+		act := ""
+		for _, v := range csv {
+			act = act + strings.Join(v, "")
+		}
+
+		assert.Equal(t, test.exp, act, msg)
+	}
+}
+
+func haveHeader(m http.Header, key, needle string) bool {
+	var s []string
+	var ok bool
+
+	if s, ok = m[key]; !ok {
+		return false
+	}
+
+	for _, v := range s {
+		if v == needle {
+			return true
+		}
+	}
+	return false
+}
--- a/tpl/data/resources.go
+++ b/tpl/data/resources.go
@@ -14,14 +14,10 @@
 package data
 
 import (
-	"bytes"
-	"encoding/csv"
-	"encoding/json"
-	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
@@ -67,14 +63,16 @@
 }
 
 // getRemote loads the content of a remote file. This method is thread safe.
-func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
+func getRemote(req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
+	url := req.URL.String()
+
 	c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
-	if c != nil && err == nil {
-		return c, nil
-	}
 	if err != nil {
 		return nil, err
 	}
+	if c != nil {
+		return c, nil
+	}
 
 	// avoid race condition with locks, block other goroutines if the current url is processing
 	remoteURLLock.URLLock(url)
@@ -82,27 +80,34 @@
 
 	// avoid multiple locks due to calling getCache twice
 	c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
-	if c != nil && err == nil {
-		return c, nil
-	}
 	if err != nil {
 		return nil, err
 	}
+	if c != nil {
+		return c, nil
+	}
 
 	jww.INFO.Printf("Downloading: %s ...", url)
-	res, err := hc.Get(url)
+	res, err := hc.Do(req)
 	if err != nil {
 		return nil, err
 	}
+
+	if res.StatusCode < 200 || res.StatusCode > 299 {
+		return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode))
+	}
+
 	c, err = ioutil.ReadAll(res.Body)
 	res.Body.Close()
 	if err != nil {
 		return nil, err
 	}
+
 	err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache"))
 	if err != nil {
 		return nil, err
 	}
+
 	jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url))
 	return c, nil
 }
@@ -119,90 +124,11 @@
 }
 
 // getResource loads the content of a local or remote file
-func (ns *Namespace) getResource(url string) ([]byte, error) {
-	if url == "" {
-		return nil, nil
+func (ns *Namespace) getResource(req *http.Request) ([]byte, error) {
+	switch req.URL.Scheme {
+	case "":
+		return getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg)
+	default:
+		return getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client)
 	}
-	if strings.Contains(url, "://") {
-		return getRemote(url, ns.deps.Fs.Source, ns.deps.Cfg, http.DefaultClient)
-	}
-	return getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
-}
-
-// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
-// If you provide multiple parts they will be joined together to the final URL.
-// GetJSON returns nil or parsed JSON to use in a short code.
-func (ns *Namespace) GetJSON(urlParts ...string) interface{} {
-	var v interface{}
-	url := strings.Join(urlParts, "")
-
-	for i := 0; i <= resRetries; i++ {
-		c, err := ns.getResource(url)
-		if err != nil {
-			jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
-			return nil
-		}
-
-		err = json.Unmarshal(c, &v)
-		if err != nil {
-			jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
-			jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
-			time.Sleep(resSleep)
-			deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
-			continue
-		}
-		break
-	}
-	return v
-}
-
-// parseCSV parses bytes of CSV data into a slice slice string or an error
-func parseCSV(c []byte, sep string) ([][]string, error) {
-	if len(sep) != 1 {
-		return nil, errors.New("Incorrect length of csv separator: " + sep)
-	}
-	b := bytes.NewReader(c)
-	r := csv.NewReader(b)
-	rSep := []rune(sep)
-	r.Comma = rSep[0]
-	r.FieldsPerRecord = 0
-	return r.ReadAll()
-}
-
-// GetCSV expects a data separator and one or n-parts of a URL to a resource which
-// can either be a local or a remote one.
-// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
-// If you provide multiple parts for the URL they will be joined together to the final URL.
-// GetCSV returns nil or a slice slice to use in a short code.
-func (ns *Namespace) GetCSV(sep string, urlParts ...string) [][]string {
-	var d [][]string
-	url := strings.Join(urlParts, "")
-
-	var clearCacheSleep = func(i int, u string) {
-		jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
-		time.Sleep(resSleep)
-		deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
-	}
-
-	for i := 0; i <= resRetries; i++ {
-		c, err := ns.getResource(url)
-
-		if err == nil && !bytes.Contains(c, []byte(sep)) {
-			err = errors.New("Cannot find separator " + sep + " in CSV.")
-		}
-
-		if err != nil {
-			jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
-			clearCacheSleep(i, url)
-			continue
-		}
-
-		if d, err = parseCSV(c, sep); err != nil {
-			jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
-			clearCacheSleep(i, url)
-			continue
-		}
-		break
-	}
-	return d
 }
--- a/tpl/data/resources_test.go
+++ b/tpl/data/resources_test.go
@@ -19,7 +19,6 @@
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -31,58 +30,9 @@
 	"github.com/spf13/hugo/hugofs"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func TestScpCache(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		path    string
-		content []byte
-		ignore  bool
-	}{
-		{"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false},
-		{"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false},
-		{"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false},
-		{"трям/трям", []byte(`T€st трям/трям Content 123`), false},
-		{"은행", []byte(`T€st C은행ontent 123`), false},
-		{"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false},
-		{"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true},
-	}
-
-	fs := new(afero.MemMapFs)
-
-	for _, test := range tests {
-		cfg := viper.New()
-		c, err := getCache(test.path, fs, cfg, test.ignore)
-		if err != nil {
-			t.Errorf("Error getting cache: %s", err)
-		}
-		if c != nil {
-			t.Errorf("There is content where there should not be anything: %s", string(c))
-		}
-
-		err = writeCache(test.path, test.content, fs, cfg, test.ignore)
-		if err != nil {
-			t.Errorf("Error writing cache: %s", err)
-		}
-
-		c, err = getCache(test.path, fs, cfg, test.ignore)
-		if err != nil {
-			t.Errorf("Error getting cache after writing: %s", err)
-		}
-		if test.ignore {
-			if c != nil {
-				t.Errorf("Cache ignored but content is not nil: %s", string(c))
-			}
-		} else {
-			if !bytes.Equal(c, test.content) {
-				t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
-			}
-		}
-	}
-}
-
 func TestScpGetLocal(t *testing.T) {
 	t.Parallel()
 	v := viper.New()
@@ -146,7 +96,11 @@
 	}
 
 	for _, test := range tests {
+		msg := fmt.Sprintf("%v", test)
 
+		req, err := http.NewRequest("GET", test.path, nil)
+		require.NoError(t, err, msg)
+
 		srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
 			w.Write(test.content)
 		})
@@ -154,25 +108,18 @@
 
 		cfg := viper.New()
 
-		c, err := getRemote(test.path, fs, cfg, cl)
-		if err != nil {
-			t.Errorf("Error getting resource content: %s", err)
-		}
-		if !bytes.Equal(c, test.content) {
-			t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c))
-		}
-		cc, cErr := getCache(test.path, fs, cfg, test.ignore)
-		if cErr != nil {
-			t.Error(cErr)
-		}
+		c, err := getRemote(req, fs, cfg, cl)
+		require.NoError(t, err, msg)
+		assert.Equal(t, string(test.content), string(c))
+
+		c, err = getCache(req.URL.String(), fs, cfg, test.ignore)
+		require.NoError(t, err, msg)
+
 		if test.ignore {
-			if cc != nil {
-				t.Errorf("Cache ignored but content is not nil: %s", string(cc))
-			}
+			assert.Empty(t, c, msg)
 		} else {
-			if !bytes.Equal(cc, test.content) {
-				t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(cc))
-			}
+			assert.Equal(t, string(test.content), string(c))
+
 		}
 	}
 }
@@ -179,16 +126,20 @@
 
 func TestScpGetRemoteParallel(t *testing.T) {
 	t.Parallel()
-	fs := new(afero.MemMapFs)
+
+	ns := New(newDeps(viper.New()))
+
 	content := []byte(`T€st Content 123`)
-	url := "http://Foo.Bar/foo_Bar-Foo"
 	srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
 		w.Write(content)
 	})
 	defer func() { srv.Close() }()
 
-	for _, ignoreCache := range []bool{false, true} {
+	url := "http://Foo.Bar/foo_Bar-Foo"
+	req, err := http.NewRequest("GET", url, nil)
+	require.NoError(t, err)
 
+	for _, ignoreCache := range []bool{false, true} {
 		cfg := viper.New()
 		cfg.Set("ignoreCache", ignoreCache)
 
@@ -199,13 +150,9 @@
 			go func(gor int) {
 				defer wg.Done()
 				for j := 0; j < 10; j++ {
-					c, err := getRemote(url, fs, cfg, cl)
-					if err != nil {
-						t.Errorf("Error getting resource content: %s", err)
-					}
-					if !bytes.Equal(c, content) {
-						t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(content), string(c))
-					}
+					c, err := getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, cl)
+					assert.NoError(t, err)
+					assert.Equal(t, string(content), string(c))
 
 					time.Sleep(23 * time.Millisecond)
 				}
@@ -213,137 +160,6 @@
 		}
 
 		wg.Wait()
-	}
-
-	t.Log("Done!")
-}
-
-func TestParseCSV(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		csv []byte
-		sep string
-		exp string
-		err bool
-	}{
-		{[]byte("a,b,c\nd,e,f\n"), "", "", true},
-		{[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
-		{[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
-		{[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
-		{[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
-		{[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
-	}
-	for _, test := range tests {
-		csv, err := parseCSV(test.csv, test.sep)
-		if test.err && err == nil {
-			t.Error("Expecting an error")
-		}
-		if test.err {
-			continue
-		}
-		if !test.err && err != nil {
-			t.Error(err)
-		}
-
-		act := ""
-		for _, v := range csv {
-			act = act + strings.Join(v, "")
-		}
-
-		if act != test.exp {
-			t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv)
-		}
-
-	}
-}
-
-func TestGetJSONFailParse(t *testing.T) {
-	t.Parallel()
-
-	ns := New(newDeps(viper.New()))
-
-	reqCount := 0
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if reqCount > 0 {
-			w.Header().Add("Content-type", "application/json")
-			fmt.Fprintln(w, `{"gomeetup":["Sydney", "San Francisco", "Stockholm"]}`)
-		} else {
-			w.WriteHeader(http.StatusInternalServerError)
-			fmt.Fprintln(w, `ERROR 500`)
-		}
-		reqCount++
-	}))
-	defer ts.Close()
-	url := ts.URL + "/test.json"
-
-	want := map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}}
-	have := ns.GetJSON(url)
-	assert.NotNil(t, have)
-	if have != nil {
-		assert.EqualValues(t, want, have)
-	}
-}
-
-func TestGetCSVFailParseSep(t *testing.T) {
-	t.Parallel()
-
-	ns := New(newDeps(viper.New()))
-
-	reqCount := 0
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if reqCount > 0 {
-			w.Header().Add("Content-type", "application/json")
-			fmt.Fprintln(w, `gomeetup,city`)
-			fmt.Fprintln(w, `yes,Sydney`)
-			fmt.Fprintln(w, `yes,San Francisco`)
-			fmt.Fprintln(w, `yes,Stockholm`)
-		} else {
-			w.WriteHeader(http.StatusInternalServerError)
-			fmt.Fprintln(w, `ERROR 500`)
-		}
-		reqCount++
-	}))
-	defer ts.Close()
-	url := ts.URL + "/test.csv"
-
-	want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}
-	have := ns.GetCSV(",", url)
-	assert.NotNil(t, have)
-	if have != nil {
-		assert.EqualValues(t, want, have)
-	}
-}
-
-func TestGetCSVFailParse(t *testing.T) {
-	t.Parallel()
-
-	ns := New(newDeps(viper.New()))
-
-	reqCount := 0
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Add("Content-type", "application/json")
-		if reqCount > 0 {
-			fmt.Fprintln(w, `gomeetup,city`)
-			fmt.Fprintln(w, `yes,Sydney`)
-			fmt.Fprintln(w, `yes,San Francisco`)
-			fmt.Fprintln(w, `yes,Stockholm`)
-		} else {
-			fmt.Fprintln(w, `gomeetup,city`)
-			fmt.Fprintln(w, `yes,Sydney,Bondi,`) // wrong number of fields in line
-			fmt.Fprintln(w, `yes,San Francisco`)
-			fmt.Fprintln(w, `yes,Stockholm`)
-		}
-		reqCount++
-	}))
-	defer ts.Close()
-	url := ts.URL + "/test.csv"
-
-	want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}
-	have := ns.GetCSV(",", url)
-	assert.NotNil(t, have)
-	if have != nil {
-		assert.EqualValues(t, want, have)
 	}
 }