Vendor license files

Add functions to test if a file is likely to be legal notice and copies
those files, up through the root of a dependency to the vendor location.

Closes #245

Changelog.md

+# v20 2016/11/13
+
+* Attempt to include license files when vendoring. (@client9)
+
 # v19 2016/11/3
 
 * Fix conflict error message. Revisions were swapped. Also better selection of package that needs update.

license.go

+package main
+
+import (
+	"strings"
+)
+
+// LicenseFilePrefix is a list of filename prefixes that indicate it
+//  might contain a software license
+var LicenseFilePrefix = []string{
+	"license",
+	"copying",
+	"unlicense",
+	"copyright",
+	"copyleft",
+}
+
+// LegalFileSubstring are substrings that indicate the file is likely
+// to contain some type of legal declaration.  "legal" is often used
+// that it might moved to LicenseFilePrefix
+var LegalFileSubstring = []string{
+	"legal",
+	"notice",
+	"disclaimer",
+	"patent",
+	"third-party",
+	"thirdparty",
+}
+
+// IsLicenseFile returns true if the filename might be contain a
+// software license
+func IsLicenseFile(filename string) bool {
+	lowerfile := strings.ToLower(filename)
+	for _, prefix := range LicenseFilePrefix {
+		if strings.HasPrefix(lowerfile, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsLegalFile returns true if the file is likely to contain some type
+// of of legal declaration or licensing information
+func IsLegalFile(filename string) bool {
+	lowerfile := strings.ToLower(filename)
+	for _, prefix := range LicenseFilePrefix {
+		if strings.HasPrefix(lowerfile, prefix) {
+			return true
+		}
+	}
+	for _, substring := range LegalFileSubstring {
+		if strings.Index(lowerfile, substring) != -1 {
+			return true
+		}
+	}
+	return false
+}

license_test.go

+package main
+
+import (
+	"testing"
+)
+
+// most of these were found via google searches
+//  site:github.com FILENAME
+//
+func TestLicenseFiles(t *testing.T) {
+	var testcases = []struct {
+		filename string
+		license  bool
+		legal    bool
+	}{
+		{"license", true, true},
+		{"License", true, true},
+		{"LICENSE.md", true, true},
+		{"LICENSE.rst", true, true},
+		{"LICENSE.txt", true, true},
+		{"copying", true, true},
+		{"COPYING.txt", true, true},
+		{"unlicense", true, true},
+		{"copyright", true, true},
+		{"COPYRIGHT.txt", true, true},
+		{"copyleft", true, true},
+		{"COPYLEFT.txt", true, true},
+		{"copyleft.txt", true, true},
+		{"Copyleft.txt", true, true},
+		{"copyleft-next-0.2.1.txt", true, true},
+		{"legal", false, true},
+		{"notice", false, true},
+		{"NOTICE", false, true},
+		{"disclaimer", false, true},
+		{"patent", false, true},
+		{"patents", false, true},
+		{"third-party", false, true},
+		{"thirdparty", false, true},
+		{"thirdparty.txt", false, true},
+		{"license-ThirdParty.txt", true, true},
+		{"LICENSE-ThirdParty.txt", true, true},
+		{"THIRDPARTY.md", false, true},
+		{"third-party.md", false, true},
+		{"THIRD-PARTY.txt", false, true},
+		{"extensions-third-party.md", false, true},
+		{"ThirdParty.md", false, true},
+		{"third-party-licenses.md", false, true},
+		{"0070-01-01-third-party.md", false, true},
+		{"LEGAL.txt", false, true},
+		{"legal.txt", false, true},
+		{"Legal.md", false, true},
+		{"LEGAL.md", false, true},
+		{"legal.rst", false, true},
+		{"Legal.rtf", false, true},
+		{"legal.rtf", false, true},
+		{"PATENTS.TXT", false, true},
+		{"ttf-PATENTS.txt", false, true},
+		{"patents.txt", false, true},
+		{"INRIA-DISCLAIMER.txt", false, true},
+
+		{"MPL-2.0-no-copyleft-exception.txt", false, false},
+	}
+
+	for pos, tt := range testcases {
+		license := IsLicenseFile(tt.filename)
+		if tt.license != license {
+			if license {
+				t.Errorf("%d/file %q is not marked as license", pos, tt.filename)
+			} else {
+				t.Errorf("%d/file %q was marked incorrectly as a license", pos, tt.filename)
+			}
+		}
+
+		legal := IsLegalFile(tt.filename)
+		if tt.legal != legal {
+			if legal {
+				t.Errorf("%d/File %q is not marked as legal file", pos, tt.filename)
+			} else {
+				t.Errorf("%d/File %q was marked incorrectly as a legal file", pos, tt.filename)
+			}
+		}
+	}
+}

rewrite_test.go

@@ -242,7 +242,7 @@ func TestRewrite(t *testing.T) {
 
 	const gopath = "godeptest"
 	defer os.RemoveAll(gopath)
-	for _, test := range cases {
+	for pos, test := range cases {
 		err := os.RemoveAll(gopath)
 		if err != nil {
 			t.Fatal(err)
@@ -261,6 +261,6 @@ func TestRewrite(t *testing.T) {
 			t.Errorf("Unexpected tempfiles: %+v", tempFiles)
 		}
 
-		checkTree(t, &node{src, "", test.want})
+		checkTree(t, pos, &node{src, "", test.want})
 	}
 }

save.go

@@ -267,6 +267,8 @@ func removeSrc(srcdir string, deps []Dependency) error {
 }
 
 func copySrc(dir string, deps []Dependency) error {
+	// mapping to see if we visited a parent directory already
+	visited := make(map[string]bool)
 	ok := true
 	for _, dep := range deps {
 		srcdir := filepath.Join(dep.ws, "src")
@@ -280,6 +282,8 @@ func copySrc(dir string, deps []Dependency) error {
 			log.Println(err)
 			ok = false
 		}
+
+		// copy actual dependency
 		vf := dep.vcs.listFiles(dep.dir)
 		w := fs.Walk(dep.dir)
 		for w.Step() {
@@ -289,10 +293,43 @@ func copySrc(dir string, deps []Dependency) error {
 				ok = false
 			}
 		}
+
+		// Look for legal files in root
+		//  some packages are imports as a sub-package but license info
+		//  is at root:  exampleorg/common has license file in exampleorg
+		//
+		if dep.ImportPath == dep.root {
+			// we are already at root
+			continue
+		}
+
+		// prevent copying twice This could happen if we have
+		//   two subpackages listed someorg/common and
+		//   someorg/anotherpack which has their license in
+		//   the parent dir of someorg
+		rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root))
+		if visited[rootdir] {
+			continue
+		}
+		visited[rootdir] = true
+		vf = dep.vcs.listFiles(rootdir)
+		w = fs.Walk(rootdir)
+		for w.Step() {
+			fname := filepath.Base(w.Path())
+			if IsLegalFile(fname) {
+				err = copyPkgFile(vf, dir, srcdir, w)
+				if err != nil {
+					log.Println(err)
+					ok = false
+				}
+			}
+		}
 	}
+
 	if !ok {
 		return errorCopyingSourceCode
 	}
+
 	return nil
 }
 

save_test.go

@@ -1075,6 +1075,44 @@ func TestSave(t *testing.T) {
 				},
 			},
 		},
+		{ // Copy legal files in parent and dependency directory
+			cwd: "C",
+			start: []*node{
+				{
+					"C",
+					"",
+					[]*node{
+						{"main.go", pkg("main", "D/P", "D/Q"), nil},
+						{"+git", "", nil},
+					},
+				},
+				{
+					"D",
+					"",
+					[]*node{
+						{"LICENSE", pkg("D"), nil},
+						{"P/main.go", pkg("P"), nil},
+						{"P/LICENSE", pkg("P"), nil},
+						{"Q/main.go", pkg("Q"), nil},
+						{"+git", "D1", nil},
+					},
+				},
+			},
+			want: []*node{
+				{"C/main.go", pkg("main", "D/P", "D/Q"), nil},
+				{"C/Godeps/_workspace/src/D/LICENSE", pkg("D"), nil},
+				{"C/Godeps/_workspace/src/D/P/main.go", pkg("P"), nil},
+				{"C/Godeps/_workspace/src/D/P/LICENSE", pkg("P"), nil},
+				{"C/Godeps/_workspace/src/D/Q/main.go", pkg("Q"), nil},
+			},
+			wdep: Godeps{
+				ImportPath: "C",
+				Deps: []Dependency{
+					{ImportPath: "D/P", Comment: "D1"},
+					{ImportPath: "D/Q", Comment: "D1"},
+				},
+			},
+		},
 	}
 
 	wd, err := os.Getwd()
@@ -1083,7 +1121,7 @@ func TestSave(t *testing.T) {
 	}
 	const scratch = "godeptest"
 	defer os.RemoveAll(scratch)
-	for _, test := range cases {
+	for pos, test := range cases {
 		err = os.RemoveAll(scratch)
 		if err != nil {
 			t.Fatal(err)
@@ -1120,7 +1158,7 @@ func TestSave(t *testing.T) {
 			panic(err)
 		}
 
-		checkTree(t, &node{src, "", test.want})
+		checkTree(t, pos, &node{src, "", test.want})
 
 		f, err := os.Open(filepath.Join(dir, "Godeps/Godeps.json"))
 		if err != nil {
@@ -1198,7 +1236,7 @@ func makeTree(t *testing.T, tree *node, altpath string) (gopath string) {
 	return gopath
 }
 
-func checkTree(t *testing.T, want *node) {
+func checkTree(t *testing.T, pos int, want *node) {
 	walkTree(want, want.path, func(path string, n *node) {
 		body := n.body.(string)
 		switch {
@@ -1209,17 +1247,17 @@ func checkTree(t *testing.T, want *node) {
 		case n.entries == nil && body == "(absent)":
 			body, err := ioutil.ReadFile(path)
 			if !os.IsNotExist(err) {
-				t.Errorf("checkTree: %s = %s want absent", path, string(body))
+				t.Errorf("%d checkTree: %s = %s want absent", pos, path, string(body))
 				return
 			}
 		case n.entries == nil:
 			gbody, err := ioutil.ReadFile(path)
 			if err != nil {
-				t.Errorf("checkTree: %v", err)
+				t.Errorf("%d checkTree: %v", pos, err)
 				return
 			}
 			if got := string(gbody); got != body {
-				t.Errorf("%s = got: %q want: %q", path, got, body)
+				t.Errorf("%d %s = got: %q want: %q", pos, path, got, body)
 			}
 		default:
 			os.MkdirAll(path, 0770)

update_test.go

@@ -372,7 +372,7 @@ func TestUpdate(t *testing.T) {
 	}
 	const gopath = "godeptest"
 	defer os.RemoveAll(gopath)
-	for _, test := range cases {
+	for pos, test := range cases {
 		err = os.RemoveAll(gopath)
 		if err != nil {
 			t.Fatal(err)
@@ -400,7 +400,7 @@ func TestUpdate(t *testing.T) {
 			panic(err)
 		}
 
-		checkTree(t, &node{src, "", test.want})
+		checkTree(t, pos, &node{src, "", test.want})
 
 		f, err := os.Open(filepath.Join(dir, "Godeps/Godeps.json"))
 		if err != nil {

version.go

 package main
 
-const version = 19
+const version = 20