1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package zip
47
48 import (
49 "archive/zip"
50 "bytes"
51 "errors"
52 "fmt"
53 "io"
54 "io/ioutil"
55 "os"
56 "os/exec"
57 "path"
58 "path/filepath"
59 "strings"
60 "unicode"
61 "unicode/utf8"
62
63 "golang.org/x/mod/module"
64 )
65
66 const (
67
68
69
70 MaxZipFile = 500 << 20
71
72
73
74 MaxGoMod = 16 << 20
75
76
77
78 MaxLICENSE = 16 << 20
79 )
80
81
82
83 type File interface {
84
85
86 Path() string
87
88
89
90 Lstat() (os.FileInfo, error)
91
92
93
94 Open() (io.ReadCloser, error)
95 }
96
97
98
99
100
101
102
103 type CheckedFiles struct {
104
105 Valid []string
106
107
108
109 Omitted []FileError
110
111
112
113 Invalid []FileError
114
115
116
117
118 SizeError error
119 }
120
121
122
123
124
125 func (cf CheckedFiles) Err() error {
126 if cf.SizeError != nil {
127 return cf.SizeError
128 }
129 if len(cf.Invalid) > 0 {
130 return FileErrorList(cf.Invalid)
131 }
132 return nil
133 }
134
135 type FileErrorList []FileError
136
137 func (el FileErrorList) Error() string {
138 buf := &strings.Builder{}
139 sep := ""
140 for _, e := range el {
141 buf.WriteString(sep)
142 buf.WriteString(e.Error())
143 sep = "\n"
144 }
145 return buf.String()
146 }
147
148 type FileError struct {
149 Path string
150 Err error
151 }
152
153 func (e FileError) Error() string {
154 return fmt.Sprintf("%s: %s", e.Path, e.Err)
155 }
156
157 func (e FileError) Unwrap() error {
158 return e.Err
159 }
160
161 var (
162
163 errPathNotClean = errors.New("file path is not clean")
164 errPathNotRelative = errors.New("file path is not relative")
165 errGoModCase = errors.New("go.mod files must have lowercase names")
166 errGoModSize = fmt.Errorf("go.mod file too large (max size is %d bytes)", MaxGoMod)
167 errLICENSESize = fmt.Errorf("LICENSE file too large (max size is %d bytes)", MaxLICENSE)
168
169
170 errVCS = errors.New("directory is a version control repository")
171 errVendored = errors.New("file is in vendor directory")
172 errSubmoduleFile = errors.New("file is in another module")
173 errSubmoduleDir = errors.New("directory is in another module")
174 errHgArchivalTxt = errors.New("file is inserted by 'hg archive' and is always omitted")
175 errSymlink = errors.New("file is a symbolic link")
176 errNotRegular = errors.New("not a regular file")
177 )
178
179
180
181
182
183
184
185
186
187
188
189
190 func CheckFiles(files []File) (CheckedFiles, error) {
191 cf, _, _ := checkFiles(files)
192 return cf, cf.Err()
193 }
194
195
196
197
198
199
200 func checkFiles(files []File) (cf CheckedFiles, validFiles []File, validSizes []int64) {
201 errPaths := make(map[string]struct{})
202 addError := func(path string, omitted bool, err error) {
203 if _, ok := errPaths[path]; ok {
204 return
205 }
206 errPaths[path] = struct{}{}
207 fe := FileError{Path: path, Err: err}
208 if omitted {
209 cf.Omitted = append(cf.Omitted, fe)
210 } else {
211 cf.Invalid = append(cf.Invalid, fe)
212 }
213 }
214
215
216
217
218 haveGoMod := make(map[string]bool)
219 for _, f := range files {
220 p := f.Path()
221 dir, base := path.Split(p)
222 if strings.EqualFold(base, "go.mod") {
223 info, err := f.Lstat()
224 if err != nil {
225 addError(p, false, err)
226 continue
227 }
228 if info.Mode().IsRegular() {
229 haveGoMod[dir] = true
230 }
231 }
232 }
233
234 inSubmodule := func(p string) bool {
235 for {
236 dir, _ := path.Split(p)
237 if dir == "" {
238 return false
239 }
240 if haveGoMod[dir] {
241 return true
242 }
243 p = dir[:len(dir)-1]
244 }
245 }
246
247 collisions := make(collisionChecker)
248 maxSize := int64(MaxZipFile)
249 for _, f := range files {
250 p := f.Path()
251 if p != path.Clean(p) {
252 addError(p, false, errPathNotClean)
253 continue
254 }
255 if path.IsAbs(p) {
256 addError(p, false, errPathNotRelative)
257 continue
258 }
259 if isVendoredPackage(p) {
260
261 addError(p, true, errVendored)
262 continue
263 }
264 if inSubmodule(p) {
265
266 addError(p, true, errSubmoduleFile)
267 continue
268 }
269 if p == ".hg_archival.txt" {
270
271
272 addError(p, true, errHgArchivalTxt)
273 continue
274 }
275 if err := module.CheckFilePath(p); err != nil {
276 addError(p, false, err)
277 continue
278 }
279 if strings.ToLower(p) == "go.mod" && p != "go.mod" {
280 addError(p, false, errGoModCase)
281 continue
282 }
283 info, err := f.Lstat()
284 if err != nil {
285 addError(p, false, err)
286 continue
287 }
288 if err := collisions.check(p, info.IsDir()); err != nil {
289 addError(p, false, err)
290 continue
291 }
292 if info.Mode()&os.ModeType == os.ModeSymlink {
293
294 addError(p, true, errSymlink)
295 continue
296 }
297 if !info.Mode().IsRegular() {
298 addError(p, true, errNotRegular)
299 continue
300 }
301 size := info.Size()
302 if size >= 0 && size <= maxSize {
303 maxSize -= size
304 } else if cf.SizeError == nil {
305 cf.SizeError = fmt.Errorf("module source tree too large (max size is %d bytes)", MaxZipFile)
306 }
307 if p == "go.mod" && size > MaxGoMod {
308 addError(p, false, errGoModSize)
309 continue
310 }
311 if p == "LICENSE" && size > MaxLICENSE {
312 addError(p, false, errLICENSESize)
313 continue
314 }
315
316 cf.Valid = append(cf.Valid, p)
317 validFiles = append(validFiles, f)
318 validSizes = append(validSizes, info.Size())
319 }
320
321 return cf, validFiles, validSizes
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337 func CheckDir(dir string) (CheckedFiles, error) {
338
339
340 files, omitted, err := listFilesInDir(dir)
341 if err != nil {
342 return CheckedFiles{}, err
343 }
344 cf, cfErr := CheckFiles(files)
345 _ = cfErr
346
347
348
349
350 for i := range cf.Valid {
351 cf.Valid[i] = filepath.Join(dir, cf.Valid[i])
352 }
353 cf.Omitted = append(cf.Omitted, omitted...)
354 for i := range cf.Omitted {
355 cf.Omitted[i].Path = filepath.Join(dir, cf.Omitted[i].Path)
356 }
357 for i := range cf.Invalid {
358 cf.Invalid[i].Path = filepath.Join(dir, cf.Invalid[i].Path)
359 }
360 return cf, cf.Err()
361 }
362
363
364
365
366
367
368
369
370
371
372
373
374 func CheckZip(m module.Version, zipFile string) (CheckedFiles, error) {
375 f, err := os.Open(zipFile)
376 if err != nil {
377 return CheckedFiles{}, err
378 }
379 defer f.Close()
380 _, cf, err := checkZip(m, f)
381 return cf, err
382 }
383
384
385
386 func checkZip(m module.Version, f *os.File) (*zip.Reader, CheckedFiles, error) {
387
388 if vers := module.CanonicalVersion(m.Version); vers != m.Version {
389 return nil, CheckedFiles{}, fmt.Errorf("version %q is not canonical (should be %q)", m.Version, vers)
390 }
391 if err := module.Check(m.Path, m.Version); err != nil {
392 return nil, CheckedFiles{}, err
393 }
394
395
396 info, err := f.Stat()
397 if err != nil {
398 return nil, CheckedFiles{}, err
399 }
400 zipSize := info.Size()
401 if zipSize > MaxZipFile {
402 cf := CheckedFiles{SizeError: fmt.Errorf("module zip file is too large (%d bytes; limit is %d bytes)", zipSize, MaxZipFile)}
403 return nil, cf, cf.Err()
404 }
405
406
407 var cf CheckedFiles
408 addError := func(zf *zip.File, err error) {
409 cf.Invalid = append(cf.Invalid, FileError{Path: zf.Name, Err: err})
410 }
411 z, err := zip.NewReader(f, zipSize)
412 if err != nil {
413 return nil, CheckedFiles{}, err
414 }
415 prefix := fmt.Sprintf("%s@%s/", m.Path, m.Version)
416 collisions := make(collisionChecker)
417 var size int64
418 for _, zf := range z.File {
419 if !strings.HasPrefix(zf.Name, prefix) {
420 addError(zf, fmt.Errorf("path does not have prefix %q", prefix))
421 continue
422 }
423 name := zf.Name[len(prefix):]
424 if name == "" {
425 continue
426 }
427 isDir := strings.HasSuffix(name, "/")
428 if isDir {
429 name = name[:len(name)-1]
430 }
431 if path.Clean(name) != name {
432 addError(zf, errPathNotClean)
433 continue
434 }
435 if err := module.CheckFilePath(name); err != nil {
436 addError(zf, err)
437 continue
438 }
439 if err := collisions.check(name, isDir); err != nil {
440 addError(zf, err)
441 continue
442 }
443 if isDir {
444 continue
445 }
446 if base := path.Base(name); strings.EqualFold(base, "go.mod") {
447 if base != name {
448 addError(zf, fmt.Errorf("go.mod file not in module root directory"))
449 continue
450 }
451 if name != "go.mod" {
452 addError(zf, errGoModCase)
453 continue
454 }
455 }
456 sz := int64(zf.UncompressedSize64)
457 if sz >= 0 && MaxZipFile-size >= sz {
458 size += sz
459 } else if cf.SizeError == nil {
460 cf.SizeError = fmt.Errorf("total uncompressed size of module contents too large (max size is %d bytes)", MaxZipFile)
461 }
462 if name == "go.mod" && sz > MaxGoMod {
463 addError(zf, fmt.Errorf("go.mod file too large (max size is %d bytes)", MaxGoMod))
464 continue
465 }
466 if name == "LICENSE" && sz > MaxLICENSE {
467 addError(zf, fmt.Errorf("LICENSE file too large (max size is %d bytes)", MaxLICENSE))
468 continue
469 }
470 cf.Valid = append(cf.Valid, zf.Name)
471 }
472
473 return z, cf, cf.Err()
474 }
475
476
477
478
479
480
481
482
483
484
485 func Create(w io.Writer, m module.Version, files []File) (err error) {
486 defer func() {
487 if err != nil {
488 err = &zipError{verb: "create zip", err: err}
489 }
490 }()
491
492
493
494 if vers := module.CanonicalVersion(m.Version); vers != m.Version {
495 return fmt.Errorf("version %q is not canonical (should be %q)", m.Version, vers)
496 }
497 if err := module.Check(m.Path, m.Version); err != nil {
498 return err
499 }
500
501
502
503 cf, validFiles, validSizes := checkFiles(files)
504 if err := cf.Err(); err != nil {
505 return err
506 }
507
508
509 zw := zip.NewWriter(w)
510 prefix := fmt.Sprintf("%s@%s/", m.Path, m.Version)
511
512 addFile := func(f File, path string, size int64) error {
513 rc, err := f.Open()
514 if err != nil {
515 return err
516 }
517 defer rc.Close()
518 w, err := zw.Create(prefix + path)
519 if err != nil {
520 return err
521 }
522 lr := &io.LimitedReader{R: rc, N: size + 1}
523 if _, err := io.Copy(w, lr); err != nil {
524 return err
525 }
526 if lr.N <= 0 {
527 return fmt.Errorf("file %q is larger than declared size", path)
528 }
529 return nil
530 }
531
532 for i, f := range validFiles {
533 p := f.Path()
534 size := validSizes[i]
535 if err := addFile(f, p, size); err != nil {
536 return err
537 }
538 }
539
540 return zw.Close()
541 }
542
543
544
545
546
547
548
549
550
551
552
553
554 func CreateFromDir(w io.Writer, m module.Version, dir string) (err error) {
555 defer func() {
556 if zerr, ok := err.(*zipError); ok {
557 zerr.path = dir
558 } else if err != nil {
559 err = &zipError{verb: "create zip from directory", path: dir, err: err}
560 }
561 }()
562
563 files, _, err := listFilesInDir(dir)
564 if err != nil {
565 return err
566 }
567
568 return Create(w, m, files)
569 }
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586 func CreateFromVCS(w io.Writer, m module.Version, repoRoot, revision, subdir string) (err error) {
587 defer func() {
588 if zerr, ok := err.(*zipError); ok {
589 zerr.path = repoRoot
590 } else if err != nil {
591 err = &zipError{verb: "create zip from version control system", path: repoRoot, err: err}
592 }
593 }()
594
595 var filesToCreate []File
596
597 switch {
598 case isGitRepo(repoRoot):
599 files, err := filesInGitRepo(repoRoot, revision, subdir)
600 if err != nil {
601 return err
602 }
603
604 filesToCreate = files
605 default:
606 return &UnrecognizedVCSError{RepoRoot: repoRoot}
607 }
608
609 return Create(w, m, filesToCreate)
610 }
611
612
613
614 type UnrecognizedVCSError struct {
615 RepoRoot string
616 }
617
618 func (e *UnrecognizedVCSError) Error() string {
619 return fmt.Sprintf("could not find a recognized version control system at %q", e.RepoRoot)
620 }
621
622
623 func filesInGitRepo(dir, rev, subdir string) ([]File, error) {
624 stderr := bytes.Buffer{}
625 stdout := bytes.Buffer{}
626
627
628
629
630
631
632
633
634
635
636
637
638
639 cmd := exec.Command("git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", rev)
640 if subdir != "" {
641 cmd.Args = append(cmd.Args, subdir)
642 }
643 cmd.Dir = dir
644 cmd.Env = append(os.Environ(), "PWD="+dir)
645 cmd.Stdout = &stdout
646 cmd.Stderr = &stderr
647 if err := cmd.Run(); err != nil {
648 return nil, fmt.Errorf("error running `git archive`: %w, %s", err, stderr.String())
649 }
650
651 rawReader := bytes.NewReader(stdout.Bytes())
652 zipReader, err := zip.NewReader(rawReader, int64(stdout.Len()))
653 if err != nil {
654 return nil, err
655 }
656
657 var fs []File
658 for _, zf := range zipReader.File {
659 if !strings.HasPrefix(zf.Name, subdir) || strings.HasSuffix(zf.Name, "/") {
660 continue
661 }
662
663 n := strings.TrimPrefix(zf.Name, subdir)
664 if n == "" {
665 continue
666 }
667 n = strings.TrimPrefix(n, string(filepath.Separator))
668
669 fs = append(fs, zipFile{
670 name: n,
671 f: zf,
672 })
673 }
674
675 return fs, nil
676 }
677
678
679 func isGitRepo(dir string) bool {
680 stdout := &bytes.Buffer{}
681 cmd := exec.Command("git", "rev-parse", "--git-dir")
682 cmd.Dir = dir
683 cmd.Env = append(os.Environ(), "PWD="+dir)
684 cmd.Stdout = stdout
685 if err := cmd.Run(); err != nil {
686 return false
687 }
688 gitDir := strings.TrimSpace(stdout.String())
689 if !filepath.IsAbs(gitDir) {
690 gitDir = filepath.Join(dir, gitDir)
691 }
692 wantDir := filepath.Join(dir, ".git")
693 return wantDir == gitDir
694 }
695
696 type dirFile struct {
697 filePath, slashPath string
698 info os.FileInfo
699 }
700
701 func (f dirFile) Path() string { return f.slashPath }
702 func (f dirFile) Lstat() (os.FileInfo, error) { return f.info, nil }
703 func (f dirFile) Open() (io.ReadCloser, error) { return os.Open(f.filePath) }
704
705 type zipFile struct {
706 name string
707 f *zip.File
708 }
709
710 func (f zipFile) Path() string { return f.name }
711 func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil }
712 func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
713
714
715
716
717
718
719
720 func isVendoredPackage(name string) bool {
721 var i int
722 if strings.HasPrefix(name, "vendor/") {
723 i += len("vendor/")
724 } else if j := strings.Index(name, "/vendor/"); j >= 0 {
725
726
727
728
729
730
731 i += len("/vendor/")
732 } else {
733 return false
734 }
735 return strings.Contains(name[i:], "/")
736 }
737
738
739
740
741
742
743
744
745
746
747 func Unzip(dir string, m module.Version, zipFile string) (err error) {
748 defer func() {
749 if err != nil {
750 err = &zipError{verb: "unzip", path: zipFile, err: err}
751 }
752 }()
753
754
755
756 if files, _ := ioutil.ReadDir(dir); len(files) > 0 {
757 return fmt.Errorf("target directory %v exists and is not empty", dir)
758 }
759
760
761 f, err := os.Open(zipFile)
762 if err != nil {
763 return err
764 }
765 defer f.Close()
766 z, cf, err := checkZip(m, f)
767 if err != nil {
768 return err
769 }
770 if err := cf.Err(); err != nil {
771 return err
772 }
773
774
775 prefix := fmt.Sprintf("%s@%s/", m.Path, m.Version)
776 if err := os.MkdirAll(dir, 0777); err != nil {
777 return err
778 }
779 for _, zf := range z.File {
780 name := zf.Name[len(prefix):]
781 if name == "" || strings.HasSuffix(name, "/") {
782 continue
783 }
784 dst := filepath.Join(dir, name)
785 if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil {
786 return err
787 }
788 w, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0444)
789 if err != nil {
790 return err
791 }
792 r, err := zf.Open()
793 if err != nil {
794 w.Close()
795 return err
796 }
797 lr := &io.LimitedReader{R: r, N: int64(zf.UncompressedSize64) + 1}
798 _, err = io.Copy(w, lr)
799 r.Close()
800 if err != nil {
801 w.Close()
802 return err
803 }
804 if err := w.Close(); err != nil {
805 return err
806 }
807 if lr.N <= 0 {
808 return fmt.Errorf("uncompressed size of file %s is larger than declared size (%d bytes)", zf.Name, zf.UncompressedSize64)
809 }
810 }
811
812 return nil
813 }
814
815
816
817
818
819
820 type collisionChecker map[string]pathInfo
821
822 type pathInfo struct {
823 path string
824 isDir bool
825 }
826
827 func (cc collisionChecker) check(p string, isDir bool) error {
828 fold := strToFold(p)
829 if other, ok := cc[fold]; ok {
830 if p != other.path {
831 return fmt.Errorf("case-insensitive file name collision: %q and %q", other.path, p)
832 }
833 if isDir != other.isDir {
834 return fmt.Errorf("entry %q is both a file and a directory", p)
835 }
836 if !isDir {
837 return fmt.Errorf("multiple entries for file %q", p)
838 }
839
840
841
842 } else {
843 cc[fold] = pathInfo{path: p, isDir: isDir}
844 }
845
846 if parent := path.Dir(p); parent != "." {
847 return cc.check(parent, true)
848 }
849 return nil
850 }
851
852
853
854
855 func listFilesInDir(dir string) (files []File, omitted []FileError, err error) {
856 err = filepath.Walk(dir, func(filePath string, info os.FileInfo, err error) error {
857 if err != nil {
858 return err
859 }
860 relPath, err := filepath.Rel(dir, filePath)
861 if err != nil {
862 return err
863 }
864 slashPath := filepath.ToSlash(relPath)
865
866
867
868
869
870 if isVendoredPackage(slashPath) {
871 omitted = append(omitted, FileError{Path: slashPath, Err: errVendored})
872 return nil
873 }
874
875 if info.IsDir() {
876 if filePath == dir {
877
878 return nil
879 }
880
881
882
883
884 switch filepath.Base(filePath) {
885 case ".bzr", ".git", ".hg", ".svn":
886 omitted = append(omitted, FileError{Path: slashPath, Err: errVCS})
887 return filepath.SkipDir
888 }
889
890
891 if goModInfo, err := os.Lstat(filepath.Join(filePath, "go.mod")); err == nil && !goModInfo.IsDir() {
892 omitted = append(omitted, FileError{Path: slashPath, Err: errSubmoduleDir})
893 return filepath.SkipDir
894 }
895 return nil
896 }
897
898
899
900 if !info.Mode().IsRegular() {
901 omitted = append(omitted, FileError{Path: slashPath, Err: errNotRegular})
902 return nil
903 }
904
905 files = append(files, dirFile{
906 filePath: filePath,
907 slashPath: slashPath,
908 info: info,
909 })
910 return nil
911 })
912 if err != nil {
913 return nil, nil, err
914 }
915 return files, omitted, nil
916 }
917
918 type zipError struct {
919 verb, path string
920 err error
921 }
922
923 func (e *zipError) Error() string {
924 if e.path == "" {
925 return fmt.Sprintf("%s: %v", e.verb, e.err)
926 } else {
927 return fmt.Sprintf("%s %s: %v", e.verb, e.path, e.err)
928 }
929 }
930
931 func (e *zipError) Unwrap() error {
932 return e.err
933 }
934
935
936
937
938
939
940
941
942
943 func strToFold(s string) string {
944
945
946 for i := 0; i < len(s); i++ {
947 c := s[i]
948 if c >= utf8.RuneSelf || 'A' <= c && c <= 'Z' {
949 goto Slow
950 }
951 }
952 return s
953
954 Slow:
955 var buf bytes.Buffer
956 for _, r := range s {
957
958
959
960 for {
961 r0 := r
962 r = unicode.SimpleFold(r0)
963 if r <= r0 {
964 break
965 }
966 }
967
968 if 'A' <= r && r <= 'Z' {
969 r += 'a' - 'A'
970 }
971 buf.WriteRune(r)
972 }
973 return buf.String()
974 }
975
View as plain text