soong_zip: add --ignore_missing_files flag

soong_zip builds a list of files to zip early and then starts
zipping them all.  If a directory being zipped is concurrently
modified, a file that existed when soong_zip started may not
still exist.  Add a flag that continues when an expected file
does not exist.  Print a warning, since this should be rare
in normal usages but is a sign of a problem if it happens
regularly.

Test: zip_test.go
Test: m checkbuild
Test: m platform
Change-Id: I78426fe66fded8528ddd436c0f71a7442183cfeb

3 files changed, 73 insertions, 26 deletions

diff --git a/zip/cmd/main.go b/zip/cmd/main.go
index 4a084912..b4f75f7a 100644
--- a/zip/cmd/main.go
+++ b/zip/cmd/main.go
@@ -136,7 +136,7 @@ func main() {
         compLevel := flags.Int("L", 5, "deflate compression level (0-9)")
         emulateJar := flags.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'")
         writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
-
+        ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
         symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
 
         parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
@@ -200,6 +200,7 @@ func main() {
                 NonDeflatedFiles:         nonDeflatedFiles,
                 WriteIfChanged:           *writeIfChanged,
                 StoreSymlinks:            *symlinks,
+                IgnoreMissingFiles:       *ignoreMissingFiles,
         })
         if err != nil {
                 fmt.Fprintln(os.Stderr, "error:", err.Error())
diff --git a/zip/zip.go b/zip/zip.go
index d8507df2..774966a7 100644
--- a/zip/zip.go
+++ b/zip/zip.go
@@ -188,9 +188,11 @@ type ZipWriter struct {
         compressorPool sync.Pool
         compLevel      int
 
-        followSymlinks pathtools.ShouldFollowSymlinks
+        followSymlinks     pathtools.ShouldFollowSymlinks
+        ignoreMissingFiles bool
 
-        fs pathtools.FileSystem
+        stderr io.Writer
+        fs     pathtools.FileSystem
 }
 
 type zipEntry struct {
@@ -215,7 +217,9 @@ type ZipArgs struct {
         NonDeflatedFiles         map[string]bool
         WriteIfChanged           bool
         StoreSymlinks            bool
+        IgnoreMissingFiles       bool
 
+        Stderr     io.Writer
         Filesystem pathtools.FileSystem
 }
 
@@ -271,19 +275,25 @@ func ZipTo(args ZipArgs, w io.Writer) error {
         followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
 
         z := &ZipWriter{
-                time:           jar.DefaultTime,
-                createdDirs:    make(map[string]string),
-                createdFiles:   make(map[string]string),
-                directories:    args.AddDirectoryEntriesToZip,
-                compLevel:      args.CompressionLevel,
-                followSymlinks: followSymlinks,
-                fs:             args.Filesystem,
+                time:               jar.DefaultTime,
+                createdDirs:        make(map[string]string),
+                createdFiles:       make(map[string]string),
+                directories:        args.AddDirectoryEntriesToZip,
+                compLevel:          args.CompressionLevel,
+                followSymlinks:     followSymlinks,
+                ignoreMissingFiles: args.IgnoreMissingFiles,
+                stderr:             args.Stderr,
+                fs:                 args.Filesystem,
         }
 
         if z.fs == nil {
                 z.fs = pathtools.OsFs
         }
 
+        if z.stderr == nil {
+                z.stderr = os.Stderr
+        }
+
         pathMappings := []pathMapping{}
 
         noCompression := args.CompressionLevel == 0
@@ -301,29 +311,44 @@ func ZipTo(args ZipArgs, w io.Writer) error {
                                 return err
                         }
                         if len(globbed) == 0 {
-                                return &os.PathError{
-                                        Op:   "stat",
+                                err := &os.PathError{
+                                        Op:   "lstat",
                                         Path: s,
                                         Err:  os.ErrNotExist,
                                 }
+                                if args.IgnoreMissingFiles {
+                                        fmt.Fprintln(args.Stderr, "warning:", err)
+                                } else {
+                                        return err
+                                }
                         }
                         srcs = append(srcs, globbed...)
                 }
                 if fa.GlobDir != "" {
                         if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
                                 return err
-                        } else if !exists {
-                                return &os.PathError{
-                                        Op:   "stat",
+                        } else if !exists && !args.IgnoreMissingFiles {
+                                err := &os.PathError{
+                                        Op:   "lstat",
                                         Path: fa.GlobDir,
                                         Err:  os.ErrNotExist,
                                 }
-                        } else if !isDir {
-                                return &os.PathError{
-                                        Op:   "stat",
+                                if args.IgnoreMissingFiles {
+                                        fmt.Fprintln(args.Stderr, "warning:", err)
+                                } else {
+                                        return err
+                                }
+                        } else if !isDir && !args.IgnoreMissingFiles {
+                                err := &os.PathError{
+                                        Op:   "lstat",
                                         Path: fa.GlobDir,
                                         Err:  syscall.ENOTDIR,
                                 }
+                                if args.IgnoreMissingFiles {
+                                        fmt.Fprintln(args.Stderr, "warning:", err)
+                                } else {
+                                        return err
+                                }
                         }
                         globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
                         if err != nil {
@@ -576,6 +601,10 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
         }
 
         if err != nil {
+                if os.IsNotExist(err) && z.ignoreMissingFiles {
+                        fmt.Fprintln(z.stderr, "warning:", err)
+                        return nil
+                }
                 return err
         } else if s.IsDir() {
                 if z.directories {
diff --git a/zip/zip_test.go b/zip/zip_test.go
index a08fb126..93c5f3de 100644
--- a/zip/zip_test.go
+++ b/zip/zip_test.go
@@ -98,14 +98,15 @@ func fileArgsBuilder() *FileArgsBuilder {
 
 func TestZip(t *testing.T) {
         testCases := []struct {
-                name             string
-                args             *FileArgsBuilder
-                compressionLevel int
-                emulateJar       bool
-                nonDeflatedFiles map[string]bool
-                dirEntries       bool
-                manifest         string
-                storeSymlinks    bool
+                name               string
+                args               *FileArgsBuilder
+                compressionLevel   int
+                emulateJar         bool
+                nonDeflatedFiles   map[string]bool
+                dirEntries         bool
+                manifest           string
+                storeSymlinks      bool
+                ignoreMissingFiles bool
 
                 files []zip.FileHeader
                 err   error
@@ -338,6 +339,20 @@ func TestZip(t *testing.T) {
                                 fh("a/a/b", fileB, zip.Deflate),
                         },
                 },
+                {
+                        name: "ignore missing files",
+                        args: fileArgsBuilder().
+                                File("a/a/a").
+                                File("a/a/b").
+                                File("missing"),
+                        compressionLevel:   9,
+                        ignoreMissingFiles: true,
+
+                        files: []zip.FileHeader{
+                                fh("a/a/a", fileA, zip.Deflate),
+                                fh("a/a/b", fileB, zip.Deflate),
+                        },
+                },
 
                 // errors
                 {
@@ -381,7 +396,9 @@ func TestZip(t *testing.T) {
                         args.NonDeflatedFiles = test.nonDeflatedFiles
                         args.ManifestSourcePath = test.manifest
                         args.StoreSymlinks = test.storeSymlinks
+                        args.IgnoreMissingFiles = test.ignoreMissingFiles
                         args.Filesystem = mockFs
+                        args.Stderr = &bytes.Buffer{}
 
                         buf := &bytes.Buffer{}
                         err := ZipTo(args, buf)