"Fossies" - the Fresh Open Source Software Archive

Member "gdrive-2.1.1/drive/sync_upload.go" (28 May 2021, 13096 Bytes) of package /linux/misc/old/gdrive-2.1.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 package drive
    2 
    3 import (
    4     "bytes"
    5     "fmt"
    6     "google.golang.org/api/drive/v3"
    7     "google.golang.org/api/googleapi"
    8     "io"
    9     "os"
   10     "path/filepath"
   11     "sort"
   12     "time"
   13 )
   14 
   15 type UploadSyncArgs struct {
   16     Out              io.Writer
   17     Progress         io.Writer
   18     Path             string
   19     RootId           string
   20     DryRun           bool
   21     DeleteExtraneous bool
   22     ChunkSize        int64
   23     Timeout          time.Duration
   24     Resolution       ConflictResolution
   25     Comparer         FileComparer
   26 }
   27 
   28 func (self *Drive) UploadSync(args UploadSyncArgs) error {
   29     if args.ChunkSize > intMax()-1 {
   30         return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax()-1)
   31     }
   32 
   33     fmt.Fprintln(args.Out, "Starting sync...")
   34     started := time.Now()
   35 
   36     // Create root directory if it does not exist
   37     rootDir, err := self.prepareSyncRoot(args)
   38     if err != nil {
   39         return err
   40     }
   41 
   42     fmt.Fprintln(args.Out, "Collecting local and remote file information...")
   43     files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
   44     if err != nil {
   45         return err
   46     }
   47 
   48     // Find missing and changed files
   49     changedFiles := files.filterChangedLocalFiles()
   50     missingFiles := files.filterMissingRemoteFiles()
   51 
   52     fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))
   53 
   54     // Ensure that there is enough free space on drive
   55     if ok, msg := self.checkRemoteFreeSpace(missingFiles, changedFiles); !ok {
   56         return fmt.Errorf(msg)
   57     }
   58 
   59     // Ensure that we don't overwrite any remote changes
   60     if args.Resolution == NoResolution {
   61         err = ensureNoRemoteModifications(changedFiles)
   62         if err != nil {
   63             return fmt.Errorf("Conflict detected!\nThe following files have changed and the remote file are newer than it's local counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
   64         }
   65     }
   66 
   67     // Create missing directories
   68     files, err = self.createMissingRemoteDirs(files, args)
   69     if err != nil {
   70         return err
   71     }
   72 
   73     // Upload missing files
   74     err = self.uploadMissingFiles(missingFiles, files, args)
   75     if err != nil {
   76         return err
   77     }
   78 
   79     // Update modified files
   80     err = self.updateChangedFiles(changedFiles, rootDir, args)
   81     if err != nil {
   82         return err
   83     }
   84 
   85     // Delete extraneous files on drive
   86     if args.DeleteExtraneous {
   87         err = self.deleteExtraneousRemoteFiles(files, args)
   88         if err != nil {
   89             return err
   90         }
   91     }
   92     fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))
   93 
   94     return nil
   95 }
   96 
   97 func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) {
   98     fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
   99     f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do()
  100     if err != nil {
  101         return nil, fmt.Errorf("Failed to find root dir: %s", err)
  102     }
  103 
  104     // Ensure file is a directory
  105     if !isDir(f) {
  106         return nil, fmt.Errorf("Provided root id is not a directory")
  107     }
  108 
  109     // Return directory if syncRoot property is already set
  110     if _, ok := f.AppProperties["syncRoot"]; ok {
  111         return f, nil
  112     }
  113 
  114     // This is the first time this directory have been used for sync
  115     // Check if the directory is empty
  116     isEmpty, err := self.dirIsEmpty(f.Id)
  117     if err != nil {
  118         return nil, fmt.Errorf("Failed to check if root dir is empty: %s", err)
  119     }
  120 
  121     // Ensure that the directory is empty
  122     if !isEmpty {
  123         return nil, fmt.Errorf("Root directory is not empty, the initial sync requires an empty directory")
  124     }
  125 
  126     // Update directory with syncRoot property
  127     dstFile := &drive.File{
  128         AppProperties: map[string]string{"sync": "true", "syncRoot": "true"},
  129     }
  130 
  131     f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do()
  132     if err != nil {
  133         return nil, fmt.Errorf("Failed to update root directory: %s", err)
  134     }
  135 
  136     return f, nil
  137 }
  138 
  139 func (self *Drive) createMissingRemoteDirs(files *syncFiles, args UploadSyncArgs) (*syncFiles, error) {
  140     missingDirs := files.filterMissingRemoteDirs()
  141     missingCount := len(missingDirs)
  142 
  143     if missingCount > 0 {
  144         fmt.Fprintf(args.Out, "\n%d remote directories are missing\n", missingCount)
  145     }
  146 
  147     // Sort directories so that the dirs with the shortest path comes first
  148     sort.Sort(byLocalPathLength(missingDirs))
  149 
  150     for i, lf := range missingDirs {
  151         parentPath := parentFilePath(lf.relPath)
  152         parent, ok := files.findRemoteByPath(parentPath)
  153         if !ok {
  154             return nil, fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
  155         }
  156 
  157         fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i+1, missingCount, filepath.Join(files.root.file.Name, lf.relPath))
  158 
  159         f, err := self.createMissingRemoteDir(createMissingRemoteDirArgs{
  160             name:     lf.info.Name(),
  161             parentId: parent.file.Id,
  162             rootId:   args.RootId,
  163             dryRun:   args.DryRun,
  164             try:      0,
  165         })
  166         if err != nil {
  167             return nil, err
  168         }
  169 
  170         files.remote = append(files.remote, &RemoteFile{
  171             relPath: lf.relPath,
  172             file:    f,
  173         })
  174     }
  175 
  176     return files, nil
  177 }
  178 
  179 type createMissingRemoteDirArgs struct {
  180     name     string
  181     parentId string
  182     rootId   string
  183     dryRun   bool
  184     try      int
  185 }
  186 
  187 func (self *Drive) uploadMissingFiles(missingFiles []*LocalFile, files *syncFiles, args UploadSyncArgs) error {
  188     missingCount := len(missingFiles)
  189 
  190     if missingCount > 0 {
  191         fmt.Fprintf(args.Out, "\n%d remote files are missing\n", missingCount)
  192     }
  193 
  194     for i, lf := range missingFiles {
  195         parentPath := parentFilePath(lf.relPath)
  196         parent, ok := files.findRemoteByPath(parentPath)
  197         if !ok {
  198             return fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
  199         }
  200 
  201         fmt.Fprintf(args.Out, "[%04d/%04d] Uploading %s -> %s\n", i+1, missingCount, lf.relPath, filepath.Join(files.root.file.Name, lf.relPath))
  202 
  203         err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
  204         if err != nil {
  205             return err
  206         }
  207     }
  208 
  209     return nil
  210 }
  211 
  212 func (self *Drive) updateChangedFiles(changedFiles []*changedFile, root *drive.File, args UploadSyncArgs) error {
  213     changedCount := len(changedFiles)
  214 
  215     if changedCount > 0 {
  216         fmt.Fprintf(args.Out, "\n%d local files has changed\n", changedCount)
  217     }
  218 
  219     for i, cf := range changedFiles {
  220         if skip, reason := checkRemoteConflict(cf, args.Resolution); skip {
  221             fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i+1, changedCount, cf.local.relPath, reason)
  222             continue
  223         }
  224 
  225         fmt.Fprintf(args.Out, "[%04d/%04d] Updating %s -> %s\n", i+1, changedCount, cf.local.relPath, filepath.Join(root.Name, cf.local.relPath))
  226 
  227         err := self.updateChangedFile(cf, args, 0)
  228         if err != nil {
  229             return err
  230         }
  231     }
  232 
  233     return nil
  234 }
  235 
  236 func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSyncArgs) error {
  237     extraneousFiles := files.filterExtraneousRemoteFiles()
  238     extraneousCount := len(extraneousFiles)
  239 
  240     if extraneousCount > 0 {
  241         fmt.Fprintf(args.Out, "\n%d remote files are extraneous\n", extraneousCount)
  242     }
  243 
  244     // Sort files so that the files with the longest path comes first
  245     sort.Sort(sort.Reverse(byRemotePathLength(extraneousFiles)))
  246 
  247     for i, rf := range extraneousFiles {
  248         fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i+1, extraneousCount, filepath.Join(files.root.file.Name, rf.relPath))
  249 
  250         err := self.deleteRemoteFile(rf, args, 0)
  251         if err != nil {
  252             return err
  253         }
  254     }
  255 
  256     return nil
  257 }
  258 
  259 func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*drive.File, error) {
  260     dstFile := &drive.File{
  261         Name:          args.name,
  262         MimeType:      DirectoryMimeType,
  263         Parents:       []string{args.parentId},
  264         AppProperties: map[string]string{"sync": "true", "syncRootId": args.rootId},
  265     }
  266 
  267     if args.dryRun {
  268         return dstFile, nil
  269     }
  270 
  271     f, err := self.service.Files.Create(dstFile).Do()
  272     if err != nil {
  273         if isBackendOrRateLimitError(err) && args.try < MaxErrorRetries {
  274             exponentialBackoffSleep(args.try)
  275             args.try++
  276             return self.createMissingRemoteDir(args)
  277         } else {
  278             return nil, fmt.Errorf("Failed to create directory: %s", err)
  279         }
  280     }
  281 
  282     return f, nil
  283 }
  284 
  285 func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
  286     if args.DryRun {
  287         return nil
  288     }
  289 
  290     srcFile, err := os.Open(lf.absPath)
  291     if err != nil {
  292         return fmt.Errorf("Failed to open file: %s", err)
  293     }
  294 
  295     // Close file on function exit
  296     defer srcFile.Close()
  297 
  298     // Instantiate drive file
  299     dstFile := &drive.File{
  300         Name:          lf.info.Name(),
  301         Parents:       []string{parentId},
  302         AppProperties: map[string]string{"sync": "true", "syncRootId": args.RootId},
  303     }
  304 
  305     // Chunk size option
  306     chunkSize := googleapi.ChunkSize(int(args.ChunkSize))
  307 
  308     // Wrap file in progress reader
  309     progressReader := getProgressReader(srcFile, args.Progress, lf.info.Size())
  310 
  311     // Wrap reader in timeout reader
  312     reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout)
  313 
  314     _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Context(ctx).Media(reader, chunkSize).Do()
  315     if err != nil {
  316         if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
  317             exponentialBackoffSleep(try)
  318             try++
  319             return self.uploadMissingFile(parentId, lf, args, try)
  320         } else if isTimeoutError(err) {
  321             return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout)
  322         } else {
  323             return fmt.Errorf("Failed to upload file: %s", err)
  324         }
  325     }
  326 
  327     return nil
  328 }
  329 
  330 func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try int) error {
  331     if args.DryRun {
  332         return nil
  333     }
  334 
  335     srcFile, err := os.Open(cf.local.absPath)
  336     if err != nil {
  337         return fmt.Errorf("Failed to open file: %s", err)
  338     }
  339 
  340     // Close file on function exit
  341     defer srcFile.Close()
  342 
  343     // Instantiate drive file
  344     dstFile := &drive.File{}
  345 
  346     // Chunk size option
  347     chunkSize := googleapi.ChunkSize(int(args.ChunkSize))
  348 
  349     // Wrap file in progress reader
  350     progressReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size())
  351 
  352     // Wrap reader in timeout reader
  353     reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout)
  354 
  355     _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Context(ctx).Media(reader, chunkSize).Do()
  356     if err != nil {
  357         if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
  358             exponentialBackoffSleep(try)
  359             try++
  360             return self.updateChangedFile(cf, args, try)
  361         } else if isTimeoutError(err) {
  362             return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout)
  363         } else {
  364             return fmt.Errorf("Failed to update file: %s", err)
  365         }
  366     }
  367 
  368     return nil
  369 }
  370 
  371 func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int) error {
  372     if args.DryRun {
  373         return nil
  374     }
  375 
  376     err := self.service.Files.Delete(rf.file.Id).Do()
  377     if err != nil {
  378         if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
  379             exponentialBackoffSleep(try)
  380             try++
  381             return self.deleteRemoteFile(rf, args, try)
  382         } else {
  383             return fmt.Errorf("Failed to delete file: %s", err)
  384         }
  385     }
  386 
  387     return nil
  388 }
  389 
  390 func (self *Drive) dirIsEmpty(id string) (bool, error) {
  391     query := fmt.Sprintf("'%s' in parents", id)
  392     fileList, err := self.service.Files.List().Q(query).Do()
  393     if err != nil {
  394         return false, fmt.Errorf("Empty dir check failed: ", err)
  395     }
  396 
  397     return len(fileList.Files) == 0, nil
  398 }
  399 
  400 func checkRemoteConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
  401     // No conflict unless remote file was last modified
  402     if cf.compareModTime() != RemoteLastModified {
  403         return false, ""
  404     }
  405 
  406     // Don't skip if want to keep the local file
  407     if resolution == KeepLocal {
  408         return false, ""
  409     }
  410 
  411     // Skip if we want to keep the remote file
  412     if resolution == KeepRemote {
  413         return true, "conflicting file, keeping remote file"
  414     }
  415 
  416     if resolution == KeepLargest {
  417         largest := cf.compareSize()
  418 
  419         // Skip if the remote file is largest
  420         if largest == RemoteLargestSize {
  421             return true, "conflicting file, remote file is largest, keeping remote"
  422         }
  423 
  424         // Don't skip if the local file is largest
  425         if largest == LocalLargestSize {
  426             return false, ""
  427         }
  428 
  429         // Keep remote if both files have the same size
  430         if largest == EqualSize {
  431             return true, "conflicting file, file sizes are equal, keeping remote"
  432         }
  433     }
  434 
  435     // The conditionals above should cover all cases,
  436     // unless the programmer did something wrong,
  437     // in which case we default to being non-destructive and skip the file
  438     return true, "conflicting file, unhandled case"
  439 }
  440 
  441 func ensureNoRemoteModifications(files []*changedFile) error {
  442     conflicts := findRemoteConflicts(files)
  443     if len(conflicts) == 0 {
  444         return nil
  445     }
  446 
  447     buffer := bytes.NewBufferString("")
  448     formatConflicts(conflicts, buffer)
  449     return fmt.Errorf(buffer.String())
  450 }
  451 
  452 func (self *Drive) checkRemoteFreeSpace(missingFiles []*LocalFile, changedFiles []*changedFile) (bool, string) {
  453     about, err := self.service.About.Get().Fields("storageQuota").Do()
  454     if err != nil {
  455         return false, fmt.Sprintf("Failed to determine free space: %s", err)
  456     }
  457 
  458     quota := about.StorageQuota
  459     if quota.Limit == 0 {
  460         return true, ""
  461     }
  462 
  463     freeSpace := quota.Limit - quota.Usage
  464 
  465     var totalSize int64
  466 
  467     for _, lf := range missingFiles {
  468         totalSize += lf.Size()
  469     }
  470 
  471     for _, cf := range changedFiles {
  472         totalSize += cf.local.Size()
  473     }
  474 
  475     if totalSize > freeSpace {
  476         return false, fmt.Sprintf("Not enough free space, have %s need %s", formatSize(freeSpace, false), formatSize(totalSize, false))
  477     }
  478 
  479     return true, ""
  480 }