"Fossies" - the Fresh Open Source Software Archive

Member "gdrive-2.1.1/drive/sync.go" (28 May 2021, 12788 Bytes) of package /linux/misc/old/gdrive-2.1.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 package drive
    2 
    3 import (
    4     "fmt"
    5     "github.com/sabhiram/go-gitignore"
    6     "github.com/soniakeys/graph"
    7     "google.golang.org/api/drive/v3"
    8     "google.golang.org/api/googleapi"
    9     "io"
   10     "os"
   11     "path/filepath"
   12     "strings"
   13     "text/tabwriter"
   14     "time"
   15 )
   16 
   17 const DefaultIgnoreFile = ".gdriveignore"
   18 
   19 type ModTime int
   20 
   21 const (
   22     LocalLastModified ModTime = iota
   23     RemoteLastModified
   24     EqualModifiedTime
   25 )
   26 
   27 type LargestSize int
   28 
   29 const (
   30     LocalLargestSize LargestSize = iota
   31     RemoteLargestSize
   32     EqualSize
   33 )
   34 
   35 type ConflictResolution int
   36 
   37 const (
   38     NoResolution ConflictResolution = iota
   39     KeepLocal
   40     KeepRemote
   41     KeepLargest
   42 )
   43 
   44 func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp FileComparer) (*syncFiles, error) {
   45     localCh := make(chan struct {
   46         files []*LocalFile
   47         err   error
   48     })
   49     remoteCh := make(chan struct {
   50         files []*RemoteFile
   51         err   error
   52     })
   53 
   54     go func() {
   55         files, err := prepareLocalFiles(localPath)
   56         localCh <- struct {
   57             files []*LocalFile
   58             err   error
   59         }{files, err}
   60     }()
   61 
   62     go func() {
   63         files, err := self.prepareRemoteFiles(root, "")
   64         remoteCh <- struct {
   65             files []*RemoteFile
   66             err   error
   67         }{files, err}
   68     }()
   69 
   70     local := <-localCh
   71     if local.err != nil {
   72         return nil, local.err
   73     }
   74 
   75     remote := <-remoteCh
   76     if remote.err != nil {
   77         return nil, remote.err
   78     }
   79 
   80     return &syncFiles{
   81         root:    &RemoteFile{file: root},
   82         local:   local.files,
   83         remote:  remote.files,
   84         compare: cmp,
   85     }, nil
   86 }
   87 
   88 func (self *Drive) isSyncFile(id string) (bool, error) {
   89     f, err := self.service.Files.Get(id).Fields("appProperties").Do()
   90     if err != nil {
   91         return false, fmt.Errorf("Failed to get file: %s", err)
   92     }
   93 
   94     _, ok := f.AppProperties["sync"]
   95     return ok, nil
   96 }
   97 
   98 func prepareLocalFiles(root string) ([]*LocalFile, error) {
   99     var files []*LocalFile
  100 
  101     // Get absolute root path
  102     absRootPath, err := filepath.Abs(root)
  103     if err != nil {
  104         return nil, err
  105     }
  106 
  107     // Prepare ignorer
  108     shouldIgnore, err := prepareIgnorer(filepath.Join(absRootPath, DefaultIgnoreFile))
  109     if err != nil {
  110         return nil, err
  111     }
  112 
  113     err = filepath.Walk(absRootPath, func(absPath string, info os.FileInfo, err error) error {
  114         if err != nil {
  115             return err
  116         }
  117 
  118         // Skip root directory
  119         if absPath == absRootPath {
  120             return nil
  121         }
  122 
  123         // Skip files that are not a directory or regular file
  124         if !info.IsDir() && !info.Mode().IsRegular() {
  125             return nil
  126         }
  127 
  128         // Get relative path from root
  129         relPath, err := filepath.Rel(absRootPath, absPath)
  130         if err != nil {
  131             return err
  132         }
  133 
  134         // Skip file if it is ignored by ignore file
  135         if shouldIgnore(relPath) {
  136             return nil
  137         }
  138 
  139         files = append(files, &LocalFile{
  140             absPath: absPath,
  141             relPath: relPath,
  142             info:    info,
  143         })
  144 
  145         return nil
  146     })
  147 
  148     if err != nil {
  149         return nil, fmt.Errorf("Failed to prepare local files: %s", err)
  150     }
  151 
  152     return files, err
  153 }
  154 
  155 func (self *Drive) prepareRemoteFiles(rootDir *drive.File, sortOrder string) ([]*RemoteFile, error) {
  156     // Find all files which has rootDir as root
  157     listArgs := listAllFilesArgs{
  158         query:     fmt.Sprintf("appProperties has {key='syncRootId' and value='%s'}", rootDir.Id),
  159         fields:    []googleapi.Field{"nextPageToken", "files(id,name,parents,md5Checksum,mimeType,size,modifiedTime)"},
  160         sortOrder: sortOrder,
  161     }
  162     files, err := self.listAllFiles(listArgs)
  163     if err != nil {
  164         return nil, fmt.Errorf("Failed listing files: %s", err)
  165     }
  166 
  167     if err := checkFiles(files); err != nil {
  168         return nil, err
  169     }
  170 
  171     relPaths, err := prepareRemoteRelPaths(rootDir, files)
  172     if err != nil {
  173         return nil, err
  174     }
  175 
  176     var remoteFiles []*RemoteFile
  177     for _, f := range files {
  178         relPath, ok := relPaths[f.Id]
  179         if !ok {
  180             return nil, fmt.Errorf("File %s does not have a valid parent", f.Id)
  181         }
  182         remoteFiles = append(remoteFiles, &RemoteFile{
  183             relPath: relPath,
  184             file:    f,
  185         })
  186     }
  187 
  188     return remoteFiles, nil
  189 }
  190 
  191 func prepareRemoteRelPaths(root *drive.File, files []*drive.File) (map[string]string, error) {
  192     // The tree only holds integer values so we use
  193     // maps to lookup file by index and index by file id
  194     indexLookup := map[string]graph.NI{}
  195     fileLookup := map[graph.NI]*drive.File{}
  196 
  197     // All files includes root dir
  198     allFiles := append([]*drive.File{root}, files...)
  199 
  200     // Prepare lookup maps
  201     for i, f := range allFiles {
  202         indexLookup[f.Id] = graph.NI(i)
  203         fileLookup[graph.NI(i)] = f
  204     }
  205 
  206     // This will hold 'parent index' -> 'file index' relationships
  207     pathEnds := make([]graph.PathEnd, len(allFiles))
  208 
  209     // Prepare parent -> file relationships
  210     for i, f := range allFiles {
  211         if f == root {
  212             pathEnds[i] = graph.PathEnd{From: -1}
  213             continue
  214         }
  215 
  216         // Lookup index of parent
  217         parentIdx, found := indexLookup[f.Parents[0]]
  218         if !found {
  219             return nil, fmt.Errorf("Could not find parent of %s (%s)", f.Id, f.Name)
  220         }
  221         pathEnds[i] = graph.PathEnd{From: parentIdx}
  222     }
  223 
  224     // Create parent pointer tree and calculate path lengths
  225     tree := &graph.FromList{Paths: pathEnds}
  226     tree.RecalcLeaves()
  227     tree.RecalcLen()
  228 
  229     // This will hold a map of file id => relative path
  230     paths := map[string]string{}
  231 
  232     // Find relative path from root for all files
  233     for _, f := range allFiles {
  234         if f == root {
  235             continue
  236         }
  237 
  238         // Find nodes between root and file
  239         nodes := tree.PathTo(indexLookup[f.Id], nil)
  240 
  241         // This will hold the name of all paths between root and
  242         // file (exluding root and including file itself)
  243         pathNames := []string{}
  244 
  245         // Lookup file for each node and grab name
  246         for _, n := range nodes {
  247             file := fileLookup[n]
  248             if file == root {
  249                 continue
  250             }
  251             pathNames = append(pathNames, file.Name)
  252         }
  253 
  254         // Join path names to form relative path and add to map
  255         paths[f.Id] = filepath.Join(pathNames...)
  256     }
  257 
  258     return paths, nil
  259 }
  260 
  261 func checkFiles(files []*drive.File) error {
  262     uniq := map[string]string{}
  263 
  264     for _, f := range files {
  265         // Ensure all files have exactly one parent
  266         if len(f.Parents) != 1 {
  267             return fmt.Errorf("File %s does not have exacly one parent", f.Id)
  268         }
  269 
  270         // Ensure that there are no duplicate files
  271         uniqKey := f.Name + f.Parents[0]
  272         if dupeId, isDupe := uniq[uniqKey]; isDupe {
  273             return fmt.Errorf("Found name collision between %s and %s", f.Id, dupeId)
  274         }
  275         uniq[uniqKey] = f.Id
  276     }
  277 
  278     return nil
  279 }
  280 
  281 type LocalFile struct {
  282     absPath string
  283     relPath string
  284     info    os.FileInfo
  285 }
  286 
  287 type RemoteFile struct {
  288     relPath string
  289     file    *drive.File
  290 }
  291 
  292 type changedFile struct {
  293     local  *LocalFile
  294     remote *RemoteFile
  295 }
  296 
  297 type syncFiles struct {
  298     root    *RemoteFile
  299     local   []*LocalFile
  300     remote  []*RemoteFile
  301     compare FileComparer
  302 }
  303 
  304 type FileComparer interface {
  305     Changed(*LocalFile, *RemoteFile) bool
  306 }
  307 
  308 func (self LocalFile) AbsPath() string {
  309     return self.absPath
  310 }
  311 
  312 func (self LocalFile) Size() int64 {
  313     return self.info.Size()
  314 }
  315 
  316 func (self LocalFile) Modified() time.Time {
  317     return self.info.ModTime()
  318 }
  319 
  320 func (self RemoteFile) Md5() string {
  321     return self.file.Md5Checksum
  322 }
  323 
  324 func (self RemoteFile) Size() int64 {
  325     return self.file.Size
  326 }
  327 
  328 func (self RemoteFile) Modified() time.Time {
  329     t, _ := time.Parse(time.RFC3339, self.file.ModifiedTime)
  330     return t
  331 }
  332 
  333 func (self *changedFile) compareModTime() ModTime {
  334     localTime := self.local.Modified()
  335     remoteTime := self.remote.Modified()
  336 
  337     if localTime.After(remoteTime) {
  338         return LocalLastModified
  339     }
  340 
  341     if remoteTime.After(localTime) {
  342         return RemoteLastModified
  343     }
  344 
  345     return EqualModifiedTime
  346 }
  347 
  348 func (self *changedFile) compareSize() LargestSize {
  349     localSize := self.local.Size()
  350     remoteSize := self.remote.Size()
  351 
  352     if localSize > remoteSize {
  353         return LocalLargestSize
  354     }
  355 
  356     if remoteSize > localSize {
  357         return RemoteLargestSize
  358     }
  359 
  360     return EqualSize
  361 }
  362 
  363 func (self *syncFiles) filterMissingRemoteDirs() []*LocalFile {
  364     var files []*LocalFile
  365 
  366     for _, lf := range self.local {
  367         if lf.info.IsDir() && !self.existsRemote(lf) {
  368             files = append(files, lf)
  369         }
  370     }
  371 
  372     return files
  373 }
  374 
  375 func (self *syncFiles) filterMissingLocalDirs() []*RemoteFile {
  376     var files []*RemoteFile
  377 
  378     for _, rf := range self.remote {
  379         if isDir(rf.file) && !self.existsLocal(rf) {
  380             files = append(files, rf)
  381         }
  382     }
  383 
  384     return files
  385 }
  386 
  387 func (self *syncFiles) filterMissingRemoteFiles() []*LocalFile {
  388     var files []*LocalFile
  389 
  390     for _, lf := range self.local {
  391         if !lf.info.IsDir() && !self.existsRemote(lf) {
  392             files = append(files, lf)
  393         }
  394     }
  395 
  396     return files
  397 }
  398 
  399 func (self *syncFiles) filterMissingLocalFiles() []*RemoteFile {
  400     var files []*RemoteFile
  401 
  402     for _, rf := range self.remote {
  403         if !isDir(rf.file) && !self.existsLocal(rf) {
  404             files = append(files, rf)
  405         }
  406     }
  407 
  408     return files
  409 }
  410 
  411 func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
  412     var files []*changedFile
  413 
  414     for _, lf := range self.local {
  415         // Skip directories
  416         if lf.info.IsDir() {
  417             continue
  418         }
  419 
  420         // Skip files that don't exist on drive
  421         rf, found := self.findRemoteByPath(lf.relPath)
  422         if !found {
  423             continue
  424         }
  425 
  426         // Check if file has changed
  427         if self.compare.Changed(lf, rf) {
  428             files = append(files, &changedFile{
  429                 local:  lf,
  430                 remote: rf,
  431             })
  432         }
  433     }
  434 
  435     return files
  436 }
  437 
  438 func (self *syncFiles) filterChangedRemoteFiles() []*changedFile {
  439     var files []*changedFile
  440 
  441     for _, rf := range self.remote {
  442         // Skip directories
  443         if isDir(rf.file) {
  444             continue
  445         }
  446 
  447         // Skip local files that don't exist
  448         lf, found := self.findLocalByPath(rf.relPath)
  449         if !found {
  450             continue
  451         }
  452 
  453         // Check if file has changed
  454         if self.compare.Changed(lf, rf) {
  455             files = append(files, &changedFile{
  456                 local:  lf,
  457                 remote: rf,
  458             })
  459         }
  460     }
  461 
  462     return files
  463 }
  464 
  465 func (self *syncFiles) filterExtraneousRemoteFiles() []*RemoteFile {
  466     var files []*RemoteFile
  467 
  468     for _, rf := range self.remote {
  469         if !self.existsLocal(rf) {
  470             files = append(files, rf)
  471         }
  472     }
  473 
  474     return files
  475 }
  476 
  477 func (self *syncFiles) filterExtraneousLocalFiles() []*LocalFile {
  478     var files []*LocalFile
  479 
  480     for _, lf := range self.local {
  481         if !self.existsRemote(lf) {
  482             files = append(files, lf)
  483         }
  484     }
  485 
  486     return files
  487 }
  488 
  489 func (self *syncFiles) existsRemote(lf *LocalFile) bool {
  490     _, found := self.findRemoteByPath(lf.relPath)
  491     return found
  492 }
  493 
  494 func (self *syncFiles) existsLocal(rf *RemoteFile) bool {
  495     _, found := self.findLocalByPath(rf.relPath)
  496     return found
  497 }
  498 
  499 func (self *syncFiles) findRemoteByPath(relPath string) (*RemoteFile, bool) {
  500     if relPath == "." {
  501         return self.root, true
  502     }
  503 
  504     for _, rf := range self.remote {
  505         if relPath == rf.relPath {
  506             return rf, true
  507         }
  508     }
  509 
  510     return nil, false
  511 }
  512 
  513 func (self *syncFiles) findLocalByPath(relPath string) (*LocalFile, bool) {
  514     for _, lf := range self.local {
  515         if relPath == lf.relPath {
  516             return lf, true
  517         }
  518     }
  519 
  520     return nil, false
  521 }
  522 
  523 func findLocalConflicts(files []*changedFile) []*changedFile {
  524     var conflicts []*changedFile
  525 
  526     for _, cf := range files {
  527         if cf.compareModTime() == LocalLastModified {
  528             conflicts = append(conflicts, cf)
  529         }
  530     }
  531 
  532     return conflicts
  533 }
  534 
  535 func findRemoteConflicts(files []*changedFile) []*changedFile {
  536     var conflicts []*changedFile
  537 
  538     for _, cf := range files {
  539         if cf.compareModTime() == RemoteLastModified {
  540             conflicts = append(conflicts, cf)
  541         }
  542     }
  543 
  544     return conflicts
  545 }
  546 
  547 type byLocalPathLength []*LocalFile
  548 
  549 func (self byLocalPathLength) Len() int {
  550     return len(self)
  551 }
  552 
  553 func (self byLocalPathLength) Swap(i, j int) {
  554     self[i], self[j] = self[j], self[i]
  555 }
  556 
  557 func (self byLocalPathLength) Less(i, j int) bool {
  558     return pathLength(self[i].relPath) < pathLength(self[j].relPath)
  559 }
  560 
  561 type byRemotePathLength []*RemoteFile
  562 
  563 func (self byRemotePathLength) Len() int {
  564     return len(self)
  565 }
  566 
  567 func (self byRemotePathLength) Swap(i, j int) {
  568     self[i], self[j] = self[j], self[i]
  569 }
  570 
  571 func (self byRemotePathLength) Less(i, j int) bool {
  572     return pathLength(self[i].relPath) < pathLength(self[j].relPath)
  573 }
  574 
  575 type byRemotePath []*RemoteFile
  576 
  577 func (self byRemotePath) Len() int {
  578     return len(self)
  579 }
  580 
  581 func (self byRemotePath) Swap(i, j int) {
  582     self[i], self[j] = self[j], self[i]
  583 }
  584 
  585 func (self byRemotePath) Less(i, j int) bool {
  586     return strings.ToLower(self[i].relPath) < strings.ToLower(self[j].relPath)
  587 }
  588 
  589 type ignoreFunc func(string) bool
  590 
  591 func prepareIgnorer(path string) (ignoreFunc, error) {
  592     acceptAll := func(string) bool {
  593         return false
  594     }
  595 
  596     if !fileExists(path) {
  597         return acceptAll, nil
  598     }
  599 
  600     ignorer, err := ignore.CompileIgnoreFile(path)
  601     if err != nil {
  602         return acceptAll, fmt.Errorf("Failed to prepare ignorer: %s", err)
  603     }
  604 
  605     return ignorer.MatchesPath, nil
  606 }
  607 
  608 func formatConflicts(conflicts []*changedFile, out io.Writer) {
  609     w := new(tabwriter.Writer)
  610     w.Init(out, 0, 0, 3, ' ', 0)
  611 
  612     fmt.Fprintln(w, "Path\tSize Local\tSize Remote\tModified Local\tModified Remote")
  613 
  614     for _, cf := range conflicts {
  615         fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
  616             truncateString(cf.local.relPath, 60),
  617             formatSize(cf.local.Size(), false),
  618             formatSize(cf.remote.Size(), false),
  619             cf.local.Modified().Local().Format("Jan _2 2006 15:04:05.000"),
  620             cf.remote.Modified().Local().Format("Jan _2 2006 15:04:05.000"),
  621         )
  622     }
  623 
  624     w.Flush()
  625 }