"Fossies" - the Fresh Open Source Software Archive 
Member "gdrive-2.1.1/drive/sync.go" (28 May 2021, 12788 Bytes) of package /linux/misc/gdrive-2.1.1.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
1 package drive
2
3 import (
4 "fmt"
5 "github.com/sabhiram/go-gitignore"
6 "github.com/soniakeys/graph"
7 "google.golang.org/api/drive/v3"
8 "google.golang.org/api/googleapi"
9 "io"
10 "os"
11 "path/filepath"
12 "strings"
13 "text/tabwriter"
14 "time"
15 )
16
17 const DefaultIgnoreFile = ".gdriveignore"
18
19 type ModTime int
20
21 const (
22 LocalLastModified ModTime = iota
23 RemoteLastModified
24 EqualModifiedTime
25 )
26
27 type LargestSize int
28
29 const (
30 LocalLargestSize LargestSize = iota
31 RemoteLargestSize
32 EqualSize
33 )
34
35 type ConflictResolution int
36
37 const (
38 NoResolution ConflictResolution = iota
39 KeepLocal
40 KeepRemote
41 KeepLargest
42 )
43
44 func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp FileComparer) (*syncFiles, error) {
45 localCh := make(chan struct {
46 files []*LocalFile
47 err error
48 })
49 remoteCh := make(chan struct {
50 files []*RemoteFile
51 err error
52 })
53
54 go func() {
55 files, err := prepareLocalFiles(localPath)
56 localCh <- struct {
57 files []*LocalFile
58 err error
59 }{files, err}
60 }()
61
62 go func() {
63 files, err := self.prepareRemoteFiles(root, "")
64 remoteCh <- struct {
65 files []*RemoteFile
66 err error
67 }{files, err}
68 }()
69
70 local := <-localCh
71 if local.err != nil {
72 return nil, local.err
73 }
74
75 remote := <-remoteCh
76 if remote.err != nil {
77 return nil, remote.err
78 }
79
80 return &syncFiles{
81 root: &RemoteFile{file: root},
82 local: local.files,
83 remote: remote.files,
84 compare: cmp,
85 }, nil
86 }
87
88 func (self *Drive) isSyncFile(id string) (bool, error) {
89 f, err := self.service.Files.Get(id).Fields("appProperties").Do()
90 if err != nil {
91 return false, fmt.Errorf("Failed to get file: %s", err)
92 }
93
94 _, ok := f.AppProperties["sync"]
95 return ok, nil
96 }
97
98 func prepareLocalFiles(root string) ([]*LocalFile, error) {
99 var files []*LocalFile
100
101 // Get absolute root path
102 absRootPath, err := filepath.Abs(root)
103 if err != nil {
104 return nil, err
105 }
106
107 // Prepare ignorer
108 shouldIgnore, err := prepareIgnorer(filepath.Join(absRootPath, DefaultIgnoreFile))
109 if err != nil {
110 return nil, err
111 }
112
113 err = filepath.Walk(absRootPath, func(absPath string, info os.FileInfo, err error) error {
114 if err != nil {
115 return err
116 }
117
118 // Skip root directory
119 if absPath == absRootPath {
120 return nil
121 }
122
123 // Skip files that are not a directory or regular file
124 if !info.IsDir() && !info.Mode().IsRegular() {
125 return nil
126 }
127
128 // Get relative path from root
129 relPath, err := filepath.Rel(absRootPath, absPath)
130 if err != nil {
131 return err
132 }
133
134 // Skip file if it is ignored by ignore file
135 if shouldIgnore(relPath) {
136 return nil
137 }
138
139 files = append(files, &LocalFile{
140 absPath: absPath,
141 relPath: relPath,
142 info: info,
143 })
144
145 return nil
146 })
147
148 if err != nil {
149 return nil, fmt.Errorf("Failed to prepare local files: %s", err)
150 }
151
152 return files, err
153 }
154
155 func (self *Drive) prepareRemoteFiles(rootDir *drive.File, sortOrder string) ([]*RemoteFile, error) {
156 // Find all files which has rootDir as root
157 listArgs := listAllFilesArgs{
158 query: fmt.Sprintf("appProperties has {key='syncRootId' and value='%s'}", rootDir.Id),
159 fields: []googleapi.Field{"nextPageToken", "files(id,name,parents,md5Checksum,mimeType,size,modifiedTime)"},
160 sortOrder: sortOrder,
161 }
162 files, err := self.listAllFiles(listArgs)
163 if err != nil {
164 return nil, fmt.Errorf("Failed listing files: %s", err)
165 }
166
167 if err := checkFiles(files); err != nil {
168 return nil, err
169 }
170
171 relPaths, err := prepareRemoteRelPaths(rootDir, files)
172 if err != nil {
173 return nil, err
174 }
175
176 var remoteFiles []*RemoteFile
177 for _, f := range files {
178 relPath, ok := relPaths[f.Id]
179 if !ok {
180 return nil, fmt.Errorf("File %s does not have a valid parent", f.Id)
181 }
182 remoteFiles = append(remoteFiles, &RemoteFile{
183 relPath: relPath,
184 file: f,
185 })
186 }
187
188 return remoteFiles, nil
189 }
190
191 func prepareRemoteRelPaths(root *drive.File, files []*drive.File) (map[string]string, error) {
192 // The tree only holds integer values so we use
193 // maps to lookup file by index and index by file id
194 indexLookup := map[string]graph.NI{}
195 fileLookup := map[graph.NI]*drive.File{}
196
197 // All files includes root dir
198 allFiles := append([]*drive.File{root}, files...)
199
200 // Prepare lookup maps
201 for i, f := range allFiles {
202 indexLookup[f.Id] = graph.NI(i)
203 fileLookup[graph.NI(i)] = f
204 }
205
206 // This will hold 'parent index' -> 'file index' relationships
207 pathEnds := make([]graph.PathEnd, len(allFiles))
208
209 // Prepare parent -> file relationships
210 for i, f := range allFiles {
211 if f == root {
212 pathEnds[i] = graph.PathEnd{From: -1}
213 continue
214 }
215
216 // Lookup index of parent
217 parentIdx, found := indexLookup[f.Parents[0]]
218 if !found {
219 return nil, fmt.Errorf("Could not find parent of %s (%s)", f.Id, f.Name)
220 }
221 pathEnds[i] = graph.PathEnd{From: parentIdx}
222 }
223
224 // Create parent pointer tree and calculate path lengths
225 tree := &graph.FromList{Paths: pathEnds}
226 tree.RecalcLeaves()
227 tree.RecalcLen()
228
229 // This will hold a map of file id => relative path
230 paths := map[string]string{}
231
232 // Find relative path from root for all files
233 for _, f := range allFiles {
234 if f == root {
235 continue
236 }
237
238 // Find nodes between root and file
239 nodes := tree.PathTo(indexLookup[f.Id], nil)
240
241 // This will hold the name of all paths between root and
242 // file (exluding root and including file itself)
243 pathNames := []string{}
244
245 // Lookup file for each node and grab name
246 for _, n := range nodes {
247 file := fileLookup[n]
248 if file == root {
249 continue
250 }
251 pathNames = append(pathNames, file.Name)
252 }
253
254 // Join path names to form relative path and add to map
255 paths[f.Id] = filepath.Join(pathNames...)
256 }
257
258 return paths, nil
259 }
260
261 func checkFiles(files []*drive.File) error {
262 uniq := map[string]string{}
263
264 for _, f := range files {
265 // Ensure all files have exactly one parent
266 if len(f.Parents) != 1 {
267 return fmt.Errorf("File %s does not have exacly one parent", f.Id)
268 }
269
270 // Ensure that there are no duplicate files
271 uniqKey := f.Name + f.Parents[0]
272 if dupeId, isDupe := uniq[uniqKey]; isDupe {
273 return fmt.Errorf("Found name collision between %s and %s", f.Id, dupeId)
274 }
275 uniq[uniqKey] = f.Id
276 }
277
278 return nil
279 }
280
281 type LocalFile struct {
282 absPath string
283 relPath string
284 info os.FileInfo
285 }
286
287 type RemoteFile struct {
288 relPath string
289 file *drive.File
290 }
291
292 type changedFile struct {
293 local *LocalFile
294 remote *RemoteFile
295 }
296
297 type syncFiles struct {
298 root *RemoteFile
299 local []*LocalFile
300 remote []*RemoteFile
301 compare FileComparer
302 }
303
304 type FileComparer interface {
305 Changed(*LocalFile, *RemoteFile) bool
306 }
307
308 func (self LocalFile) AbsPath() string {
309 return self.absPath
310 }
311
312 func (self LocalFile) Size() int64 {
313 return self.info.Size()
314 }
315
316 func (self LocalFile) Modified() time.Time {
317 return self.info.ModTime()
318 }
319
320 func (self RemoteFile) Md5() string {
321 return self.file.Md5Checksum
322 }
323
324 func (self RemoteFile) Size() int64 {
325 return self.file.Size
326 }
327
328 func (self RemoteFile) Modified() time.Time {
329 t, _ := time.Parse(time.RFC3339, self.file.ModifiedTime)
330 return t
331 }
332
333 func (self *changedFile) compareModTime() ModTime {
334 localTime := self.local.Modified()
335 remoteTime := self.remote.Modified()
336
337 if localTime.After(remoteTime) {
338 return LocalLastModified
339 }
340
341 if remoteTime.After(localTime) {
342 return RemoteLastModified
343 }
344
345 return EqualModifiedTime
346 }
347
348 func (self *changedFile) compareSize() LargestSize {
349 localSize := self.local.Size()
350 remoteSize := self.remote.Size()
351
352 if localSize > remoteSize {
353 return LocalLargestSize
354 }
355
356 if remoteSize > localSize {
357 return RemoteLargestSize
358 }
359
360 return EqualSize
361 }
362
363 func (self *syncFiles) filterMissingRemoteDirs() []*LocalFile {
364 var files []*LocalFile
365
366 for _, lf := range self.local {
367 if lf.info.IsDir() && !self.existsRemote(lf) {
368 files = append(files, lf)
369 }
370 }
371
372 return files
373 }
374
375 func (self *syncFiles) filterMissingLocalDirs() []*RemoteFile {
376 var files []*RemoteFile
377
378 for _, rf := range self.remote {
379 if isDir(rf.file) && !self.existsLocal(rf) {
380 files = append(files, rf)
381 }
382 }
383
384 return files
385 }
386
387 func (self *syncFiles) filterMissingRemoteFiles() []*LocalFile {
388 var files []*LocalFile
389
390 for _, lf := range self.local {
391 if !lf.info.IsDir() && !self.existsRemote(lf) {
392 files = append(files, lf)
393 }
394 }
395
396 return files
397 }
398
399 func (self *syncFiles) filterMissingLocalFiles() []*RemoteFile {
400 var files []*RemoteFile
401
402 for _, rf := range self.remote {
403 if !isDir(rf.file) && !self.existsLocal(rf) {
404 files = append(files, rf)
405 }
406 }
407
408 return files
409 }
410
411 func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
412 var files []*changedFile
413
414 for _, lf := range self.local {
415 // Skip directories
416 if lf.info.IsDir() {
417 continue
418 }
419
420 // Skip files that don't exist on drive
421 rf, found := self.findRemoteByPath(lf.relPath)
422 if !found {
423 continue
424 }
425
426 // Check if file has changed
427 if self.compare.Changed(lf, rf) {
428 files = append(files, &changedFile{
429 local: lf,
430 remote: rf,
431 })
432 }
433 }
434
435 return files
436 }
437
438 func (self *syncFiles) filterChangedRemoteFiles() []*changedFile {
439 var files []*changedFile
440
441 for _, rf := range self.remote {
442 // Skip directories
443 if isDir(rf.file) {
444 continue
445 }
446
447 // Skip local files that don't exist
448 lf, found := self.findLocalByPath(rf.relPath)
449 if !found {
450 continue
451 }
452
453 // Check if file has changed
454 if self.compare.Changed(lf, rf) {
455 files = append(files, &changedFile{
456 local: lf,
457 remote: rf,
458 })
459 }
460 }
461
462 return files
463 }
464
465 func (self *syncFiles) filterExtraneousRemoteFiles() []*RemoteFile {
466 var files []*RemoteFile
467
468 for _, rf := range self.remote {
469 if !self.existsLocal(rf) {
470 files = append(files, rf)
471 }
472 }
473
474 return files
475 }
476
477 func (self *syncFiles) filterExtraneousLocalFiles() []*LocalFile {
478 var files []*LocalFile
479
480 for _, lf := range self.local {
481 if !self.existsRemote(lf) {
482 files = append(files, lf)
483 }
484 }
485
486 return files
487 }
488
489 func (self *syncFiles) existsRemote(lf *LocalFile) bool {
490 _, found := self.findRemoteByPath(lf.relPath)
491 return found
492 }
493
494 func (self *syncFiles) existsLocal(rf *RemoteFile) bool {
495 _, found := self.findLocalByPath(rf.relPath)
496 return found
497 }
498
499 func (self *syncFiles) findRemoteByPath(relPath string) (*RemoteFile, bool) {
500 if relPath == "." {
501 return self.root, true
502 }
503
504 for _, rf := range self.remote {
505 if relPath == rf.relPath {
506 return rf, true
507 }
508 }
509
510 return nil, false
511 }
512
513 func (self *syncFiles) findLocalByPath(relPath string) (*LocalFile, bool) {
514 for _, lf := range self.local {
515 if relPath == lf.relPath {
516 return lf, true
517 }
518 }
519
520 return nil, false
521 }
522
523 func findLocalConflicts(files []*changedFile) []*changedFile {
524 var conflicts []*changedFile
525
526 for _, cf := range files {
527 if cf.compareModTime() == LocalLastModified {
528 conflicts = append(conflicts, cf)
529 }
530 }
531
532 return conflicts
533 }
534
535 func findRemoteConflicts(files []*changedFile) []*changedFile {
536 var conflicts []*changedFile
537
538 for _, cf := range files {
539 if cf.compareModTime() == RemoteLastModified {
540 conflicts = append(conflicts, cf)
541 }
542 }
543
544 return conflicts
545 }
546
547 type byLocalPathLength []*LocalFile
548
549 func (self byLocalPathLength) Len() int {
550 return len(self)
551 }
552
553 func (self byLocalPathLength) Swap(i, j int) {
554 self[i], self[j] = self[j], self[i]
555 }
556
557 func (self byLocalPathLength) Less(i, j int) bool {
558 return pathLength(self[i].relPath) < pathLength(self[j].relPath)
559 }
560
561 type byRemotePathLength []*RemoteFile
562
563 func (self byRemotePathLength) Len() int {
564 return len(self)
565 }
566
567 func (self byRemotePathLength) Swap(i, j int) {
568 self[i], self[j] = self[j], self[i]
569 }
570
571 func (self byRemotePathLength) Less(i, j int) bool {
572 return pathLength(self[i].relPath) < pathLength(self[j].relPath)
573 }
574
575 type byRemotePath []*RemoteFile
576
577 func (self byRemotePath) Len() int {
578 return len(self)
579 }
580
581 func (self byRemotePath) Swap(i, j int) {
582 self[i], self[j] = self[j], self[i]
583 }
584
585 func (self byRemotePath) Less(i, j int) bool {
586 return strings.ToLower(self[i].relPath) < strings.ToLower(self[j].relPath)
587 }
588
589 type ignoreFunc func(string) bool
590
591 func prepareIgnorer(path string) (ignoreFunc, error) {
592 acceptAll := func(string) bool {
593 return false
594 }
595
596 if !fileExists(path) {
597 return acceptAll, nil
598 }
599
600 ignorer, err := ignore.CompileIgnoreFile(path)
601 if err != nil {
602 return acceptAll, fmt.Errorf("Failed to prepare ignorer: %s", err)
603 }
604
605 return ignorer.MatchesPath, nil
606 }
607
608 func formatConflicts(conflicts []*changedFile, out io.Writer) {
609 w := new(tabwriter.Writer)
610 w.Init(out, 0, 0, 3, ' ', 0)
611
612 fmt.Fprintln(w, "Path\tSize Local\tSize Remote\tModified Local\tModified Remote")
613
614 for _, cf := range conflicts {
615 fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
616 truncateString(cf.local.relPath, 60),
617 formatSize(cf.local.Size(), false),
618 formatSize(cf.remote.Size(), false),
619 cf.local.Modified().Local().Format("Jan _2 2006 15:04:05.000"),
620 cf.remote.Modified().Local().Format("Jan _2 2006 15:04:05.000"),
621 )
622 }
623
624 w.Flush()
625 }