"Fossies" - the Fresh Open Source Software Archive 
Member "gdrive-2.1.1/drive/sync_upload.go" (28 May 2021, 13096 Bytes) of package /linux/misc/gdrive-2.1.1.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
1 package drive
2
3 import (
4 "bytes"
5 "fmt"
6 "google.golang.org/api/drive/v3"
7 "google.golang.org/api/googleapi"
8 "io"
9 "os"
10 "path/filepath"
11 "sort"
12 "time"
13 )
14
15 type UploadSyncArgs struct {
16 Out io.Writer
17 Progress io.Writer
18 Path string
19 RootId string
20 DryRun bool
21 DeleteExtraneous bool
22 ChunkSize int64
23 Timeout time.Duration
24 Resolution ConflictResolution
25 Comparer FileComparer
26 }
27
28 func (self *Drive) UploadSync(args UploadSyncArgs) error {
29 if args.ChunkSize > intMax()-1 {
30 return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax()-1)
31 }
32
33 fmt.Fprintln(args.Out, "Starting sync...")
34 started := time.Now()
35
36 // Create root directory if it does not exist
37 rootDir, err := self.prepareSyncRoot(args)
38 if err != nil {
39 return err
40 }
41
42 fmt.Fprintln(args.Out, "Collecting local and remote file information...")
43 files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
44 if err != nil {
45 return err
46 }
47
48 // Find missing and changed files
49 changedFiles := files.filterChangedLocalFiles()
50 missingFiles := files.filterMissingRemoteFiles()
51
52 fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))
53
54 // Ensure that there is enough free space on drive
55 if ok, msg := self.checkRemoteFreeSpace(missingFiles, changedFiles); !ok {
56 return fmt.Errorf(msg)
57 }
58
59 // Ensure that we don't overwrite any remote changes
60 if args.Resolution == NoResolution {
61 err = ensureNoRemoteModifications(changedFiles)
62 if err != nil {
63 return fmt.Errorf("Conflict detected!\nThe following files have changed and the remote file are newer than it's local counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
64 }
65 }
66
67 // Create missing directories
68 files, err = self.createMissingRemoteDirs(files, args)
69 if err != nil {
70 return err
71 }
72
73 // Upload missing files
74 err = self.uploadMissingFiles(missingFiles, files, args)
75 if err != nil {
76 return err
77 }
78
79 // Update modified files
80 err = self.updateChangedFiles(changedFiles, rootDir, args)
81 if err != nil {
82 return err
83 }
84
85 // Delete extraneous files on drive
86 if args.DeleteExtraneous {
87 err = self.deleteExtraneousRemoteFiles(files, args)
88 if err != nil {
89 return err
90 }
91 }
92 fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))
93
94 return nil
95 }
96
97 func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) {
98 fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
99 f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do()
100 if err != nil {
101 return nil, fmt.Errorf("Failed to find root dir: %s", err)
102 }
103
104 // Ensure file is a directory
105 if !isDir(f) {
106 return nil, fmt.Errorf("Provided root id is not a directory")
107 }
108
109 // Return directory if syncRoot property is already set
110 if _, ok := f.AppProperties["syncRoot"]; ok {
111 return f, nil
112 }
113
114 // This is the first time this directory have been used for sync
115 // Check if the directory is empty
116 isEmpty, err := self.dirIsEmpty(f.Id)
117 if err != nil {
118 return nil, fmt.Errorf("Failed to check if root dir is empty: %s", err)
119 }
120
121 // Ensure that the directory is empty
122 if !isEmpty {
123 return nil, fmt.Errorf("Root directory is not empty, the initial sync requires an empty directory")
124 }
125
126 // Update directory with syncRoot property
127 dstFile := &drive.File{
128 AppProperties: map[string]string{"sync": "true", "syncRoot": "true"},
129 }
130
131 f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do()
132 if err != nil {
133 return nil, fmt.Errorf("Failed to update root directory: %s", err)
134 }
135
136 return f, nil
137 }
138
139 func (self *Drive) createMissingRemoteDirs(files *syncFiles, args UploadSyncArgs) (*syncFiles, error) {
140 missingDirs := files.filterMissingRemoteDirs()
141 missingCount := len(missingDirs)
142
143 if missingCount > 0 {
144 fmt.Fprintf(args.Out, "\n%d remote directories are missing\n", missingCount)
145 }
146
147 // Sort directories so that the dirs with the shortest path comes first
148 sort.Sort(byLocalPathLength(missingDirs))
149
150 for i, lf := range missingDirs {
151 parentPath := parentFilePath(lf.relPath)
152 parent, ok := files.findRemoteByPath(parentPath)
153 if !ok {
154 return nil, fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
155 }
156
157 fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i+1, missingCount, filepath.Join(files.root.file.Name, lf.relPath))
158
159 f, err := self.createMissingRemoteDir(createMissingRemoteDirArgs{
160 name: lf.info.Name(),
161 parentId: parent.file.Id,
162 rootId: args.RootId,
163 dryRun: args.DryRun,
164 try: 0,
165 })
166 if err != nil {
167 return nil, err
168 }
169
170 files.remote = append(files.remote, &RemoteFile{
171 relPath: lf.relPath,
172 file: f,
173 })
174 }
175
176 return files, nil
177 }
178
179 type createMissingRemoteDirArgs struct {
180 name string
181 parentId string
182 rootId string
183 dryRun bool
184 try int
185 }
186
187 func (self *Drive) uploadMissingFiles(missingFiles []*LocalFile, files *syncFiles, args UploadSyncArgs) error {
188 missingCount := len(missingFiles)
189
190 if missingCount > 0 {
191 fmt.Fprintf(args.Out, "\n%d remote files are missing\n", missingCount)
192 }
193
194 for i, lf := range missingFiles {
195 parentPath := parentFilePath(lf.relPath)
196 parent, ok := files.findRemoteByPath(parentPath)
197 if !ok {
198 return fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
199 }
200
201 fmt.Fprintf(args.Out, "[%04d/%04d] Uploading %s -> %s\n", i+1, missingCount, lf.relPath, filepath.Join(files.root.file.Name, lf.relPath))
202
203 err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
204 if err != nil {
205 return err
206 }
207 }
208
209 return nil
210 }
211
212 func (self *Drive) updateChangedFiles(changedFiles []*changedFile, root *drive.File, args UploadSyncArgs) error {
213 changedCount := len(changedFiles)
214
215 if changedCount > 0 {
216 fmt.Fprintf(args.Out, "\n%d local files has changed\n", changedCount)
217 }
218
219 for i, cf := range changedFiles {
220 if skip, reason := checkRemoteConflict(cf, args.Resolution); skip {
221 fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i+1, changedCount, cf.local.relPath, reason)
222 continue
223 }
224
225 fmt.Fprintf(args.Out, "[%04d/%04d] Updating %s -> %s\n", i+1, changedCount, cf.local.relPath, filepath.Join(root.Name, cf.local.relPath))
226
227 err := self.updateChangedFile(cf, args, 0)
228 if err != nil {
229 return err
230 }
231 }
232
233 return nil
234 }
235
236 func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSyncArgs) error {
237 extraneousFiles := files.filterExtraneousRemoteFiles()
238 extraneousCount := len(extraneousFiles)
239
240 if extraneousCount > 0 {
241 fmt.Fprintf(args.Out, "\n%d remote files are extraneous\n", extraneousCount)
242 }
243
244 // Sort files so that the files with the longest path comes first
245 sort.Sort(sort.Reverse(byRemotePathLength(extraneousFiles)))
246
247 for i, rf := range extraneousFiles {
248 fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i+1, extraneousCount, filepath.Join(files.root.file.Name, rf.relPath))
249
250 err := self.deleteRemoteFile(rf, args, 0)
251 if err != nil {
252 return err
253 }
254 }
255
256 return nil
257 }
258
259 func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*drive.File, error) {
260 dstFile := &drive.File{
261 Name: args.name,
262 MimeType: DirectoryMimeType,
263 Parents: []string{args.parentId},
264 AppProperties: map[string]string{"sync": "true", "syncRootId": args.rootId},
265 }
266
267 if args.dryRun {
268 return dstFile, nil
269 }
270
271 f, err := self.service.Files.Create(dstFile).Do()
272 if err != nil {
273 if isBackendOrRateLimitError(err) && args.try < MaxErrorRetries {
274 exponentialBackoffSleep(args.try)
275 args.try++
276 return self.createMissingRemoteDir(args)
277 } else {
278 return nil, fmt.Errorf("Failed to create directory: %s", err)
279 }
280 }
281
282 return f, nil
283 }
284
285 func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
286 if args.DryRun {
287 return nil
288 }
289
290 srcFile, err := os.Open(lf.absPath)
291 if err != nil {
292 return fmt.Errorf("Failed to open file: %s", err)
293 }
294
295 // Close file on function exit
296 defer srcFile.Close()
297
298 // Instantiate drive file
299 dstFile := &drive.File{
300 Name: lf.info.Name(),
301 Parents: []string{parentId},
302 AppProperties: map[string]string{"sync": "true", "syncRootId": args.RootId},
303 }
304
305 // Chunk size option
306 chunkSize := googleapi.ChunkSize(int(args.ChunkSize))
307
308 // Wrap file in progress reader
309 progressReader := getProgressReader(srcFile, args.Progress, lf.info.Size())
310
311 // Wrap reader in timeout reader
312 reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout)
313
314 _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Context(ctx).Media(reader, chunkSize).Do()
315 if err != nil {
316 if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
317 exponentialBackoffSleep(try)
318 try++
319 return self.uploadMissingFile(parentId, lf, args, try)
320 } else if isTimeoutError(err) {
321 return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout)
322 } else {
323 return fmt.Errorf("Failed to upload file: %s", err)
324 }
325 }
326
327 return nil
328 }
329
330 func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try int) error {
331 if args.DryRun {
332 return nil
333 }
334
335 srcFile, err := os.Open(cf.local.absPath)
336 if err != nil {
337 return fmt.Errorf("Failed to open file: %s", err)
338 }
339
340 // Close file on function exit
341 defer srcFile.Close()
342
343 // Instantiate drive file
344 dstFile := &drive.File{}
345
346 // Chunk size option
347 chunkSize := googleapi.ChunkSize(int(args.ChunkSize))
348
349 // Wrap file in progress reader
350 progressReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size())
351
352 // Wrap reader in timeout reader
353 reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout)
354
355 _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Context(ctx).Media(reader, chunkSize).Do()
356 if err != nil {
357 if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
358 exponentialBackoffSleep(try)
359 try++
360 return self.updateChangedFile(cf, args, try)
361 } else if isTimeoutError(err) {
362 return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout)
363 } else {
364 return fmt.Errorf("Failed to update file: %s", err)
365 }
366 }
367
368 return nil
369 }
370
371 func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int) error {
372 if args.DryRun {
373 return nil
374 }
375
376 err := self.service.Files.Delete(rf.file.Id).Do()
377 if err != nil {
378 if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
379 exponentialBackoffSleep(try)
380 try++
381 return self.deleteRemoteFile(rf, args, try)
382 } else {
383 return fmt.Errorf("Failed to delete file: %s", err)
384 }
385 }
386
387 return nil
388 }
389
390 func (self *Drive) dirIsEmpty(id string) (bool, error) {
391 query := fmt.Sprintf("'%s' in parents", id)
392 fileList, err := self.service.Files.List().Q(query).Do()
393 if err != nil {
394 return false, fmt.Errorf("Empty dir check failed: ", err)
395 }
396
397 return len(fileList.Files) == 0, nil
398 }
399
400 func checkRemoteConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
401 // No conflict unless remote file was last modified
402 if cf.compareModTime() != RemoteLastModified {
403 return false, ""
404 }
405
406 // Don't skip if want to keep the local file
407 if resolution == KeepLocal {
408 return false, ""
409 }
410
411 // Skip if we want to keep the remote file
412 if resolution == KeepRemote {
413 return true, "conflicting file, keeping remote file"
414 }
415
416 if resolution == KeepLargest {
417 largest := cf.compareSize()
418
419 // Skip if the remote file is largest
420 if largest == RemoteLargestSize {
421 return true, "conflicting file, remote file is largest, keeping remote"
422 }
423
424 // Don't skip if the local file is largest
425 if largest == LocalLargestSize {
426 return false, ""
427 }
428
429 // Keep remote if both files have the same size
430 if largest == EqualSize {
431 return true, "conflicting file, file sizes are equal, keeping remote"
432 }
433 }
434
435 // The conditionals above should cover all cases,
436 // unless the programmer did something wrong,
437 // in which case we default to being non-destructive and skip the file
438 return true, "conflicting file, unhandled case"
439 }
440
441 func ensureNoRemoteModifications(files []*changedFile) error {
442 conflicts := findRemoteConflicts(files)
443 if len(conflicts) == 0 {
444 return nil
445 }
446
447 buffer := bytes.NewBufferString("")
448 formatConflicts(conflicts, buffer)
449 return fmt.Errorf(buffer.String())
450 }
451
452 func (self *Drive) checkRemoteFreeSpace(missingFiles []*LocalFile, changedFiles []*changedFile) (bool, string) {
453 about, err := self.service.About.Get().Fields("storageQuota").Do()
454 if err != nil {
455 return false, fmt.Sprintf("Failed to determine free space: %s", err)
456 }
457
458 quota := about.StorageQuota
459 if quota.Limit == 0 {
460 return true, ""
461 }
462
463 freeSpace := quota.Limit - quota.Usage
464
465 var totalSize int64
466
467 for _, lf := range missingFiles {
468 totalSize += lf.Size()
469 }
470
471 for _, cf := range changedFiles {
472 totalSize += cf.local.Size()
473 }
474
475 if totalSize > freeSpace {
476 return false, fmt.Sprintf("Not enough free space, have %s need %s", formatSize(freeSpace, false), formatSize(totalSize, false))
477 }
478
479 return true, ""
480 }