"Fossies" - the Fresh Open Source Software Archive 
Member "gdrive-2.1.1/drive/sync_download.go" (28 May 2021, 8828 Bytes) of package /linux/misc/gdrive-2.1.1.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
1 package drive
2
3 import (
4 "bytes"
5 "fmt"
6 "google.golang.org/api/drive/v3"
7 "google.golang.org/api/googleapi"
8 "io"
9 "os"
10 "path/filepath"
11 "sort"
12 "time"
13 )
14
15 type DownloadSyncArgs struct {
16 Out io.Writer
17 Progress io.Writer
18 RootId string
19 Path string
20 DryRun bool
21 DeleteExtraneous bool
22 Timeout time.Duration
23 Resolution ConflictResolution
24 Comparer FileComparer
25 }
26
27 func (self *Drive) DownloadSync(args DownloadSyncArgs) error {
28 fmt.Fprintln(args.Out, "Starting sync...")
29 started := time.Now()
30
31 // Get remote root dir
32 rootDir, err := self.getSyncRoot(args.RootId)
33 if err != nil {
34 return err
35 }
36
37 fmt.Fprintln(args.Out, "Collecting file information...")
38 files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
39 if err != nil {
40 return err
41 }
42
43 // Find changed files
44 changedFiles := files.filterChangedRemoteFiles()
45
46 fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))
47
48 // Ensure that we don't overwrite any local changes
49 if args.Resolution == NoResolution {
50 err = ensureNoLocalModifications(changedFiles)
51 if err != nil {
52 return fmt.Errorf("Conflict detected!\nThe following files have changed and the local file are newer than it's remote counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
53 }
54 }
55
56 // Create missing directories
57 err = self.createMissingLocalDirs(files, args)
58 if err != nil {
59 return err
60 }
61
62 // Download missing files
63 err = self.downloadMissingFiles(files, args)
64 if err != nil {
65 return err
66 }
67
68 // Download files that has changed
69 err = self.downloadChangedFiles(changedFiles, args)
70 if err != nil {
71 return err
72 }
73
74 // Delete extraneous local files
75 if args.DeleteExtraneous {
76 err = self.deleteExtraneousLocalFiles(files, args)
77 if err != nil {
78 return err
79 }
80 }
81 fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))
82
83 return nil
84 }
85
86 func (self *Drive) getSyncRoot(rootId string) (*drive.File, error) {
87 fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
88 f, err := self.service.Files.Get(rootId).Fields(fields...).Do()
89 if err != nil {
90 return nil, fmt.Errorf("Failed to find root dir: %s", err)
91 }
92
93 // Ensure file is a directory
94 if !isDir(f) {
95 return nil, fmt.Errorf("Provided root id is not a directory")
96 }
97
98 // Ensure directory is a proper syncRoot
99 if _, ok := f.AppProperties["syncRoot"]; !ok {
100 return nil, fmt.Errorf("Provided id is not a sync root directory")
101 }
102
103 return f, nil
104 }
105
106 func (self *Drive) createMissingLocalDirs(files *syncFiles, args DownloadSyncArgs) error {
107 missingDirs := files.filterMissingLocalDirs()
108 missingCount := len(missingDirs)
109
110 if missingCount > 0 {
111 fmt.Fprintf(args.Out, "\n%d local directories are missing\n", missingCount)
112 }
113
114 // Sort directories so that the dirs with the shortest path comes first
115 sort.Sort(byRemotePathLength(missingDirs))
116
117 for i, rf := range missingDirs {
118 absPath, err := filepath.Abs(filepath.Join(args.Path, rf.relPath))
119 if err != nil {
120 return fmt.Errorf("Failed to determine local absolute path: %s", err)
121 }
122 fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i+1, missingCount, filepath.Join(filepath.Base(args.Path), rf.relPath))
123
124 if args.DryRun {
125 continue
126 }
127
128 os.MkdirAll(absPath, 0775)
129 }
130
131 return nil
132 }
133
134 func (self *Drive) downloadMissingFiles(files *syncFiles, args DownloadSyncArgs) error {
135 missingFiles := files.filterMissingLocalFiles()
136 missingCount := len(missingFiles)
137
138 if missingCount > 0 {
139 fmt.Fprintf(args.Out, "\n%d local files are missing\n", missingCount)
140 }
141
142 for i, rf := range missingFiles {
143 absPath, err := filepath.Abs(filepath.Join(args.Path, rf.relPath))
144 if err != nil {
145 return fmt.Errorf("Failed to determine local absolute path: %s", err)
146 }
147 fmt.Fprintf(args.Out, "[%04d/%04d] Downloading %s -> %s\n", i+1, missingCount, rf.relPath, filepath.Join(filepath.Base(args.Path), rf.relPath))
148
149 err = self.downloadRemoteFile(rf.file.Id, absPath, args, 0)
150 if err != nil {
151 return err
152 }
153 }
154
155 return nil
156 }
157
158 func (self *Drive) downloadChangedFiles(changedFiles []*changedFile, args DownloadSyncArgs) error {
159 changedCount := len(changedFiles)
160
161 if changedCount > 0 {
162 fmt.Fprintf(args.Out, "\n%d remote files has changed\n", changedCount)
163 }
164
165 for i, cf := range changedFiles {
166 if skip, reason := checkLocalConflict(cf, args.Resolution); skip {
167 fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i+1, changedCount, cf.remote.relPath, reason)
168 continue
169 }
170
171 absPath, err := filepath.Abs(filepath.Join(args.Path, cf.remote.relPath))
172 if err != nil {
173 return fmt.Errorf("Failed to determine local absolute path: %s", err)
174 }
175 fmt.Fprintf(args.Out, "[%04d/%04d] Downloading %s -> %s\n", i+1, changedCount, cf.remote.relPath, filepath.Join(filepath.Base(args.Path), cf.remote.relPath))
176
177 err = self.downloadRemoteFile(cf.remote.file.Id, absPath, args, 0)
178 if err != nil {
179 return err
180 }
181 }
182
183 return nil
184 }
185
186 func (self *Drive) downloadRemoteFile(id, fpath string, args DownloadSyncArgs, try int) error {
187 if args.DryRun {
188 return nil
189 }
190
191 // Get timeout reader wrapper and context
192 timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout)
193
194 res, err := self.service.Files.Get(id).Context(ctx).Download()
195 if err != nil {
196 if isBackendOrRateLimitError(err) && try < MaxErrorRetries {
197 exponentialBackoffSleep(try)
198 try++
199 return self.downloadRemoteFile(id, fpath, args, try)
200 } else if isTimeoutError(err) {
201 return fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout)
202 } else {
203 return fmt.Errorf("Failed to download file: %s", err)
204 }
205 }
206
207 // Close body on function exit
208 defer res.Body.Close()
209
210 // Wrap response body in progress reader
211 progressReader := getProgressReader(res.Body, args.Progress, res.ContentLength)
212
213 // Wrap reader in timeout reader
214 reader := timeoutReaderWrapper(progressReader)
215
216 // Ensure any parent directories exists
217 if err = mkdir(fpath); err != nil {
218 return err
219 }
220
221 // Download to tmp file
222 tmpPath := fpath + ".incomplete"
223
224 // Create new file
225 outFile, err := os.Create(tmpPath)
226 if err != nil {
227 return fmt.Errorf("Unable to create local file: %s", err)
228 }
229
230 // Save file to disk
231 _, err = io.Copy(outFile, reader)
232 if err != nil {
233 outFile.Close()
234 if try < MaxErrorRetries {
235 exponentialBackoffSleep(try)
236 try++
237 return self.downloadRemoteFile(id, fpath, args, try)
238 } else {
239 os.Remove(tmpPath)
240 return fmt.Errorf("Download was interrupted: %s", err)
241 }
242 }
243
244 // Close file
245 outFile.Close()
246
247 // Rename tmp file to proper filename
248 return os.Rename(tmpPath, fpath)
249 }
250
251 func (self *Drive) deleteExtraneousLocalFiles(files *syncFiles, args DownloadSyncArgs) error {
252 extraneousFiles := files.filterExtraneousLocalFiles()
253 extraneousCount := len(extraneousFiles)
254
255 if extraneousCount > 0 {
256 fmt.Fprintf(args.Out, "\n%d local files are extraneous\n", extraneousCount)
257 }
258
259 // Sort files so that the files with the longest path comes first
260 sort.Sort(sort.Reverse(byLocalPathLength(extraneousFiles)))
261
262 for i, lf := range extraneousFiles {
263 fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i+1, extraneousCount, lf.absPath)
264
265 if args.DryRun {
266 continue
267 }
268
269 err := os.Remove(lf.absPath)
270 if err != nil {
271 return fmt.Errorf("Failed to delete local file: %s", err)
272 }
273 }
274
275 return nil
276 }
277
278 func checkLocalConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
279 // No conflict unless local file was last modified
280 if cf.compareModTime() != LocalLastModified {
281 return false, ""
282 }
283
284 // Don't skip if want to keep the remote file
285 if resolution == KeepRemote {
286 return false, ""
287 }
288
289 // Skip if we want to keep the local file
290 if resolution == KeepLocal {
291 return true, "conflicting file, keeping local file"
292 }
293
294 if resolution == KeepLargest {
295 largest := cf.compareSize()
296
297 // Skip if the local file is largest
298 if largest == LocalLargestSize {
299 return true, "conflicting file, local file is largest, keeping local"
300 }
301
302 // Don't skip if the remote file is largest
303 if largest == RemoteLargestSize {
304 return false, ""
305 }
306
307 // Keep local if both files have the same size
308 if largest == EqualSize {
309 return true, "conflicting file, file sizes are equal, keeping local"
310 }
311 }
312
313 // The conditionals above should cover all cases,
314 // unless the programmer did something wrong,
315 // in which case we default to being non-destructive and skip the file
316 return true, "conflicting file, unhandled case"
317 }
318
319 func ensureNoLocalModifications(files []*changedFile) error {
320 conflicts := findLocalConflicts(files)
321 if len(conflicts) == 0 {
322 return nil
323 }
324
325 buffer := bytes.NewBufferString("")
326 formatConflicts(conflicts, buffer)
327 return fmt.Errorf(buffer.String())
328 }