"Fossies" - the Fresh Open Source Software Archive

Member "pytorch-1.8.2/tools/download_mnist.py" (23 Jul 2021, 2733 Bytes) of package /linux/misc/pytorch-1.8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "download_mnist.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 1.12.1_vs_1.13.0.

    1 import argparse
    2 import gzip
    3 import os
    4 from urllib.error import URLError
    5 from urllib.request import urlretrieve
    6 import sys
    7 
    8 MIRRORS = [
    9     'http://yann.lecun.com/exdb/mnist/',
   10     'https://ossci-datasets.s3.amazonaws.com/mnist/',
   11 ]
   12 
   13 RESOURCES = [
   14     'train-images-idx3-ubyte.gz',
   15     'train-labels-idx1-ubyte.gz',
   16     't10k-images-idx3-ubyte.gz',
   17     't10k-labels-idx1-ubyte.gz',
   18 ]
   19 
   20 
   21 def report_download_progress(chunk_number, chunk_size, file_size):
   22     if file_size != -1:
   23         percent = min(1, (chunk_number * chunk_size) / file_size)
   24         bar = '#' * int(64 * percent)
   25         sys.stdout.write('\r0% |{:<64}| {}%'.format(bar, int(percent * 100)))
   26 
   27 
   28 def download(destination_path, resource, quiet):
   29     if os.path.exists(destination_path):
   30         if not quiet:
   31             print('{} already exists, skipping ...'.format(destination_path))
   32     else:
   33         for mirror in MIRRORS:
   34             url = mirror + resource
   35             print('Downloading {} ...'.format(url))
   36             try:
   37                 hook = None if quiet else report_download_progress
   38                 urlretrieve(url, destination_path, reporthook=hook)
   39             except URLError as e:
   40                 print('Failed to download (trying next):\n{}'.format(e))
   41                 continue
   42             finally:
   43                 if not quiet:
   44                     # Just a newline.
   45                     print()
   46             break
   47         else:
   48             raise RuntimeError('Error downloading resource!')
   49 
   50 
   51 def unzip(zipped_path, quiet):
   52     unzipped_path = os.path.splitext(zipped_path)[0]
   53     if os.path.exists(unzipped_path):
   54         if not quiet:
   55             print('{} already exists, skipping ... '.format(unzipped_path))
   56         return
   57     with gzip.open(zipped_path, 'rb') as zipped_file:
   58         with open(unzipped_path, 'wb') as unzipped_file:
   59             unzipped_file.write(zipped_file.read())
   60             if not quiet:
   61                 print('Unzipped {} ...'.format(zipped_path))
   62 
   63 
   64 def main():
   65     parser = argparse.ArgumentParser(
   66         description='Download the MNIST dataset from the internet')
   67     parser.add_argument(
   68         '-d', '--destination', default='.', help='Destination directory')
   69     parser.add_argument(
   70         '-q',
   71         '--quiet',
   72         action='store_true',
   73         help="Don't report about progress")
   74     options = parser.parse_args()
   75 
   76     if not os.path.exists(options.destination):
   77         os.makedirs(options.destination)
   78 
   79     try:
   80         for resource in RESOURCES:
   81             path = os.path.join(options.destination, resource)
   82             download(path, resource, options.quiet)
   83             unzip(path, options.quiet)
   84     except KeyboardInterrupt:
   85         print('Interrupted')
   86 
   87 
   88 if __name__ == '__main__':
   89     main()