"Fossies" - the Fresh Open Source Software Archive

Member "buku-4.4/buku" (15 Jun 2020, 176856 Bytes) of package /linux/privat/buku-4.4.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "buku": 4.3_vs_4.4.

    1 #!/usr/bin/env python3
    2 #
    3 # Bookmark management utility
    4 #
    5 # Copyright © 2015-2020 Arun Prakash Jana <engineerarun@gmail.com>
    6 #
    7 # This program is free software: you can redistribute it and/or modify
    8 # it under the terms of the GNU General Public License as published by
    9 # the Free Software Foundation, either version 3 of the License, or
   10 # (at your option) any later version.
   11 #
   12 # This program is distributed in the hope that it will be useful,
   13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
   14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   15 # GNU General Public License for more details.
   16 #
   17 # You should have received a copy of the GNU General Public License
   18 # along with buku.  If not, see <http://www.gnu.org/licenses/>.
   19 
   20 from enum import Enum
   21 from itertools import chain
   22 import argparse
   23 import calendar
   24 import cgi
   25 import collections
   26 import json
   27 import logging
   28 import os
   29 import platform
   30 import re
   31 import shutil
   32 import signal
   33 import sqlite3
   34 import struct
   35 import subprocess
   36 from subprocess import Popen, PIPE, DEVNULL
   37 import sys
   38 import tempfile
   39 import threading
   40 import time
   41 from typing import Any, Dict, Iterable, List, Optional, Tuple
   42 import webbrowser
   43 import certifi
   44 import urllib3
   45 from urllib3.exceptions import LocationParseError
   46 from urllib3.util import parse_url, make_headers, Retry
   47 from bs4 import BeautifulSoup
   48 # note catch ModuleNotFoundError instead Exception
   49 # when python3.5 not supported
   50 try:
   51     import readline
   52 except Exception:
   53     import pyreadline as readline  # type: ignore
   54 try:
   55     from mypy_extensions import TypedDict
   56 except ImportError:
   57     TypedDict = None  # type: ignore
   58 
   59 __version__ = '4.4'
   60 __author__ = 'Arun Prakash Jana <engineerarun@gmail.com>'
   61 __license__ = 'GPLv3'
   62 
   63 # Global variables
   64 INTERRUPTED = False  # Received SIGINT
   65 DELIM = ','  # Delimiter used to store tags in DB
   66 SKIP_MIMES = {'.pdf', '.txt'}
   67 PROMPTMSG = 'buku (? for help): '  # Prompt message string
   68 
   69 # Default format specifiers to print records
   70 ID_STR = '%d. %s [%s]\n'
   71 ID_DB_STR = '%d. %s'
   72 MUTE_STR = '%s (L)\n'
   73 URL_STR = '   > %s\n'
   74 DESC_STR = '   + %s\n'
   75 TAG_STR = '   # %s\n'
   76 
   77 # Colormap for color output from "googler" project
   78 COLORMAP = {k: '\x1b[%sm' % v for k, v in {
   79     'a': '30', 'b': '31', 'c': '32', 'd': '33',
   80     'e': '34', 'f': '35', 'g': '36', 'h': '37',
   81     'i': '90', 'j': '91', 'k': '92', 'l': '93',
   82     'm': '94', 'n': '95', 'o': '96', 'p': '97',
   83     'A': '30;1', 'B': '31;1', 'C': '32;1', 'D': '33;1',
   84     'E': '34;1', 'F': '35;1', 'G': '36;1', 'H': '37;1',
   85     'I': '90;1', 'J': '91;1', 'K': '92;1', 'L': '93;1',
   86     'M': '94;1', 'N': '95;1', 'O': '96;1', 'P': '97;1',
   87     'x': '0', 'X': '1', 'y': '7', 'Y': '7;1', 'z': '2',
   88 }.items()}
   89 
   90 USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0'
   91 MYHEADERS = None  # Default dictionary of headers
   92 MYPROXY = None  # Default proxy
   93 TEXT_BROWSERS = ['elinks', 'links', 'links2', 'lynx', 'w3m', 'www-browser']
   94 IGNORE_FF_BOOKMARK_FOLDERS = frozenset(["placesRoot", "bookmarksMenuFolder"])
   95 
   96 # Set up logging
   97 LOGGER = logging.getLogger()
   98 LOGDBG = LOGGER.debug
   99 LOGERR = LOGGER.error
  100 
  101 
  102 class BukuCrypt:
  103     """Class to handle encryption and decryption of
  104     the database file. Functionally a separate entity.
  105 
  106     Involves late imports in the static functions but it
  107     saves ~100ms each time. Given that encrypt/decrypt are
  108     not done automatically and any one should be called at
  109     a time, this doesn't seem to be an outrageous approach.
  110     """
  111 
  112     # Crypto constants
  113     BLOCKSIZE = 0x10000  # 64 KB blocks
  114     SALT_SIZE = 0x20
  115     CHUNKSIZE = 0x80000  # Read/write 512 KB chunks
  116 
  117     @staticmethod
  118     def get_filehash(filepath):
  119         """Get the SHA256 hash of a file.
  120 
  121         Parameters
  122         ----------
  123         filepath : str
  124             Path to the file.
  125 
  126         Returns
  127         -------
  128         hash : bytes
  129             Hash digest of file.
  130         """
  131 
  132         from hashlib import sha256
  133 
  134         with open(filepath, 'rb') as fp:
  135             hasher = sha256()
  136             buf = fp.read(BukuCrypt.BLOCKSIZE)
  137             while len(buf) > 0:
  138                 hasher.update(buf)
  139                 buf = fp.read(BukuCrypt.BLOCKSIZE)
  140 
  141             return hasher.digest()
  142 
  143     @staticmethod
  144     def encrypt_file(iterations, dbfile=None):
  145         """Encrypt the bookmarks database file.
  146 
  147         Parameters
  148         ----------
  149         iterations : int
  150             Number of iterations for key generation.
  151         dbfile : str, optional
  152             Custom database file path (including filename).
  153         """
  154 
  155         try:
  156             from cryptography.hazmat.backends import default_backend
  157             from cryptography.hazmat.primitives.ciphers import (Cipher, modes, algorithms)
  158             from getpass import getpass
  159             from hashlib import sha256
  160         except ImportError:
  161             LOGERR('cryptography lib(s) missing')
  162             sys.exit(1)
  163 
  164         if iterations < 1:
  165             LOGERR('Iterations must be >= 1')
  166             sys.exit(1)
  167 
  168         if not dbfile:
  169             dbfile = os.path.join(BukuDb.get_default_dbdir(), 'bookmarks.db')
  170         encfile = dbfile + '.enc'
  171 
  172         db_exists = os.path.exists(dbfile)
  173         enc_exists = os.path.exists(encfile)
  174 
  175         if db_exists and not enc_exists:
  176             pass
  177         elif not db_exists:
  178             LOGERR('%s missing. Already encrypted?', dbfile)
  179             sys.exit(1)
  180         else:
  181             # db_exists and enc_exists
  182             LOGERR('Both encrypted and flat DB files exist!')
  183             sys.exit(1)
  184 
  185         password = getpass()
  186         passconfirm = getpass()
  187         if not password or not passconfirm:
  188             LOGERR('Empty password')
  189             sys.exit(1)
  190         if password != passconfirm:
  191             LOGERR('Passwords do not match')
  192             sys.exit(1)
  193 
  194         try:
  195             # Get SHA256 hash of DB file
  196             dbhash = BukuCrypt.get_filehash(dbfile)
  197         except Exception as e:
  198             LOGERR(e)
  199             sys.exit(1)
  200 
  201         # Generate random 256-bit salt and key
  202         salt = os.urandom(BukuCrypt.SALT_SIZE)
  203         key = ('%s%s' % (password, salt.decode('utf-8', 'replace'))).encode('utf-8')
  204         for _ in range(iterations):
  205             key = sha256(key).digest()
  206 
  207         iv = os.urandom(16)
  208         encryptor = Cipher(
  209             algorithms.AES(key),
  210             modes.CBC(iv),
  211             backend=default_backend()
  212         ).encryptor()
  213         filesize = os.path.getsize(dbfile)
  214 
  215         try:
  216             with open(dbfile, 'rb') as infp, open(encfile, 'wb') as outfp:
  217                 outfp.write(struct.pack('<Q', filesize))
  218                 outfp.write(salt)
  219                 outfp.write(iv)
  220 
  221                 # Embed DB file hash in encrypted file
  222                 outfp.write(dbhash)
  223 
  224                 while True:
  225                     chunk = infp.read(BukuCrypt.CHUNKSIZE)
  226                     if len(chunk) == 0:
  227                         break
  228                     if len(chunk) % 16 != 0:
  229                         chunk = '%s%s' % (chunk, ' ' * (16 - len(chunk) % 16))
  230 
  231                     outfp.write(encryptor.update(chunk) + encryptor.finalize())
  232 
  233             os.remove(dbfile)
  234             print('File encrypted')
  235             sys.exit(0)
  236         except Exception as e:
  237             LOGERR(e)
  238             sys.exit(1)
  239 
  240     @staticmethod
  241     def decrypt_file(iterations, dbfile=None):
  242         """Decrypt the bookmarks database file.
  243 
  244         Parameters
  245         ----------
  246         iterations : int
  247             Number of iterations for key generation.
  248         dbfile : str, optional
  249             Custom database file path (including filename).
  250             The '.enc' suffix must be omitted.
  251         """
  252 
  253         try:
  254             from cryptography.hazmat.backends import default_backend
  255             from cryptography.hazmat.primitives.ciphers import (Cipher, modes, algorithms)
  256             from getpass import getpass
  257             from hashlib import sha256
  258         except ImportError:
  259             LOGERR('cryptography lib(s) missing')
  260             sys.exit(1)
  261 
  262         if iterations < 1:
  263             LOGERR('Decryption failed')
  264             sys.exit(1)
  265 
  266         if not dbfile:
  267             dbfile = os.path.join(BukuDb.get_default_dbdir(), 'bookmarks.db')
  268         else:
  269             dbfile = os.path.abspath(dbfile)
  270             dbpath, filename = os.path.split(dbfile)
  271 
  272         encfile = dbfile + '.enc'
  273 
  274         enc_exists = os.path.exists(encfile)
  275         db_exists = os.path.exists(dbfile)
  276 
  277         if enc_exists and not db_exists:
  278             pass
  279         elif not enc_exists:
  280             LOGERR('%s missing', encfile)
  281             sys.exit(1)
  282         else:
  283             # db_exists and enc_exists
  284             LOGERR('Both encrypted and flat DB files exist!')
  285             sys.exit(1)
  286 
  287         password = getpass()
  288         if not password:
  289             LOGERR('Decryption failed')
  290             sys.exit(1)
  291 
  292         try:
  293             with open(encfile, 'rb') as infp:
  294                 size = struct.unpack('<Q', infp.read(struct.calcsize('Q')))[0]
  295 
  296                 # Read 256-bit salt and generate key
  297                 salt = infp.read(32)
  298                 key = ('%s%s' % (password, salt.decode('utf-8', 'replace'))).encode('utf-8')
  299                 for _ in range(iterations):
  300                     key = sha256(key).digest()
  301 
  302                 iv = infp.read(16)
  303                 decryptor = Cipher(
  304                     algorithms.AES(key),
  305                     modes.CBC(iv),
  306                     backend=default_backend(),
  307                 ).decryptor()
  308 
  309                 # Get original DB file's SHA256 hash from encrypted file
  310                 enchash = infp.read(32)
  311 
  312                 with open(dbfile, 'wb') as outfp:
  313                     while True:
  314                         chunk = infp.read(BukuCrypt.CHUNKSIZE)
  315                         if len(chunk) == 0:
  316                             break
  317 
  318                         outfp.write(decryptor.update(chunk) + decryptor.finalize())
  319 
  320                     outfp.truncate(size)
  321 
  322             # Match hash of generated file with that of original DB file
  323             dbhash = BukuCrypt.get_filehash(dbfile)
  324             if dbhash != enchash:
  325                 os.remove(dbfile)
  326                 LOGERR('Decryption failed')
  327                 sys.exit(1)
  328             else:
  329                 os.remove(encfile)
  330                 print('File decrypted')
  331         except struct.error:
  332             LOGERR('Tainted file')
  333             sys.exit(1)
  334         except Exception as e:
  335             LOGERR(e)
  336             sys.exit(1)
  337 
  338 
  339 BookmarkVar = Tuple[int, str, Optional[str], str, str, int]
  340 
  341 
  342 class BukuDb:
  343     """Abstracts all database operations.
  344 
  345     Attributes
  346     ----------
  347     conn : sqlite database connection.
  348     cur : sqlite database cursor.
  349     json : string
  350         Empty string if results should be printed in JSON format to stdout.
  351         Nonempty string if results should be printed in JSON format to file. The string has to be a valid path.
  352         None if the results should be printed as human-readable plaintext.
  353     field_filter : int
  354         Indicates format for displaying bookmarks. Default is 0.
  355     chatty : bool
  356         Sets the verbosity of the APIs. Default is False.
  357     """
  358 
  359     def __init__(
  360             self, json: Optional[str] = None, field_filter: Optional[int] = 0, chatty: Optional[bool] = False,
  361             dbfile: Optional[str] = None, colorize: Optional[bool] = True) -> None:
  362         """Database initialization API.
  363 
  364         Parameters
  365         ----------
  366         json : string
  367             Empty string if results should be printed in JSON format to stdout.
  368             Nonempty string if results should be printed in JSON format to file. The string has to be a valid path.
  369             None if the results should be printed as human-readable plaintext.
  370         field_filter : int, optional
  371             Indicates format for displaying bookmarks. Default is 0.
  372         chatty : bool, optional
  373             Sets the verbosity of the APIs. Default is False.
  374         colorize : bool, optional
  375             Indicates whether color should be used in output. Default is True.
  376         """
  377 
  378         self.json = json
  379         self.field_filter = field_filter
  380         self.chatty = chatty
  381         self.colorize = colorize
  382         self.conn, self.cur = BukuDb.initdb(dbfile, self.chatty)
  383 
  384     @staticmethod
  385     def get_default_dbdir():
  386         """Determine the directory path where dbfile will be stored.
  387 
  388         If the platform is Windows, use %APPDATA%
  389         else if $XDG_DATA_HOME is defined, use it
  390         else if $HOME exists, use it
  391         else use the current directory.
  392 
  393         Returns
  394         -------
  395         str
  396             Path to database file.
  397         """
  398 
  399         data_home = os.environ.get('XDG_DATA_HOME')
  400         if data_home is None:
  401             if os.environ.get('HOME') is None:
  402                 if sys.platform == 'win32':
  403                     data_home = os.environ.get('APPDATA')
  404                     if data_home is None:
  405                         return os.path.abspath('.')
  406                 else:
  407                     return os.path.abspath('.')
  408             else:
  409                 data_home = os.path.join(os.environ.get('HOME'), '.local', 'share')
  410 
  411         return os.path.join(data_home, 'buku')
  412 
  413     @staticmethod
  414     def initdb(dbfile: Optional[str] = None, chatty: Optional[bool] = False) -> Tuple[sqlite3.Connection, sqlite3.Cursor]:
  415         """Initialize the database connection.
  416 
  417         Create DB file and/or bookmarks table if they don't exist.
  418         Alert on encryption options on first execution.
  419 
  420         Parameters
  421         ----------
  422         dbfile : str, optional
  423             Custom database file path (including filename).
  424         chatty : bool
  425             If True, shows informative message on DB creation.
  426 
  427         Returns
  428         -------
  429         tuple
  430             (connection, cursor).
  431         """
  432 
  433         if not dbfile:
  434             dbpath = BukuDb.get_default_dbdir()
  435             filename = 'bookmarks.db'
  436             dbfile = os.path.join(dbpath, filename)
  437         else:
  438             dbfile = os.path.abspath(dbfile)
  439             dbpath, filename = os.path.split(dbfile)
  440 
  441         try:
  442             if not os.path.exists(dbpath):
  443                 os.makedirs(dbpath)
  444         except Exception as e:
  445             LOGERR(e)
  446             os._exit(1)
  447 
  448         db_exists = os.path.exists(dbfile)
  449         enc_exists = os.path.exists(dbfile + '.enc')
  450 
  451         if db_exists and not enc_exists:
  452             pass
  453         elif enc_exists and not db_exists:
  454             LOGERR('Unlock database first')
  455             sys.exit(1)
  456         elif db_exists and enc_exists:
  457             LOGERR('Both encrypted and flat DB files exist!')
  458             sys.exit(1)
  459         elif chatty:
  460             # not db_exists and not enc_exists
  461             print('DB file is being created at %s.\nYou should encrypt it.' % dbfile)
  462 
  463         try:
  464             # Create a connection
  465             conn = sqlite3.connect(dbfile, check_same_thread=False)
  466             conn.create_function('REGEXP', 2, regexp)
  467             cur = conn.cursor()
  468 
  469             # Create table if it doesn't exist
  470             # flags: designed to be extended in future using bitwise masks
  471             # Masks:
  472             #     0b00000001: set title immutable
  473             cur.execute('CREATE TABLE if not exists bookmarks ('
  474                         'id integer PRIMARY KEY, '
  475                         'URL text NOT NULL UNIQUE, '
  476                         'metadata text default \'\', '
  477                         'tags text default \',\', '
  478                         'desc text default \'\', '
  479                         'flags integer default 0)')
  480             conn.commit()
  481         except Exception as e:
  482             LOGERR('initdb(): %s', e)
  483             sys.exit(1)
  484 
  485         return (conn, cur)
  486 
  487     def get_rec_all(self):
  488         """Get all the bookmarks in the database.
  489 
  490         Returns
  491         -------
  492         list
  493             A list of tuples representing bookmark records.
  494         """
  495 
  496         self.cur.execute('SELECT * FROM bookmarks')
  497         return self.cur.fetchall()
  498 
  499     def get_rec_by_id(self, index: int) -> Optional[BookmarkVar]:
  500         """Get a bookmark from database by its ID.
  501 
  502         Parameters
  503         ----------
  504         index : int
  505             DB index of bookmark record.
  506 
  507         Returns
  508         -------
  509         tuple or None
  510             Bookmark data, or None if index is not found.
  511         """
  512 
  513         self.cur.execute('SELECT * FROM bookmarks WHERE id = ? LIMIT 1', (index,))
  514         resultset = self.cur.fetchall()
  515         return resultset[0] if resultset else None
  516 
  517     def get_rec_id(self, url):
  518         """Check if URL already exists in DB.
  519 
  520         Parameters
  521         ----------
  522         url : str
  523             A URL to search for in the DB.
  524 
  525         Returns
  526         -------
  527         int
  528             DB index, or -1 if URL not found in DB.
  529         """
  530 
  531         self.cur.execute('SELECT id FROM bookmarks WHERE URL = ? LIMIT 1', (url,))
  532         resultset = self.cur.fetchall()
  533         return resultset[0][0] if resultset else -1
  534 
  535     def get_max_id(self) -> int:
  536         """Fetch the ID of the last record.
  537 
  538         Returns
  539         -------
  540         int
  541             ID of the record if any record exists, else -1.
  542         """
  543 
  544         self.cur.execute('SELECT MAX(id) from bookmarks')
  545         resultset = self.cur.fetchall()
  546         return -1 if resultset[0][0] is None else resultset[0][0]
  547 
  548     def add_rec(
  549             self,
  550             url: str,
  551             title_in: Optional[str] = None,
  552             tags_in: Optional[str] = None,
  553             desc: Optional[str] = None,
  554             immutable: Optional[int] = 0,
  555             delay_commit: Optional[bool] = False,
  556             fetch: Optional[bool] = True) -> int:
  557         """Add a new bookmark.
  558 
  559         Parameters
  560         ----------
  561         url : str
  562             URL to bookmark.
  563         title_in :str, optional
  564             Title to add manually. Default is None.
  565         tags_in : str, optional
  566             Comma-separated tags to add manually.
  567             Must start and end with comma. Default is None.
  568         desc : str, optional
  569             Description of the bookmark. Default is None.
  570         immutable : int, optional
  571             Indicates whether to disable title fetch from web.
  572             Default is 0.
  573         delay_commit : bool, optional
  574             True if record should not be committed to the DB,
  575             leaving commit responsibility to caller. Default is False.
  576         fetch : bool, optional
  577             Fetch page from web and parse for data
  578 
  579         Returns
  580         -------
  581         int
  582             DB index of new bookmark on success, -1 on failure.
  583         """
  584 
  585         # Return error for empty URL
  586         if not url or url == '':
  587             LOGERR('Invalid URL')
  588             return -1
  589 
  590         # Ensure that the URL does not exist in DB already
  591         id = self.get_rec_id(url)
  592         if id != -1:
  593             LOGERR('URL [%s] already exists at index %d', url, id)
  594             return -1
  595 
  596         if fetch:
  597             # Fetch data
  598             ptitle, pdesc, ptags, mime, bad = network_handler(url)
  599             if bad:
  600                 print('Malformed URL\n')
  601             elif mime:
  602                 LOGDBG('HTTP HEAD requested')
  603             elif ptitle == '' and title_in is None:
  604                 print('No title\n')
  605             else:
  606                 LOGDBG('Title: [%s]', ptitle)
  607         else:
  608             ptitle = pdesc = ptags = ''
  609             LOGDBG('ptags: [%s]', ptags)
  610 
  611         if title_in is not None:
  612             ptitle = title_in
  613 
  614         # Fix up tags, if broken
  615         tags_in = delim_wrap(tags_in)
  616 
  617         # Process description
  618         if desc is None:
  619             desc = '' if pdesc is None else pdesc
  620 
  621         try:
  622             flagset = 0
  623             if immutable == 1:
  624                 flagset |= immutable
  625 
  626             qry = 'INSERT INTO bookmarks(URL, metadata, tags, desc, flags) VALUES (?, ?, ?, ?, ?)'
  627             self.cur.execute(qry, (url, ptitle, tags_in, desc, flagset))
  628             if not delay_commit:
  629                 self.conn.commit()
  630             if self.chatty:
  631                 self.print_rec(self.cur.lastrowid)
  632             return self.cur.lastrowid
  633         except Exception as e:
  634             LOGERR('add_rec(): %s', e)
  635             return -1
  636 
  637     def append_tag_at_index(self, index, tags_in, delay_commit=False):
  638         """Append tags to bookmark tagset at index.
  639 
  640         Parameters
  641         ----------
  642         index : int
  643             DB index of the record. 0 indicates all records.
  644         tags_in : str
  645             Comma-separated tags to add manually.
  646         delay_commit : bool, optional
  647             True if record should not be committed to the DB,
  648             leaving commit responsibility to caller. Default is False.
  649 
  650         Returns
  651         -------
  652         bool
  653             True on success, False on failure.
  654         """
  655 
  656         if tags_in is None or tags_in == DELIM:
  657             return True
  658 
  659         if index == 0:
  660             resp = read_in('Append the tags to ALL bookmarks? (y/n): ')
  661             if resp != 'y':
  662                 return False
  663 
  664             self.cur.execute('SELECT id, tags FROM bookmarks ORDER BY id ASC')
  665         else:
  666             self.cur.execute('SELECT id, tags FROM bookmarks WHERE id = ? LIMIT 1', (index,))
  667 
  668         resultset = self.cur.fetchall()
  669         if resultset:
  670             query = 'UPDATE bookmarks SET tags = ? WHERE id = ?'
  671             for row in resultset:
  672                 tags = row[1] + tags_in[1:]
  673                 tags = parse_tags([tags])
  674                 self.cur.execute(query, (tags, row[0],))
  675                 if self.chatty and not delay_commit:
  676                     self.print_rec(row[0])
  677         else:
  678             return False
  679 
  680         if not delay_commit:
  681             self.conn.commit()
  682 
  683         return True
  684 
  685     def delete_tag_at_index(self, index, tags_in, delay_commit=False, chatty=True):
  686         """Delete tags from bookmark tagset at index.
  687 
  688         Parameters
  689         ----------
  690         index : int
  691             DB index of bookmark record. 0 indicates all records.
  692         tags_in : str
  693             Comma-separated tags to delete manually.
  694         delay_commit : bool, optional
  695             True if record should not be committed to the DB,
  696             leaving commit responsibility to caller. Default is False.
  697         chatty: bool, optional
  698             Skip confirmation when set to False.
  699 
  700         Returns
  701         -------
  702         bool
  703             True on success, False on failure.
  704         """
  705 
  706         if tags_in is None or tags_in == DELIM:
  707             return True
  708 
  709         tags_to_delete = tags_in.strip(DELIM).split(DELIM)
  710 
  711         if index == 0:
  712             if chatty:
  713                 resp = read_in('Delete the tag(s) from ALL bookmarks? (y/n): ')
  714                 if resp != 'y':
  715                     return False
  716 
  717             count = 0
  718             match = "'%' || ? || '%'"
  719             for tag in tags_to_delete:
  720                 tag = delim_wrap(tag)
  721                 q = ("UPDATE bookmarks SET tags = replace(tags, '%s', '%s') "
  722                      "WHERE tags LIKE %s" % (tag, DELIM, match))
  723                 self.cur.execute(q, (tag,))
  724                 count += self.cur.rowcount
  725 
  726             if count and not delay_commit:
  727                 self.conn.commit()
  728                 if self.chatty:
  729                     print('%d record(s) updated' % count)
  730 
  731             return True
  732 
  733         # Process a single index
  734         # Use SELECT and UPDATE to handle multiple tags at once
  735         query = 'SELECT id, tags FROM bookmarks WHERE id = ? LIMIT 1'
  736         self.cur.execute(query, (index,))
  737         resultset = self.cur.fetchall()
  738         if resultset:
  739             query = 'UPDATE bookmarks SET tags = ? WHERE id = ?'
  740             for row in resultset:
  741                 tags = row[1]
  742 
  743                 for tag in tags_to_delete:
  744                     tags = tags.replace(delim_wrap(tag), DELIM)
  745 
  746                 self.cur.execute(query, (parse_tags([tags]), row[0],))
  747                 if self.chatty and not delay_commit:
  748                     self.print_rec(row[0])
  749 
  750                 if not delay_commit:
  751                     self.conn.commit()
  752         else:
  753             return False
  754 
  755         return True
  756 
  757     def update_rec(
  758             self,
  759             index: int,
  760             url: Optional[str] = None,
  761             title_in: Optional[str] = None,
  762             tags_in: Optional[str] = None,
  763             desc: Optional[str] = None,
  764             immutable: Optional[int] = -1,
  765             threads: int = 4) -> bool:
  766         """Update an existing record at index.
  767 
  768         Update all records if index is 0 and url is not specified.
  769         URL is an exception because URLs are unique in DB.
  770 
  771         Parameters
  772         ----------
  773         index : int
  774             DB index of record. 0 indicates all records.
  775         url : str, optional
  776             Bookmark address.
  777         title_in : str, optional
  778             Title to add manually.
  779         tags_in : str, optional
  780             Comma-separated tags to add manually. Must start and end with comma.
  781             Prefix with '+,' to append to current tags.
  782             Prefix with '-,' to delete from current tags.
  783         desc : str, optional
  784             Description of bookmark.
  785         immutable : int, optional
  786             Disable title fetch from web if 1. Default is -1.
  787         threads : int, optional
  788             Number of threads to use to refresh full DB. Default is 4.
  789 
  790         Returns
  791         -------
  792         bool
  793             True on success, False on Failure.
  794         """
  795 
  796         arguments = []  # type: List[Any]
  797         query = 'UPDATE bookmarks SET'
  798         to_update = False
  799         tag_modified = False
  800         ret = False
  801 
  802         # Update URL if passed as argument
  803         if url is not None and url != '':
  804             if index == 0:
  805                 LOGERR('All URLs cannot be same')
  806                 return False
  807             query += ' URL = ?,'
  808             arguments += (url,)
  809             to_update = True
  810 
  811         # Update tags if passed as argument
  812         if tags_in is not None:
  813             if tags_in in ('+,', '-,'):
  814                 LOGERR('Please specify a tag')
  815                 return False
  816 
  817             if tags_in.startswith('+,'):
  818                 chatty = self.chatty
  819                 self.chatty = False
  820                 ret = self.append_tag_at_index(index, tags_in[1:])
  821                 self.chatty = chatty
  822                 tag_modified = True
  823             elif tags_in.startswith('-,'):
  824                 chatty = self.chatty
  825                 self.chatty = False
  826                 ret = self.delete_tag_at_index(index, tags_in[1:])
  827                 self.chatty = chatty
  828                 tag_modified = True
  829             else:
  830                 tags_in = delim_wrap(tags_in)
  831 
  832                 query += ' tags = ?,'
  833                 arguments += (tags_in,)
  834                 to_update = True
  835 
  836         # Update description if passed as an argument
  837         if desc is not None:
  838             query += ' desc = ?,'
  839             arguments += (desc,)
  840             to_update = True
  841 
  842         # Update immutable flag if passed as argument
  843         if immutable != -1:
  844             flagset = 1
  845             if immutable == 1:
  846                 query += ' flags = flags | ?,'
  847             elif immutable == 0:
  848                 query += ' flags = flags & ?,'
  849                 flagset = ~flagset
  850 
  851             arguments += (flagset,)
  852             to_update = True
  853 
  854         # Update title
  855         #
  856         # 1. If --title has no arguments, delete existing title
  857         # 2. If --title has arguments, update existing title
  858         # 3. If --title option is omitted at cmdline:
  859         #    If URL is passed, update the title from web using the URL
  860         # 4. If no other argument (url, tag, comment, immutable) passed,
  861         #    update title from web using DB URL (if title is mutable)
  862         title_to_insert = None
  863         pdesc = None
  864         ptags = None
  865         if title_in is not None:
  866             title_to_insert = title_in
  867         elif url is not None and url != '':
  868             title_to_insert, pdesc, ptags, mime, bad = network_handler(url)
  869             if bad:
  870                 print('Malformed URL')
  871             elif mime:
  872                 LOGDBG('HTTP HEAD requested')
  873             elif title_to_insert == '':
  874                 print('No title')
  875             else:
  876                 LOGDBG('Title: [%s]', title_to_insert)
  877 
  878             if not desc:
  879                 if not pdesc:
  880                     pdesc = ''
  881                 query += ' desc = ?,'
  882                 arguments += (pdesc,)
  883                 to_update = True
  884         elif not to_update and not tag_modified:
  885             ret = self.refreshdb(index, threads)
  886             if ret and index and self.chatty:
  887                 self.print_rec(index)
  888             return ret
  889 
  890         if title_to_insert is not None:
  891             query += ' metadata = ?,'
  892             arguments += (title_to_insert,)
  893             to_update = True
  894 
  895         if not to_update:  # Nothing to update
  896             # Show bookmark if tags were appended to deleted
  897             if tag_modified and self.chatty:
  898                 self.print_rec(index)
  899             return ret
  900 
  901         if index == 0:  # Update all records
  902             resp = read_in('Update ALL bookmarks? (y/n): ')
  903             if resp != 'y':
  904                 return False
  905 
  906             query = query[:-1]
  907         else:
  908             query = query[:-1] + ' WHERE id = ?'
  909             arguments += (index,)
  910 
  911         LOGDBG('update_rec query: "%s", args: %s', query, arguments)
  912 
  913         try:
  914             self.cur.execute(query, arguments)
  915             self.conn.commit()
  916             if self.cur.rowcount and self.chatty:
  917                 self.print_rec(index)
  918 
  919             if self.cur.rowcount == 0:
  920                 LOGERR('No matching index %d', index)
  921                 return False
  922         except sqlite3.IntegrityError:
  923             LOGERR('URL already exists')
  924             return False
  925         except sqlite3.OperationalError as e:
  926             LOGERR(e)
  927             return False
  928 
  929         return True
  930 
  931     def refreshdb(self, index: int, threads: int) -> bool:
  932         """Refresh ALL records in the database.
  933 
  934         Fetch title for each bookmark from the web and update the records.
  935         Doesn't update the record if fetched title is empty.
  936 
  937         Notes
  938         -----
  939             This API doesn't change DB index, URL or tags of a bookmark.
  940             This API is verbose.
  941 
  942         Parameters
  943         ----------
  944         index : int
  945             DB index of record to update. 0 indicates all records.
  946         threads: int
  947             Number of threads to use to refresh full DB. Default is 4.
  948         """
  949 
  950         if index == 0:
  951             self.cur.execute('SELECT id, url, flags FROM bookmarks ORDER BY id ASC')
  952         else:
  953             self.cur.execute('SELECT id, url, flags FROM bookmarks WHERE id = ? LIMIT 1', (index,))
  954 
  955         resultset = self.cur.fetchall()
  956         recs = len(resultset)
  957         if not recs:
  958             LOGERR('No matching index or title immutable or empty DB')
  959             return False
  960 
  961         # Set up strings to be printed
  962         if self.colorize:
  963             bad_url_str = '\x1b[1mIndex %d: Malformed URL\x1b[0m\n'
  964             mime_str = '\x1b[1mIndex %d: HTTP HEAD requested\x1b[0m\n'
  965             blank_url_str = '\x1b[1mIndex %d: No title\x1b[0m\n'
  966             success_str = 'Title: [%s]\n\x1b[92mIndex %d: updated\x1b[0m\n'
  967         else:
  968             bad_url_str = 'Index %d: Malformed URL\n'
  969             mime_str = 'Index %d: HTTP HEAD requested\n'
  970             blank_url_str = 'Index %d: No title\n'
  971             success_str = 'Title: [%s]\nIndex %d: updated\n'
  972 
  973         done = {'value': 0}  # count threads completed
  974         processed = {'value': 0}  # count number of records processed
  975 
  976         # An additional call to generate default headers
  977         # gen_headers() is called within network_handler()
  978         # However, this initial call to setup headers
  979         # ensures there is no race condition among the
  980         # initial threads to setup headers
  981         if not MYHEADERS:
  982             gen_headers()
  983 
  984         cond = threading.Condition()
  985         cond.acquire()
  986 
  987         def refresh(count, cond):
  988             """Inner function to fetch titles and update records.
  989 
  990             Parameters
  991             ----------
  992             count : int
  993                 Dummy input to adhere to convention.
  994             cond : threading condition object.
  995             """
  996 
  997             count = 0
  998 
  999             while True:
 1000                 query = 'UPDATE bookmarks SET'
 1001                 arguments = []
 1002 
 1003                 cond.acquire()
 1004                 if resultset:
 1005                     row = resultset.pop()
 1006                 else:
 1007                     cond.release()
 1008                     break
 1009                 cond.release()
 1010 
 1011                 title, desc, tags, mime, bad = network_handler(row[1], row[2] & 1)
 1012                 count += 1
 1013 
 1014                 cond.acquire()
 1015 
 1016                 if bad:
 1017                     print(bad_url_str % row[0])
 1018                     cond.release()
 1019                     continue
 1020 
 1021                 if mime:
 1022                     if self.chatty:
 1023                         print(mime_str % row[0])
 1024                     cond.release()
 1025                     continue
 1026 
 1027                 to_update = False
 1028 
 1029                 if not title or title == '':
 1030                     LOGERR(blank_url_str, row[0])
 1031                 else:
 1032                     query += ' metadata = ?,'
 1033                     arguments += (title,)
 1034                     to_update = True
 1035 
 1036                 if desc:
 1037                     query += ' desc = ?,'
 1038                     arguments += (desc,)
 1039                     to_update = True
 1040 
 1041                 if not to_update:
 1042                     cond.release()
 1043                     continue
 1044 
 1045                 query = query[:-1] + ' WHERE id = ?'
 1046                 arguments += (row[0],)
 1047                 LOGDBG('refreshdb query: "%s", args: %s', query, arguments)
 1048 
 1049                 self.cur.execute(query, arguments)
 1050 
 1051                 # Save after fetching 32 titles per thread
 1052                 if count & 0b11111 == 0:
 1053                     self.conn.commit()
 1054 
 1055                 if self.chatty:
 1056                     print(success_str % (title, row[0]))
 1057                 cond.release()
 1058 
 1059                 if INTERRUPTED:
 1060                     break
 1061 
 1062             LOGDBG('Thread %d: processed %d', threading.get_ident(), count)
 1063             with cond:
 1064                 done['value'] += 1
 1065                 processed['value'] += count
 1066                 cond.notify()
 1067 
 1068         if recs < threads:
 1069             threads = recs
 1070 
 1071         for i in range(threads):
 1072             thread = threading.Thread(target=refresh, args=(i, cond))
 1073             thread.start()
 1074 
 1075         while done['value'] < threads:
 1076             cond.wait()
 1077             LOGDBG('%d threads completed', done['value'])
 1078 
 1079         # Guard: records found == total records processed
 1080         if recs != processed['value']:
 1081             LOGERR('Records: %d, processed: %d !!!', recs, processed['value'])
 1082 
 1083         cond.release()
 1084         self.conn.commit()
 1085         return True
 1086 
 1087     def edit_update_rec(self, index, immutable=-1):
 1088         """Edit in editor and update a record.
 1089 
 1090         Parameters
 1091         ----------
 1092         index : int
 1093             DB index of the record.
 1094             Last record, if index is -1.
 1095         immutable : int, optional
 1096             Diable title fetch from web if 1. Default is -1.
 1097 
 1098         Returns
 1099         -------
 1100         bool
 1101             True if updated, else False.
 1102         """
 1103 
 1104         editor = get_system_editor()
 1105         if editor == 'none':
 1106             LOGERR('EDITOR must be set to use index with -w')
 1107             return False
 1108 
 1109         if index == -1:
 1110             # Edit the last records
 1111             index = self.get_max_id()
 1112             if index == -1:
 1113                 LOGERR('Empty database')
 1114                 return False
 1115 
 1116         rec = self.get_rec_by_id(index)
 1117         if not rec:
 1118             LOGERR('No matching index %d', index)
 1119             return False
 1120 
 1121         # If reading from DB, show empty title and desc as empty lines. We have to convert because
 1122         # even in case of add with a blank title or desc, '' is used as initializer to show '-'.
 1123         result = edit_rec(editor, rec[1], rec[2] if rec[2] != '' else None,
 1124                           rec[3], rec[4] if rec[4] != '' else None)
 1125         if result is not None:
 1126             url, title, tags, desc = result
 1127             return self.update_rec(index, url, title, tags, desc, immutable)
 1128 
 1129         if immutable != -1:
 1130             return self.update_rec(index, immutable)
 1131 
 1132         return False
 1133 
 1134     def list_using_id(self, ids=[]):
 1135         """List entries in the DB using the specified id list.
 1136 
 1137         Parameters
 1138         ----------
 1139         ids : list of ids in string form
 1140 
 1141         Returns
 1142         -------
 1143         list
 1144         """
 1145         q0 = 'SELECT * FROM bookmarks'
 1146         if ids:
 1147             q0 += ' WHERE id in ('
 1148             for idx in ids:
 1149                 if '-' in idx:
 1150                     val = idx.split('-')
 1151                     if val[0]:
 1152                         part_ids = list(map(int, val))
 1153                         part_ids[1] += 1
 1154                         part_ids = list(range(*part_ids))
 1155                     else:
 1156                         end = int(val[1])
 1157                         qtemp = 'SELECT id FROM bookmarks ORDER BY id DESC limit {0}'.format(end)
 1158                         self.cur.execute(qtemp, [])
 1159                         part_ids = list(chain.from_iterable(self.cur.fetchall()))
 1160                     q0 += ','.join(list(map(str, part_ids)))
 1161                 else:
 1162                     q0 += idx + ','
 1163             q0 = q0.rstrip(',')
 1164             q0 += ')'
 1165 
 1166         try:
 1167             self.cur.execute(q0, [])
 1168         except sqlite3.OperationalError as e:
 1169             LOGERR(e)
 1170             return None
 1171         return self.cur.fetchall()
 1172 
 1173     def searchdb(
 1174             self,
 1175             keywords: List[str],
 1176             all_keywords: Optional[bool] = False,
 1177             deep: Optional[bool] = False,
 1178             regex: Optional[bool] = False
 1179     ) -> Optional[Iterable[Any]]:
 1180         """Search DB for entries where tags, URL, or title fields match keywords.
 1181 
 1182         Parameters
 1183         ----------
 1184         keywords : list of str
 1185             Keywords to search.
 1186         all_keywords : bool, optional
 1187             True to return records matching ALL keywords.
 1188             False (default value) to return records matching ANY keyword.
 1189         deep : bool, optional
 1190             True to search for matching substrings. Default is False.
 1191         regex : bool, optional
 1192             Match a regular expression if True. Default is False.
 1193 
 1194         Returns
 1195         -------
 1196         list or None
 1197             List of search results, or None if no matches.
 1198         """
 1199         if not keywords:
 1200             return None
 1201 
 1202         # Deep query string
 1203         q1 = ("(tags LIKE ('%' || ? || '%') OR "
 1204               "URL LIKE ('%' || ? || '%') OR "
 1205               "metadata LIKE ('%' || ? || '%') OR "
 1206               "desc LIKE ('%' || ? || '%')) ")
 1207         # Non-deep query string
 1208         q2 = ('(tags REGEXP ? OR '
 1209               'URL REGEXP ? OR '
 1210               'metadata REGEXP ? OR '
 1211               'desc REGEXP ?) ')
 1212         qargs = []  # type: List[Any]
 1213 
 1214         case_statement = lambda x: 'CASE WHEN ' + x + ' THEN 1 ELSE 0 END'
 1215         if regex:
 1216             q0 = 'SELECT id, url, metadata, tags, desc, flags FROM (SELECT *, '
 1217             for token in keywords:
 1218                 if not token:
 1219                     continue
 1220 
 1221                 q0 += case_statement(q2) + ' + '
 1222                 qargs += (token, token, token, token,)
 1223 
 1224             if not qargs:
 1225                 return None
 1226 
 1227             q0 = q0[:-3] + ' AS score FROM bookmarks WHERE score > 0 ORDER BY score DESC)'
 1228         elif all_keywords:
 1229             if len(keywords) == 1 and keywords[0] == 'blank':
 1230                 q0 = "SELECT * FROM bookmarks WHERE metadata = '' OR tags = ? "
 1231                 qargs += (DELIM,)
 1232             elif len(keywords) == 1 and keywords[0] == 'immutable':
 1233                 q0 = 'SELECT * FROM bookmarks WHERE flags & 1 == 1 '
 1234             else:
 1235                 q0 = 'SELECT id, url, metadata, tags, desc, flags FROM bookmarks WHERE '
 1236                 for token in keywords:
 1237                     if not token:
 1238                         continue
 1239 
 1240                     if deep:
 1241                         q0 += q1 + 'AND '
 1242                     else:
 1243                         _pre = _post = ''
 1244                         if str.isalnum(token[0]):
 1245                             _pre = '\\b'
 1246                         if str.isalnum(token[-1]):
 1247                             _post = '\\b'
 1248                         token = _pre + re.escape(token.rstrip('/')) + _post
 1249                         q0 += q2 + 'AND '
 1250 
 1251                     qargs += (token, token, token, token,)
 1252 
 1253                 if not qargs:
 1254                     return None
 1255 
 1256                 q0 = q0[:-4]
 1257             q0 += 'ORDER BY id ASC'
 1258         elif not all_keywords:
 1259             q0 = 'SELECT id, url, metadata, tags, desc, flags FROM (SELECT *, '
 1260             for token in keywords:
 1261                 if not token:
 1262                     continue
 1263 
 1264                 if deep:
 1265                     q0 += case_statement(q1) + ' + '
 1266                 else:
 1267                     _pre = _post = ''
 1268                     if str.isalnum(token[0]):
 1269                         _pre = '\\b'
 1270                     if str.isalnum(token[-1]):
 1271                         _post = '\\b'
 1272                     token = _pre + re.escape(token.rstrip('/')) + _post
 1273                     q0 += case_statement(q2) + ' + '
 1274 
 1275                 qargs += (token, token, token, token,)
 1276 
 1277             if not qargs:
 1278                 return None
 1279 
 1280             q0 = q0[:-3] + ' AS score FROM bookmarks WHERE score > 0 ORDER BY score DESC)'
 1281         else:
 1282             LOGERR('Invalid search option')
 1283             return None
 1284 
 1285         LOGDBG('query: "%s", args: %s', q0, qargs)
 1286 
 1287         try:
 1288             self.cur.execute(q0, qargs)
 1289         except sqlite3.OperationalError as e:
 1290             LOGERR(e)
 1291             return None
 1292 
 1293         return self.cur.fetchall()
 1294 
 1295     def search_by_tag(self, tags: Optional[str]) -> Optional[List[BookmarkVar]]:
 1296         """Search bookmarks for entries with given tags.
 1297 
 1298         Parameters
 1299         ----------
 1300         tags : str
 1301             String of tags to search for.
 1302             Retrieves entries matching ANY tag if tags are
 1303             delimited with ','.
 1304             Retrieves entries matching ALL tags if tags are
 1305             delimited with '+'.
 1306 
 1307         Returns
 1308         -------
 1309         list or None
 1310             List of search results, or None if no matches.
 1311         """
 1312 
 1313         LOGDBG(tags)
 1314         if tags is None or tags == DELIM or tags == '':
 1315             return None
 1316 
 1317         tags_, search_operator, excluded_tags = prep_tag_search(tags)
 1318         if search_operator is None:
 1319             LOGERR("Cannot use both '+' and ',' in same search")
 1320             return None
 1321 
 1322         LOGDBG('tags: %s', tags_)
 1323         LOGDBG('search_operator: %s', search_operator)
 1324         LOGDBG('excluded_tags: %s', excluded_tags)
 1325 
 1326         if search_operator == 'AND':
 1327             query = ("SELECT id, url, metadata, tags, desc, flags FROM bookmarks "
 1328                      "WHERE tags LIKE '%' || ? || '%' ")
 1329             for tag in tags_[1:]:
 1330                 query += "{} tags LIKE '%' || ? || '%' ".format(search_operator)
 1331 
 1332             if excluded_tags:
 1333                 tags_.append(excluded_tags)
 1334                 query = query.replace('WHERE tags', 'WHERE (tags')
 1335                 query += ') AND tags NOT REGEXP ? '
 1336             query += 'ORDER BY id ASC'
 1337         else:
 1338             query = 'SELECT id, url, metadata, tags, desc, flags FROM (SELECT *, '
 1339             case_statement = "CASE WHEN tags LIKE '%' || ? || '%' THEN 1 ELSE 0 END"
 1340             query += case_statement
 1341 
 1342             for tag in tags_[1:]:
 1343                 query += ' + ' + case_statement
 1344 
 1345             query += ' AS score FROM bookmarks WHERE score > 0'
 1346 
 1347             if excluded_tags:
 1348                 tags_.append(excluded_tags)
 1349                 query += ' AND tags NOT REGEXP ? '
 1350 
 1351             query += ' ORDER BY score DESC)'
 1352 
 1353         LOGDBG('query: "%s", args: %s', query, tags_)
 1354         self.cur.execute(query, tuple(tags_, ))
 1355         return self.cur.fetchall()
 1356 
 1357     def search_keywords_and_filter_by_tags(
 1358             self,
 1359             keywords: List[str],
 1360             all_keywords: bool,
 1361             deep: bool,
 1362             regex: bool,
 1363             stag: str) -> Optional[List[BookmarkVar]]:
 1364         """Search bookmarks for entries with keywords and specified
 1365         criteria while filtering out entries with matching tags.
 1366 
 1367         Parameters
 1368         ----------
 1369         keywords : list of str
 1370             Keywords to search.
 1371         all_keywords : bool, optional
 1372             True to return records matching ALL keywords.
 1373             False to return records matching ANY keyword.
 1374         deep : bool, optional
 1375             True to search for matching substrings.
 1376         regex : bool, optional
 1377             Match a regular expression if True.
 1378         stag : str
 1379             String of tags to search for.
 1380             Retrieves entries matching ANY tag if tags are
 1381             delimited with ','.
 1382             Retrieves entries matching ALL tags if tags are
 1383             delimited with '+'.
 1384 
 1385         Returns
 1386         -------
 1387         list or None
 1388             List of search results, or None if no matches.
 1389         """
 1390 
 1391         keyword_results = self.searchdb(keywords, all_keywords, deep, regex)
 1392         keyword_results = keyword_results if keyword_results is not None else []
 1393         stag_results = self.search_by_tag(''.join(stag))
 1394         stag_results = stag_results if stag_results is not None else []
 1395         return list(set(keyword_results) & set(stag_results))
 1396 
 1397     def exclude_results_from_search(self, search_results, without, deep):
 1398         """Excludes records that match keyword search using without parameters
 1399 
 1400         Parameters
 1401         ----------
 1402         search_results : list
 1403             List of search results
 1404         without : list of str
 1405             Keywords to search.
 1406         deep : bool, optional
 1407             True to search for matching substrings.
 1408 
 1409         Returns
 1410         -------
 1411         list or None
 1412             List of search results, or None if no matches.
 1413         """
 1414 
 1415         return list(set(search_results) - set(self.searchdb(without, False, deep)))
 1416 
 1417     def compactdb(self, index: int, delay_commit: bool = False):
 1418         """When an entry at index is deleted, move the
 1419         last entry in DB to index, if index is lesser.
 1420 
 1421         Parameters
 1422         ----------
 1423         index : int
 1424             DB index of deleted entry.
 1425         delay_commit : bool, optional
 1426             True if record should not be committed to the DB,
 1427             leaving commit responsibility to caller. Default is False.
 1428         """
 1429 
 1430         # Return if the last index left in DB was just deleted
 1431         max_id = self.get_max_id()
 1432         if max_id == -1:
 1433             return
 1434 
 1435         query1 = 'SELECT id, URL, metadata, tags, desc, flags FROM bookmarks WHERE id = ? LIMIT 1'
 1436         query2 = 'DELETE FROM bookmarks WHERE id = ?'
 1437         query3 = 'INSERT INTO bookmarks(id, URL, metadata, tags, desc, flags) VALUES (?, ?, ?, ?, ?, ?)'
 1438 
 1439         # NOOP if the just deleted index was the last one
 1440         if max_id > index:
 1441             self.cur.execute(query1, (max_id,))
 1442             results = self.cur.fetchall()
 1443             for row in results:
 1444                 self.cur.execute(query2, (row[0],))
 1445                 self.cur.execute(query3, (index, row[1], row[2], row[3], row[4], row[5]))
 1446                 if not delay_commit:
 1447                     self.conn.commit()
 1448                 if self.chatty:
 1449                     print('Index %d moved to %d' % (row[0], index))
 1450 
 1451     def delete_rec(
 1452             self,
 1453             index: int,
 1454             low: int = 0,
 1455             high: int = 0,
 1456             is_range: bool = False,
 1457             delay_commit: bool = False
 1458     ) -> bool:
 1459         """Delete a single record or remove the table if index is None.
 1460 
 1461         Parameters
 1462         ----------
 1463         index : int
 1464             DB index of deleted entry.
 1465         low : int, optional
 1466             Actual lower index of range.
 1467         high : int, optional
 1468             Actual higher index of range.
 1469         is_range : bool, optional
 1470             A range is passed using low and high arguments.
 1471             An index is ignored if is_range is True (use dummy index).
 1472             Default is False.
 1473         delay_commit : bool, optional
 1474             True if record should not be committed to the DB,
 1475             leaving commit responsibility to caller. Default is False.
 1476 
 1477         Returns
 1478         -------
 1479         bool
 1480             True on success, False on failure.
 1481         """
 1482 
 1483         if is_range:  # Delete a range of indices
 1484             if low < 0 or high < 0:
 1485                 LOGERR('Negative range boundary')
 1486                 return False
 1487 
 1488             if low > high:
 1489                 low, high = high, low
 1490 
 1491             # If range starts from 0, delete all records
 1492             if low == 0:
 1493                 return self.cleardb()
 1494 
 1495             try:
 1496                 if self.chatty:
 1497                     self.cur.execute('SELECT COUNT(*) from bookmarks where id '
 1498                                      'BETWEEN ? AND ?', (low, high))
 1499                     count = self.cur.fetchone()
 1500                     if count[0] < 1:
 1501                         print('Index %d-%d: 0 deleted' % (low, high))
 1502                         return False
 1503 
 1504                     if self.print_rec(0, low, high, True) is True:
 1505                         resp = input('Delete these bookmarks? (y/n): ')
 1506                         if resp != 'y':
 1507                             return False
 1508 
 1509                 query = 'DELETE from bookmarks where id BETWEEN ? AND ?'
 1510                 self.cur.execute(query, (low, high))
 1511                 print('Index %d-%d: %d deleted' % (low, high, self.cur.rowcount))
 1512                 if not self.cur.rowcount:
 1513                     return False
 1514 
 1515                 # Compact DB by ascending order of index to ensure
 1516                 # the existing higher indices move only once
 1517                 # Delayed commit is forced
 1518                 for index in range(low, high + 1):
 1519                     self.compactdb(index, delay_commit=True)
 1520 
 1521                 if not delay_commit:
 1522                     self.conn.commit()
 1523             except IndexError:
 1524                 LOGERR('No matching index')
 1525                 return False
 1526         elif index == 0:  # Remove the table
 1527             return self.cleardb()
 1528         else:  # Remove a single entry
 1529             try:
 1530                 if self.chatty:
 1531                     self.cur.execute('SELECT COUNT(*) FROM bookmarks WHERE '
 1532                                      'id = ? LIMIT 1', (index,))
 1533                     count = self.cur.fetchone()
 1534                     if count[0] < 1:
 1535                         LOGERR('No matching index %d', index)
 1536                         return False
 1537 
 1538                     if self.print_rec(index) is True:
 1539                         resp = input('Delete this bookmark? (y/n): ')
 1540                         if resp != 'y':
 1541                             return False
 1542 
 1543                 query = 'DELETE FROM bookmarks WHERE id = ?'
 1544                 self.cur.execute(query, (index,))
 1545                 if self.cur.rowcount == 1:
 1546                     print('Index %d deleted' % index)
 1547                     self.compactdb(index, delay_commit=True)
 1548                     if not delay_commit:
 1549                         self.conn.commit()
 1550                 else:
 1551                     LOGERR('No matching index %d', index)
 1552                     return False
 1553             except IndexError:
 1554                 LOGERR('No matching index %d', index)
 1555                 return False
 1556             except sqlite3.OperationalError as e:
 1557                 LOGERR(e)
 1558                 return False
 1559 
 1560         return True
 1561 
 1562     def delete_resultset(self, results):
 1563         """Delete search results in descending order of DB index.
 1564 
 1565         Indices are expected to be unique and in ascending order.
 1566 
 1567         Notes
 1568         -----
 1569             This API forces a delayed commit.
 1570 
 1571         Parameters
 1572         ----------
 1573         results : list of tuples
 1574             List of results to delete from DB.
 1575 
 1576         Returns
 1577         -------
 1578         bool
 1579             True on success, False on failure.
 1580         """
 1581         resp = read_in('Delete the search results? (y/n): ')
 1582         if resp != 'y':
 1583             return False
 1584 
 1585         # delete records in reverse order
 1586         pos = len(results) - 1
 1587         while pos >= 0:
 1588             idx = results[pos][0]
 1589             self.delete_rec(idx, delay_commit=True)
 1590 
 1591             # Commit at every 200th removal
 1592             if pos % 200 == 0:
 1593                 self.conn.commit()
 1594 
 1595             pos -= 1
 1596 
 1597         return True
 1598 
 1599     def delete_rec_all(self, delay_commit=False):
 1600         """Removes all records in the Bookmarks table.
 1601 
 1602         Parameters
 1603         ----------
 1604         delay_commit : bool, optional
 1605             True if record should not be committed to the DB,
 1606             leaving commit responsibility to caller. Default is False.
 1607 
 1608         Returns
 1609         -------
 1610         bool
 1611             True on success, False on failure.
 1612         """
 1613 
 1614         try:
 1615             self.cur.execute('DELETE FROM bookmarks')
 1616             if not delay_commit:
 1617                 self.conn.commit()
 1618             return True
 1619         except Exception as e:
 1620             LOGERR('delete_rec_all(): %s', e)
 1621             return False
 1622 
 1623     def cleardb(self):
 1624         """Drops the bookmark table if it exists.
 1625 
 1626         Returns
 1627         -------
 1628         bool
 1629             True on success, False on failure.
 1630         """
 1631 
 1632         resp = read_in('Remove ALL bookmarks? (y/n): ')
 1633         if resp != 'y':
 1634             print('No bookmarks deleted')
 1635             return False
 1636 
 1637         if self.delete_rec_all():
 1638             self.cur.execute('VACUUM')
 1639             self.conn.commit()
 1640             print('All bookmarks deleted')
 1641             return True
 1642 
 1643         return False
 1644 
 1645     def print_rec(self, index=0, low=0, high=0, is_range=False):
 1646         """Print bookmark details at index or all bookmarks if index is 0.
 1647 
 1648         A negative index behaves like tail, if title is blank show "Untitled".
 1649 
 1650         Parameters
 1651         -----------
 1652         index : int, optional
 1653             DB index of record to print. 0 prints all records.
 1654         low : int, optional
 1655             Actual lower index of range.
 1656         high : int, optional
 1657             Actual higher index of range.
 1658         is_range : bool, optional
 1659             A range is passed using low and high arguments.
 1660             An index is ignored if is_range is True (use dummy index).
 1661             Default is False.
 1662 
 1663         Returns
 1664         -------
 1665         bool
 1666             True on success, False on failure.
 1667         """
 1668 
 1669         if index < 0:
 1670             # Show the last n records
 1671             _id = self.get_max_id()
 1672             if _id == -1:
 1673                 LOGERR('Empty database')
 1674                 return False
 1675 
 1676             low = (1 if _id <= -index else _id + index + 1)
 1677             high = _id
 1678             is_range = True
 1679 
 1680         if is_range:
 1681             if low < 0 or high < 0:
 1682                 LOGERR('Negative range boundary')
 1683                 return False
 1684 
 1685             if low > high:
 1686                 low, high = high, low
 1687 
 1688             try:
 1689                 # If range starts from 0 print all records
 1690                 if low == 0:
 1691                     query = 'SELECT * from bookmarks'
 1692                     resultset = self.cur.execute(query)
 1693                 else:
 1694                     query = 'SELECT * from bookmarks where id BETWEEN ? AND ?'
 1695                     resultset = self.cur.execute(query, (low, high))
 1696             except IndexError:
 1697                 LOGERR('Index out of range')
 1698                 return False
 1699         elif index != 0:  # Show record at index
 1700             try:
 1701                 query = 'SELECT * FROM bookmarks WHERE id = ? LIMIT 1'
 1702                 self.cur.execute(query, (index,))
 1703                 results = self.cur.fetchall()
 1704                 if not results:
 1705                     LOGERR('No matching index %d', index)
 1706                     return False
 1707             except IndexError:
 1708                 LOGERR('No matching index %d', index)
 1709                 return False
 1710 
 1711             if self.json is None:
 1712                 print_rec_with_filter(results, self.field_filter)
 1713             elif self.json:
 1714                 write_string_to_file(format_json(results, True, self.field_filter), self.json)
 1715             else:
 1716                 print(format_json(results, True, self.field_filter))
 1717 
 1718             return True
 1719         else:  # Show all entries
 1720             self.cur.execute('SELECT * FROM bookmarks')
 1721             resultset = self.cur.fetchall()
 1722 
 1723         if not resultset:
 1724             LOGERR('0 records')
 1725             return True
 1726 
 1727         if self.json is None:
 1728             print_rec_with_filter(resultset, self.field_filter)
 1729         elif self.json:
 1730             write_string_to_file(format_json(resultset, field_filter=self.field_filter), self.json)
 1731         else:
 1732             print(format_json(resultset, field_filter=self.field_filter))
 1733 
 1734         return True
 1735 
 1736     def get_tag_all(self):
 1737         """Get list of tags in DB.
 1738 
 1739         Returns
 1740         -------
 1741         tuple
 1742             (list of unique tags sorted alphabetically,
 1743              dictionary of {tag: usage_count}).
 1744         """
 1745 
 1746         tags = []
 1747         unique_tags = []
 1748         dic = {}
 1749         qry = 'SELECT DISTINCT tags, COUNT(tags) FROM bookmarks GROUP BY tags'
 1750         for row in self.cur.execute(qry):
 1751             tagset = row[0].strip(DELIM).split(DELIM)
 1752             for tag in tagset:
 1753                 if tag not in tags:
 1754                     dic[tag] = row[1]
 1755                     tags += (tag,)
 1756                 else:
 1757                     dic[tag] += row[1]
 1758 
 1759         if not tags:
 1760             return tags, dic
 1761 
 1762         if tags[0] == '':
 1763             unique_tags = sorted(tags[1:])
 1764         else:
 1765             unique_tags = sorted(tags)
 1766 
 1767         return unique_tags, dic
 1768 
 1769     def suggest_similar_tag(self, tagstr):
 1770         """Show list of tags those go together in DB.
 1771 
 1772         Parameters
 1773         ----------
 1774         tagstr : str
 1775             Original tag string.
 1776 
 1777         Returns
 1778         -------
 1779         str
 1780             DELIM separated string of tags.
 1781         """
 1782 
 1783         tags = tagstr.split(',')
 1784         if not len(tags):
 1785             return tagstr
 1786 
 1787         qry = 'SELECT DISTINCT tags FROM bookmarks WHERE tags LIKE ?'
 1788         tagset = set()
 1789         for tag in tags:
 1790             if tag == '':
 1791                 continue
 1792 
 1793             self.cur.execute(qry, ('%' + delim_wrap(tag) + '%',))
 1794             results = self.cur.fetchall()
 1795             for row in results:
 1796                 # update tagset with unique tags in row
 1797                 tagset |= set(row[0].strip(DELIM).split(DELIM))
 1798 
 1799         # remove user supplied tags from tagset
 1800         tagset.difference_update(tags)
 1801 
 1802         if not len(tagset):
 1803             return tagstr
 1804 
 1805         unique_tags = sorted(tagset)
 1806 
 1807         print('similar tags:\n')
 1808         for count, tag in enumerate(unique_tags):
 1809             print('%d. %s' % (count + 1, unique_tags[count]))
 1810 
 1811         selected_tags = input('\nselect: ').split()
 1812         print()
 1813         if not selected_tags:
 1814             return tagstr
 1815 
 1816         tags = [tagstr]
 1817         for index in selected_tags:
 1818             try:
 1819                 tags.append(delim_wrap(unique_tags[int(index) - 1]))
 1820             except Exception as e:
 1821                 LOGERR(e)
 1822                 continue
 1823 
 1824         return parse_tags(tags)
 1825 
 1826     def replace_tag(self, orig: str, new: Optional[List[str]] = None) -> bool:
 1827         """Replace original tag by new tags in all records.
 1828 
 1829         Remove original tag if new tag is empty.
 1830 
 1831         Parameters
 1832         ----------
 1833         orig : str
 1834             Original tag.
 1835         new : list
 1836             Replacement tags.
 1837 
 1838         Returns
 1839         -------
 1840         bool
 1841             True on success, False on failure.
 1842         """
 1843 
 1844         newtags = DELIM
 1845 
 1846         orig = delim_wrap(orig)
 1847         if new is not None:
 1848             newtags = parse_tags(new)
 1849 
 1850         if orig == newtags:
 1851             print('Tags are same.')
 1852             return False
 1853 
 1854         # Remove original tag from DB if new tagset reduces to delimiter
 1855         if newtags == DELIM:
 1856             return self.delete_tag_at_index(0, orig)
 1857 
 1858         # Update bookmarks with original tag
 1859         query = 'SELECT id, tags FROM bookmarks WHERE tags LIKE ?'
 1860         self.cur.execute(query, ('%' + orig + '%',))
 1861         results = self.cur.fetchall()
 1862         if results:
 1863             query = 'UPDATE bookmarks SET tags = ? WHERE id = ?'
 1864             for row in results:
 1865                 tags = row[1].replace(orig, newtags)
 1866                 tags = parse_tags([tags])
 1867                 self.cur.execute(query, (tags, row[0],))
 1868                 print('Index %d updated' % row[0])
 1869 
 1870             self.conn.commit()
 1871 
 1872         return True
 1873 
 1874     def get_tagstr_from_taglist(self, id_list, taglist):
 1875         """Get a string of delimiter-separated (and enclosed) string
 1876         of tags from a dictionary of tags by matching ids.
 1877 
 1878         The inputs are the outputs from BukuDb.get_tag_all().
 1879 
 1880         Parameters
 1881         ----------
 1882         id_list : list
 1883             List of ids.
 1884         taglist : list
 1885             List of tags.
 1886         Returns
 1887         -------
 1888         str
 1889             Delimiter separated and enclosed list of tags.
 1890         """
 1891 
 1892         tags = DELIM
 1893 
 1894         for id in id_list:
 1895             if is_int(id) and int(id) > 0:
 1896                 tags += taglist[int(id) - 1] + DELIM
 1897             elif '-' in id:
 1898                 vals = [int(x) for x in id.split('-')]
 1899                 if vals[0] > vals[-1]:
 1900                     vals[0], vals[-1] = vals[-1], vals[0]
 1901 
 1902                 for _id in range(vals[0], vals[-1] + 1):
 1903                     tags += taglist[_id - 1] + DELIM
 1904 
 1905         return tags
 1906 
 1907     def set_tag(self, cmdstr, taglist):
 1908         """Append, overwrite, remove tags using the symbols >>, > and << respectively.
 1909 
 1910         Parameters
 1911         ----------
 1912         cmdstr : str
 1913             Command pattern.
 1914         taglist : list
 1915             List of tags.
 1916 
 1917         Returns
 1918         -------
 1919         int
 1920             Number of indices updated on success, -1 on failure, -2 on no symbol found.
 1921         """
 1922 
 1923         if not cmdstr or not taglist:
 1924             return -1
 1925 
 1926         flag = 0  # 0: invalid, 1: append, 2: overwrite, 3: remove
 1927         index = cmdstr.find('>>')
 1928         if index == -1:
 1929             index = cmdstr.find('>')
 1930             if index != -1:
 1931                 flag = 2
 1932             else:
 1933                 index = cmdstr.find('<<')
 1934                 if index != -1:
 1935                     flag = 3
 1936         else:
 1937             flag = 1
 1938 
 1939         if not flag:
 1940             return -2
 1941 
 1942         tags = DELIM
 1943         id_list = cmdstr[:index].split()
 1944         try:
 1945             tags = self.get_tagstr_from_taglist(id_list, taglist)
 1946             if tags == DELIM:
 1947                 return -1
 1948         except ValueError:
 1949             return -1
 1950 
 1951         if flag != 2:
 1952             index += 1
 1953 
 1954         update_count = 0
 1955         query = 'UPDATE bookmarks SET tags = ? WHERE id = ?'
 1956         try:
 1957             db_id_list = cmdstr[index + 1:].split()
 1958             for id in db_id_list:
 1959                 if is_int(id) and int(id) > 0:
 1960                     if flag == 1:
 1961                         if self.append_tag_at_index(id, tags, True):
 1962                             update_count += 1
 1963                     elif flag == 2:
 1964                         tags = parse_tags([tags])
 1965                         self.cur.execute(query, (tags, id,))
 1966                         update_count += self.cur.rowcount
 1967                     else:
 1968                         self.delete_tag_at_index(id, tags, True)
 1969                         update_count += 1
 1970                 elif '-' in id:
 1971                     vals = [int(x) for x in id.split('-')]
 1972                     if vals[0] > vals[-1]:
 1973                         vals[0], vals[-1] = vals[-1], vals[0]
 1974 
 1975                     for _id in range(vals[0], vals[-1] + 1):
 1976                         if flag == 1:
 1977                             if self.append_tag_at_index(_id, tags, True):
 1978                                 update_count += 1
 1979                         elif flag == 2:
 1980                             tags = parse_tags([tags])
 1981                             self.cur.execute(query, (tags, _id,))
 1982                             update_count += self.cur.rowcount
 1983                         else:
 1984                             if self.delete_tag_at_index(_id, tags, True):
 1985                                 update_count += 1
 1986                 else:
 1987                     return -1
 1988         except ValueError:
 1989             return -1
 1990         except sqlite3.IntegrityError:
 1991             return -1
 1992 
 1993         try:
 1994             self.conn.commit()
 1995         except Exception as e:
 1996             LOGERR(e)
 1997             return -1
 1998 
 1999         return update_count
 2000 
 2001     def browse_by_index(self, index=0, low=0, high=0, is_range=False):
 2002         """Open URL at index or range of indices in browser.
 2003 
 2004         Parameters
 2005         ----------
 2006         index : int
 2007             Index to browse. 0 opens a random bookmark.
 2008         low : int
 2009             Actual lower index of range.
 2010         high : int
 2011             Higher index of range.
 2012         is_range : bool
 2013             A range is passed using low and high arguments.
 2014             If True, index is ignored. Default is False.
 2015 
 2016         Returns
 2017         -------
 2018         bool
 2019             True on success, False on failure.
 2020         """
 2021 
 2022         if is_range:
 2023             if low < 0 or high < 0:
 2024                 LOGERR('Negative range boundary')
 2025                 return False
 2026 
 2027             if low > high:
 2028                 low, high = high, low
 2029 
 2030             try:
 2031                 # If range starts from 0 throw an error
 2032                 if low <= 0:
 2033                     raise IndexError
 2034 
 2035                 qry = 'SELECT URL from bookmarks where id BETWEEN ? AND ?'
 2036                 for row in self.cur.execute(qry, (low, high)):
 2037                     browse(row[0])
 2038                 return True
 2039             except IndexError:
 2040                 LOGERR('Index out of range')
 2041                 return False
 2042 
 2043         if index < 0:
 2044             LOGERR('Invalid index %d', index)
 2045             return False
 2046 
 2047         if index == 0:
 2048             qry = 'SELECT id from bookmarks ORDER BY RANDOM() LIMIT 1'
 2049             self.cur.execute(qry)
 2050             result = self.cur.fetchone()
 2051 
 2052             # Return if no entries in DB
 2053             if result is None:
 2054                 print('No bookmarks added yet ...')
 2055                 return False
 2056 
 2057             index = result[0]
 2058             LOGDBG('Opening random index %d', index)
 2059 
 2060         qry = 'SELECT URL FROM bookmarks WHERE id = ? LIMIT 1'
 2061         try:
 2062             for row in self.cur.execute(qry, (index,)):
 2063                 browse(row[0])
 2064                 return True
 2065             LOGERR('No matching index %d', index)
 2066         except IndexError:
 2067             LOGERR('No matching index %d', index)
 2068 
 2069         return False
 2070 
 2071     def exportdb(self, filepath: str, resultset: Optional[List[BookmarkVar]] = None) -> bool:
 2072         """Export DB bookmarks to file.
 2073         Exports full DB, if resultset is None
 2074 
 2075         If destination file name ends with '.db', bookmarks are
 2076         exported to a buku database file.
 2077         If destination file name ends with '.md', bookmarks are
 2078         exported to a Markdown file.
 2079         If destination file name ends with '.org' bookmarks are
 2080         exported to a org file.
 2081         Otherwise, bookmarks are exported to a Firefox bookmarks.html
 2082         formatted file.
 2083 
 2084         Parameters
 2085         ----------
 2086         filepath : str
 2087             Path to export destination file.
 2088         resultset : list of tuples
 2089             List of results to export.
 2090 
 2091 
 2092         Returns
 2093         -------
 2094         bool
 2095             True on success, False on failure.
 2096         """
 2097 
 2098         count = 0
 2099 
 2100         if not resultset:
 2101             resultset = self.get_rec_all()
 2102             if not resultset:
 2103                 print('No records found')
 2104                 return False
 2105 
 2106         if os.path.exists(filepath):
 2107             resp = read_in(filepath + ' exists. Overwrite? (y/n): ')
 2108             if resp != 'y':
 2109                 return False
 2110 
 2111             if filepath.endswith('.db'):
 2112                 os.remove(filepath)
 2113 
 2114         if filepath.endswith('.db'):
 2115             outdb = BukuDb(dbfile=filepath)
 2116             qry = 'INSERT INTO bookmarks(URL, metadata, tags, desc, flags) VALUES (?, ?, ?, ?, ?)'
 2117             for row in resultset:
 2118                 outdb.cur.execute(qry, (row[1], row[2], row[3], row[4], row[5]))
 2119                 count += 1
 2120             outdb.conn.commit()
 2121             outdb.close()
 2122             print('%s exported' % count)
 2123             return True
 2124 
 2125         try:
 2126             outfp = open(filepath, mode='w', encoding='utf-8')
 2127         except Exception as e:
 2128             LOGERR(e)
 2129             return False
 2130 
 2131         res = {}  # type: Dict
 2132         if filepath.endswith('.md'):
 2133             res = convert_bookmark_set(resultset, 'markdown')
 2134             count += res['count']
 2135             outfp.write(res['data'])
 2136         elif filepath.endswith('.org'):
 2137             res = convert_bookmark_set(resultset, 'org')
 2138             count += res['count']
 2139             outfp.write(res['data'])
 2140         else:
 2141             res = convert_bookmark_set(resultset, 'html')
 2142             count += res['count']
 2143             outfp.write(res['data'])
 2144         outfp.close()
 2145         print('%s exported' % count)
 2146         return True
 2147 
 2148     def traverse_bm_folder(self, sublist, unique_tag, folder_name, add_parent_folder_as_tag):
 2149         """Traverse bookmark folders recursively and find bookmarks.
 2150 
 2151         Parameters
 2152         ----------
 2153         sublist : list
 2154             List of child entries in bookmark folder.
 2155         unique_tag : str
 2156             Timestamp tag in YYYYMonDD format.
 2157         folder_name : str
 2158             Name of the parent folder.
 2159         add_parent_folder_as_tag : bool
 2160             True if bookmark parent folders should be added as tags else False.
 2161 
 2162         Returns
 2163         -------
 2164         tuple
 2165             Bookmark record data.
 2166         """
 2167 
 2168         for item in sublist:
 2169             if item['type'] == 'folder':
 2170                 next_folder_name = folder_name + ',' + item['name']
 2171                 for i in self.traverse_bm_folder(
 2172                         item['children'],
 2173                         unique_tag,
 2174                         next_folder_name,
 2175                         add_parent_folder_as_tag):
 2176                     yield i
 2177             elif item['type'] == 'url':
 2178                 try:
 2179                     if is_nongeneric_url(item['url']):
 2180                         continue
 2181                 except KeyError:
 2182                     continue
 2183 
 2184                 tags = ''
 2185                 if add_parent_folder_as_tag:
 2186                     tags += folder_name
 2187                 if unique_tag:
 2188                     tags += DELIM + unique_tag
 2189                 yield (item['url'], item['name'], parse_tags([tags]), None, 0, True, False)
 2190 
 2191     def load_chrome_database(self, path, unique_tag, add_parent_folder_as_tag):
 2192         """Open Chrome Bookmarks JSON file and import data.
 2193 
 2194         Parameters
 2195         ----------
 2196         path : str
 2197             Path to Google Chrome bookmarks file.
 2198         unique_tag : str
 2199             Timestamp tag in YYYYMonDD format.
 2200         add_parent_folder_as_tag : bool
 2201             True if bookmark parent folders should be added as tags else False.
 2202         """
 2203 
 2204         with open(path, 'r', encoding="utf8") as datafile:
 2205             data = json.load(datafile)
 2206 
 2207         roots = data['roots']
 2208         for entry in roots:
 2209             # Needed to skip 'sync_transaction_version' key from roots
 2210             if isinstance(roots[entry], str):
 2211                 continue
 2212             for item in self.traverse_bm_folder(
 2213                     roots[entry]['children'],
 2214                     unique_tag,
 2215                     roots[entry]['name'],
 2216                     add_parent_folder_as_tag):
 2217                 self.add_rec(*item)
 2218 
 2219     def load_firefox_database(self, path, unique_tag, add_parent_folder_as_tag):
 2220         """Connect to Firefox sqlite db and import bookmarks into BukuDb.
 2221 
 2222         Parameters
 2223         ----------
 2224         path : str
 2225             Path to Firefox bookmarks sqlite database.
 2226         unique_tag : str
 2227             Timestamp tag in YYYYMonDD format.
 2228         add_parent_folder_as_tag : bool
 2229             True if bookmark parent folders should be added as tags else False.
 2230         """
 2231 
 2232         # Connect to input DB
 2233         conn = sqlite3.connect('file:%s?mode=ro' % path, uri=True)
 2234 
 2235         cur = conn.cursor()
 2236         res = cur.execute('SELECT DISTINCT fk, parent, title FROM moz_bookmarks WHERE type=1')
 2237         # get id's and remove duplicates
 2238         for row in res.fetchall():
 2239             # get the url
 2240             res = cur.execute('SELECT url FROM moz_places where id={}'.format(row[0]))
 2241             url = res.fetchone()[0]
 2242             if is_nongeneric_url(url):
 2243                 continue
 2244 
 2245             # get tags
 2246             res = cur.execute('SELECT parent FROM moz_bookmarks WHERE '
 2247                               'fk={} AND title IS NULL'.format(row[0]))
 2248             bm_tag_ids = [tid for item in res.fetchall() for tid in item]
 2249 
 2250             bookmark_tags = []
 2251             for bm_tag_id in bm_tag_ids:
 2252                 res = cur.execute('SELECT title FROM moz_bookmarks WHERE id={}'.format(bm_tag_id))
 2253                 bookmark_tags.append(res.fetchone()[0])
 2254 
 2255             if add_parent_folder_as_tag:
 2256                 # add folder name
 2257                 parent_id = row[1]
 2258                 while parent_id:
 2259                     res = cur.execute('SELECT title,parent FROM moz_bookmarks '
 2260                                       'WHERE id={}'.format(parent_id))
 2261                     parent = res.fetchone()
 2262                     if parent:
 2263                         title, parent_id = parent
 2264                         bookmark_tags.append(title)
 2265 
 2266             if unique_tag:
 2267                 # add timestamp tag
 2268                 bookmark_tags.append(unique_tag)
 2269 
 2270             formatted_tags = [DELIM + tag for tag in bookmark_tags]
 2271             tags = parse_tags(formatted_tags)
 2272 
 2273             # get the title
 2274             if row[2]:
 2275                 title = row[2]
 2276             else:
 2277                 title = ''
 2278 
 2279             self.add_rec(url, title, tags, None, 0, True, False)
 2280         try:
 2281             cur.close()
 2282             conn.close()
 2283         except Exception as e:
 2284             LOGERR(e)
 2285 
 2286     def auto_import_from_browser(self):
 2287         """Import bookmarks from a browser default database file.
 2288 
 2289         Supports Firefox and Google Chrome.
 2290 
 2291         Returns
 2292         -------
 2293         bool
 2294             True on success, False on failure.
 2295         """
 2296 
 2297         ff_bm_db_path = None
 2298 
 2299         if sys.platform.startswith(('linux', 'freebsd', 'openbsd')):
 2300             gc_bm_db_path = '~/.config/google-chrome/Default/Bookmarks'
 2301             cb_bm_db_path = '~/.config/chromium/Default/Bookmarks'
 2302 
 2303             default_ff_folder = os.path.expanduser('~/.mozilla/firefox')
 2304             profile = get_firefox_profile_name(default_ff_folder)
 2305             if profile:
 2306                 ff_bm_db_path = '~/.mozilla/firefox/{}/places.sqlite'.format(profile)
 2307         elif sys.platform == 'darwin':
 2308             gc_bm_db_path = '~/Library/Application Support/Google/Chrome/Default/Bookmarks'
 2309             cb_bm_db_path = '~/Library/Application Support/Chromium/Default/Bookmarks'
 2310 
 2311             default_ff_folder = os.path.expanduser('~/Library/Application Support/Firefox')
 2312             profile = get_firefox_profile_name(default_ff_folder)
 2313             if profile:
 2314                 ff_bm_db_path = ('~/Library/Application Support/Firefox/'
 2315                                  '{}/places.sqlite'.format(profile))
 2316         elif sys.platform == 'win32':
 2317             username = os.getlogin()
 2318             gc_bm_db_path = ('C:/Users/{}/AppData/Local/Google/Chrome/User Data/'
 2319                              'Default/Bookmarks'.format(username))
 2320             cb_bm_db_path = ('C:/Users/{}/AppData/Local/Chromium/User Data/'
 2321                              'Default/Bookmarks'.format(username))
 2322 
 2323             default_ff_folder = 'C:/Users/{}/AppData/Roaming/Mozilla/Firefox/'.format(username)
 2324             profile = get_firefox_profile_name(default_ff_folder)
 2325             if profile:
 2326                 ff_bm_db_path = os.path.join(default_ff_folder, '{}/places.sqlite'.format(profile))
 2327         else:
 2328             LOGERR('buku does not support {} yet'.format(sys.platform))
 2329             self.close_quit(1)
 2330 
 2331         if self.chatty:
 2332             resp = input('Generate auto-tag (YYYYMonDD)? (y/n): ')
 2333             if resp == 'y':
 2334                 newtag = gen_auto_tag()
 2335             else:
 2336                 newtag = None
 2337             resp = input('Add parent folder names as tags? (y/n): ')
 2338         else:
 2339             newtag = None
 2340             resp = 'y'
 2341         add_parent_folder_as_tag = (resp == 'y')
 2342 
 2343         resp = 'y'
 2344 
 2345         try:
 2346             if self.chatty:
 2347                 resp = input('Import bookmarks from google chrome? (y/n): ')
 2348             if resp == 'y':
 2349                 bookmarks_database = os.path.expanduser(gc_bm_db_path)
 2350                 if not os.path.exists(bookmarks_database):
 2351                     raise FileNotFoundError
 2352                 self.load_chrome_database(bookmarks_database, newtag, add_parent_folder_as_tag)
 2353         except Exception as e:
 2354             LOGERR(e)
 2355             print('Could not import bookmarks from google-chrome')
 2356 
 2357         try:
 2358             if self.chatty:
 2359                 resp = input('Import bookmarks from chromium? (y/n): ')
 2360             if resp == 'y':
 2361                 bookmarks_database = os.path.expanduser(cb_bm_db_path)
 2362                 if not os.path.exists(bookmarks_database):
 2363                     raise FileNotFoundError
 2364                 self.load_chrome_database(bookmarks_database, newtag, add_parent_folder_as_tag)
 2365         except Exception as e:
 2366             LOGERR(e)
 2367             print('Could not import bookmarks from chromium')
 2368 
 2369         try:
 2370             if self.chatty:
 2371                 resp = input('Import bookmarks from Firefox? (y/n): ')
 2372             if resp == 'y':
 2373                 bookmarks_database = os.path.expanduser(ff_bm_db_path)
 2374                 if not os.path.exists(bookmarks_database):
 2375                     raise FileNotFoundError
 2376                 self.load_firefox_database(bookmarks_database, newtag, add_parent_folder_as_tag)
 2377         except Exception as e:
 2378             LOGERR(e)
 2379             print('Could not import bookmarks from Firefox.')
 2380 
 2381         self.conn.commit()
 2382 
 2383         if newtag:
 2384             print('\nAuto-generated tag: %s' % newtag)
 2385 
 2386     def importdb(self, filepath, tacit=False):
 2387         """Import bookmarks from a HTML or a Markdown file.
 2388 
 2389         Supports Firefox, Google Chrome, and IE exported HTML bookmarks.
 2390         Supports Markdown files with extension '.md, .org'.
 2391         Supports importing bookmarks from another buku database file.
 2392 
 2393         Parameters
 2394         ----------
 2395         filepath : str
 2396             Path to file to import.
 2397         tacit : bool, optional
 2398             If True, no questions asked and folder names are automatically
 2399             imported as tags from bookmarks HTML.
 2400             If True, automatic timestamp tag is NOT added.
 2401             Default is False.
 2402 
 2403         Returns
 2404         -------
 2405         bool
 2406             True on success, False on failure.
 2407         """
 2408 
 2409         if filepath.endswith('.db'):
 2410             return self.mergedb(filepath)
 2411 
 2412         if not tacit:
 2413             resp = input('Generate auto-tag (YYYYMonDD)? (y/n): ')
 2414             if resp == 'y':
 2415                 newtag = gen_auto_tag()
 2416             else:
 2417                 newtag = None
 2418         else:
 2419             newtag = None
 2420 
 2421         if not tacit:
 2422             append_tags_resp = input('Append tags when bookmark exist? (y/n): ')
 2423         else:
 2424             append_tags_resp = 'y'
 2425 
 2426         items = []
 2427         if filepath.endswith('.md'):
 2428             items = import_md(filepath=filepath, newtag=newtag)
 2429         elif filepath.endswith('org'):
 2430             items = import_org(filepath=filepath, newtag=newtag)
 2431         elif filepath.endswith('json'):
 2432             if not tacit:
 2433                 resp = input('Add parent folder names as tags? (y/n): ')
 2434             else:
 2435                 resp = 'y'
 2436             add_bookmark_folder_as_tag = (resp == 'y')
 2437             try:
 2438                 with open(filepath, 'r', encoding='utf-8') as datafile:
 2439                     data = json.load(datafile)
 2440 
 2441                 items = import_firefox_json(data, add_bookmark_folder_as_tag, newtag)
 2442             except ValueError as e:
 2443                 LOGERR("ff_json: JSON Decode Error: {}".format(e))
 2444                 return False
 2445             except Exception as e:
 2446                 LOGERR(e)
 2447                 return False
 2448         else:
 2449             try:
 2450                 with open(filepath, mode='r', encoding='utf-8') as infp:
 2451                     soup = BeautifulSoup(infp, 'html.parser')
 2452             except ImportError:
 2453                 LOGERR('Beautiful Soup not found')
 2454                 return False
 2455             except Exception as e:
 2456                 LOGERR(e)
 2457                 return False
 2458 
 2459             if not tacit:
 2460                 resp = input('Add parent folder names as tags? (y/n): ')
 2461             else:
 2462                 resp = 'y'
 2463 
 2464             add_parent_folder_as_tag = (resp == 'y')
 2465             items = import_html(soup, add_parent_folder_as_tag, newtag)
 2466             infp.close()
 2467 
 2468         for item in items:
 2469             add_rec_res = self.add_rec(*item)
 2470             if add_rec_res == -1 and append_tags_resp == 'y':
 2471                 rec_id = self.get_rec_id(item[0])
 2472                 self.append_tag_at_index(rec_id, item[2])
 2473 
 2474         self.conn.commit()
 2475 
 2476         if newtag:
 2477             print('\nAuto-generated tag: %s' % newtag)
 2478 
 2479         return True
 2480 
 2481     def mergedb(self, path):
 2482         """Merge bookmarks from another buku database file.
 2483 
 2484         Parameters
 2485         ----------
 2486         path : str
 2487             Path to DB file to merge.
 2488 
 2489         Returns
 2490         -------
 2491         bool
 2492             True on success, False on failure.
 2493         """
 2494 
 2495         try:
 2496             # Connect to input DB
 2497             indb_conn = sqlite3.connect('file:%s?mode=ro' % path, uri=True)
 2498 
 2499             indb_cur = indb_conn.cursor()
 2500             indb_cur.execute('SELECT * FROM bookmarks')
 2501         except Exception as e:
 2502             LOGERR(e)
 2503             return False
 2504 
 2505         resultset = indb_cur.fetchall()
 2506         if resultset:
 2507             for row in resultset:
 2508                 self.add_rec(row[1], row[2], row[3], row[4], row[5], True, False)
 2509 
 2510             self.conn.commit()
 2511 
 2512         try:
 2513             indb_cur.close()
 2514             indb_conn.close()
 2515         except Exception:
 2516             pass
 2517 
 2518         return True
 2519 
 2520     def tnyfy_url(
 2521             self,
 2522             index: Optional[int] = 0,
 2523             url: Optional[str] = None,
 2524             shorten: Optional[bool] = True) -> Optional[str]:
 2525         """Shorten a URL using Google URL shortener.
 2526 
 2527         Parameters
 2528         ----------
 2529         index : int, optional (if URL is provided)
 2530             DB index of the bookmark with the URL to shorten. Default is 0.
 2531         url : str, optional (if index is provided)
 2532             URL to shorten.
 2533         shorten : bool, optional
 2534             True to shorten, False to expand. Default is False.
 2535 
 2536         Returns
 2537         -------
 2538         str
 2539             Shortened url on success, None on failure.
 2540         """
 2541 
 2542         global MYPROXY
 2543 
 2544         if not index and not url:
 2545             LOGERR('Either a valid DB index or URL required')
 2546             return None
 2547 
 2548         if index:
 2549             self.cur.execute('SELECT url FROM bookmarks WHERE id = ? LIMIT 1', (index,))
 2550             results = self.cur.fetchall()
 2551             if not results:
 2552                 return None
 2553 
 2554             url = results[0][0]
 2555 
 2556         from urllib.parse import quote_plus as qp
 2557 
 2558         url = url if url is not None else ''
 2559         urlbase = 'https://tny.im/yourls-api.php?action='
 2560         if shorten:
 2561             _u = urlbase + 'shorturl&format=simple&url=' + qp(url)
 2562         else:
 2563             _u = urlbase + 'expand&format=simple&shorturl=' + qp(url)
 2564 
 2565         if MYPROXY is None:
 2566             gen_headers()
 2567 
 2568         ca_certs = os.getenv('BUKU_CA_CERTS', default=certifi.where())
 2569         if MYPROXY:
 2570             manager = urllib3.ProxyManager(
 2571                 MYPROXY,
 2572                 num_pools=1,
 2573                 headers=MYHEADERS,
 2574                 cert_reqs='CERT_REQUIRED',
 2575                 ca_certs=ca_certs)
 2576         else:
 2577             manager = urllib3.PoolManager(num_pools=1,
 2578                                           headers={'User-Agent': USER_AGENT},
 2579                                           cert_reqs='CERT_REQUIRED',
 2580                                           ca_certs=ca_certs)
 2581 
 2582         try:
 2583             r = manager.request(
 2584                 'POST',
 2585                 _u,
 2586                 headers={
 2587                     'content-type': 'application/json',
 2588                     'User-Agent': USER_AGENT}
 2589             )
 2590         except Exception as e:
 2591             LOGERR(e)
 2592             manager.clear()
 2593             return None
 2594 
 2595         if r.status != 200:
 2596             LOGERR('[%s] %s', r.status, r.reason)
 2597             return None
 2598 
 2599         manager.clear()
 2600 
 2601         return r.data.decode(errors='replace')
 2602 
 2603     def browse_cached_url(self, arg):
 2604         """Open URL at index or URL.
 2605 
 2606         Parameters
 2607         ----------
 2608         arg : str
 2609             Index or url to browse
 2610 
 2611         Returns
 2612         -------
 2613         str
 2614             Wayback Machine URL, None if not cached
 2615         """
 2616 
 2617         from urllib.parse import quote_plus
 2618 
 2619         if is_int(arg):
 2620             rec = self.get_rec_by_id(int(arg))
 2621             if not rec:
 2622                 LOGERR('No matching index %d', int(arg))
 2623                 return None
 2624             url = rec[1]
 2625         else:
 2626             url = arg
 2627 
 2628         # Try fetching cached page from Wayback Machine
 2629         api_url = 'https://archive.org/wayback/available/?url=' + quote_plus(url)
 2630         manager = get_PoolManager()
 2631         resp = manager.request('GET', api_url)
 2632         respobj = json.loads(resp.data.decode('utf-8'))
 2633         try:
 2634             if (
 2635                     len(respobj['archived_snapshots']) and
 2636                     respobj['archived_snapshots']['closest']['available'] is True):
 2637                 manager.clear()
 2638                 return respobj['archived_snapshots']['closest']['url']
 2639         except Exception:
 2640             pass
 2641         finally:
 2642             manager.clear()
 2643 
 2644         LOGERR('Uncached')
 2645         return None
 2646 
 2647     def fixtags(self):
 2648         """Undocumented API to fix tags set in earlier versions.
 2649 
 2650         Functionalities:
 2651 
 2652         1. Remove duplicate tags
 2653         2. Sort tags
 2654         3. Use lower case to store tags
 2655         """
 2656 
 2657         to_commit = False
 2658         self.cur.execute('SELECT id, tags FROM bookmarks ORDER BY id ASC')
 2659         resultset = self.cur.fetchall()
 2660         query = 'UPDATE bookmarks SET tags = ? WHERE id = ?'
 2661         for row in resultset:
 2662             oldtags = row[1]
 2663             if oldtags == DELIM:
 2664                 continue
 2665 
 2666             tags = parse_tags([oldtags])
 2667             if tags == oldtags:
 2668                 continue
 2669 
 2670             self.cur.execute(query, (tags, row[0],))
 2671             to_commit = True
 2672 
 2673         if to_commit:
 2674             self.conn.commit()
 2675 
 2676     def close(self):
 2677         """Close a DB connection."""
 2678 
 2679         if self.conn is not None:
 2680             try:
 2681                 self.cur.close()
 2682                 self.conn.close()
 2683             except Exception:
 2684                 # ignore errors here, we're closing down
 2685                 pass
 2686 
 2687     def close_quit(self, exitval=0):
 2688         """Close a DB connection and exit.
 2689 
 2690         Parameters
 2691         ----------
 2692         exitval : int, optional
 2693             Program exit value.
 2694         """
 2695 
 2696         if self.conn is not None:
 2697             try:
 2698                 self.cur.close()
 2699                 self.conn.close()
 2700             except Exception:
 2701                 # ignore errors here, we're closing down
 2702                 pass
 2703         sys.exit(exitval)
 2704 
 2705 
 2706 class ExtendedArgumentParser(argparse.ArgumentParser):
 2707     """Extend classic argument parser."""
 2708 
 2709     @staticmethod
 2710     def program_info(file=sys.stdout):
 2711         """Print program info.
 2712 
 2713         Parameters
 2714         ----------
 2715         file : file, optional
 2716             File to write program info to. Default is sys.stdout.
 2717         """
 2718         if sys.platform == 'win32' and file == sys.stdout:
 2719             file = sys.stderr
 2720 
 2721         file.write('''
 2722 SYMBOLS:
 2723       >                    url
 2724       +                    comment
 2725       #                    tags
 2726 
 2727 Version %s
 2728 Copyright © 2015-2020 %s
 2729 License: %s
 2730 Webpage: https://github.com/jarun/buku
 2731 ''' % (__version__, __author__, __license__))
 2732 
 2733     @staticmethod
 2734     def prompt_help(file=sys.stdout):
 2735         """Print prompt help.
 2736 
 2737         Parameters
 2738         ----------
 2739         file : file, optional
 2740             File to write program info to. Default is sys.stdout.
 2741         """
 2742         file.write('''
 2743 PROMPT KEYS:
 2744     1-N                    browse search result indices and/or ranges
 2745     O [id|range [...]]     open search results/indices in GUI browser
 2746                            toggle try GUI browser if no arguments
 2747     a                      open all results in browser
 2748     s keyword [...]        search for records with ANY keyword
 2749     S keyword [...]        search for records with ALL keywords
 2750     d                      match substrings ('pen' matches 'opened')
 2751     r expression           run a regex search
 2752     t [tag, ...]           search by tags; show taglist, if no args
 2753     g taglist id|range [...] [>>|>|<<] [record id|range ...]
 2754                            append, set, remove (all or specific) tags
 2755                            search by taglist id(s) if records are omitted
 2756     n                      show next page of search results
 2757     o id|range [...]       browse bookmarks by indices and/or ranges
 2758     p id|range [...]       print bookmarks by indices and/or ranges
 2759     w [editor|id]          edit and add or update a bookmark
 2760     c id                   copy url at search result index to clipboard
 2761     ?                      show this help
 2762     q, ^D, double Enter    exit buku
 2763 
 2764 ''')
 2765 
 2766     @staticmethod
 2767     def is_colorstr(arg):
 2768         """Check if a string is a valid color string.
 2769 
 2770         Parameters
 2771         ----------
 2772         arg : str
 2773             Color string to validate.
 2774 
 2775         Returns
 2776         -------
 2777         str
 2778             Same color string that was passed as an argument.
 2779 
 2780         Raises
 2781         ------
 2782         ArgumentTypeError
 2783             If the arg is not a valid color string.
 2784         """
 2785         try:
 2786             assert len(arg) == 5
 2787             for c in arg:
 2788                 assert c in COLORMAP
 2789         except AssertionError:
 2790             raise argparse.ArgumentTypeError('%s is not a valid color string' % arg)
 2791         return arg
 2792 
 2793     # Help
 2794     def print_help(self, file=sys.stdout):
 2795         """Print help prompt.
 2796 
 2797         Parameters
 2798         ----------
 2799         file : file, optional
 2800             File to write program info to. Default is sys.stdout.
 2801         """
 2802         super(ExtendedArgumentParser, self).print_help(file)
 2803         self.program_info(file)
 2804 
 2805 
 2806 # ----------------
 2807 # Helper functions
 2808 # ----------------
 2809 
 2810 
 2811 ConverterResult = TypedDict('ConverterResult', {'data': str, 'count': int}) if TypedDict else Dict[str, Any]
 2812 
 2813 
 2814 def convert_bookmark_set(
 2815         bookmark_set: List[BookmarkVar],
 2816         export_type: str) -> ConverterResult:  # type: ignore
 2817     """Convert list of bookmark set into multiple data format.
 2818 
 2819     Parameters
 2820     ----------
 2821         bookmark_set: bookmark set
 2822         export type: one of supported type: markdown, html, org
 2823 
 2824     Returns
 2825     -------
 2826         converted data and count of converted bookmark set
 2827     """
 2828     assert export_type in ['markdown', 'html', 'org']
 2829     #  compatibility
 2830     resultset = bookmark_set
 2831 
 2832     count = 0
 2833     out = ''
 2834     if export_type == 'markdown':
 2835         for row in resultset:
 2836             if not row[2] or row[2] is None:
 2837                 out += '- [Untitled](' + row[1] + ')'
 2838             else:
 2839                 out += '- [' + row[2] + '](' + row[1] + ')'
 2840 
 2841             if row[3] != DELIM:
 2842                 out += ' <!-- TAGS: {} -->\n'.format(row[3][1:-1])
 2843             else:
 2844                 out += '\n'
 2845 
 2846             count += 1
 2847     elif export_type == 'org':
 2848         for row in resultset:
 2849             if not row[2]:
 2850                 out += '* [[{}][Untitled]]'.format(row[1])
 2851             else:
 2852                 out += '* [[{}][{}]]'.format(row[1], row[2])
 2853 
 2854             if row[3] != DELIM:
 2855                 # add additional whitespaces for tags that end or start with a colon
 2856                 tag_string = row[3].replace(',:', ', ,:').replace(':,', ':, ,')
 2857                 buku_tags = tag_string.split(DELIM)[1:-1]
 2858                 # if colons are inside a tag, add one additional colon
 2859                 buku_tags = [re.sub(r'(?<=[\w,\:]):(?=\w)', '::', tag) for tag in buku_tags]
 2860                 out += ' :{}:\n'.format(':'.join(buku_tags))
 2861             else:
 2862                 out += '\n'
 2863 
 2864             count += 1
 2865     elif export_type == 'html':
 2866         timestamp = str(int(time.time()))
 2867         out = (
 2868             '<!DOCTYPE NETSCAPE-Bookmark-file-1>\n\n'
 2869             '<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">\n'
 2870             '<TITLE>Bookmarks</TITLE>\n'
 2871             '<H1>Bookmarks</H1>\n\n'
 2872             '<DL><p>\n'
 2873             '    <DT><H3 ADD_DATE="{0}" LAST_MODIFIED="{0}" '
 2874             'PERSONAL_TOOLBAR_FOLDER="true">buku bookmarks</H3>\n'
 2875             '    <DL><p>\n'.format(timestamp))
 2876 
 2877         for row in resultset:
 2878             out += '        <DT><A HREF="%s" ADD_DATE="%s" LAST_MODIFIED="%s"' % (row[1], timestamp, timestamp)
 2879             if row[3] != DELIM:
 2880                 out += ' TAGS="' + row[3][1:-1] + '"'
 2881             out += '>{}</A>\n'.format(row[2] if row[2] else '')
 2882             if row[4] != '':
 2883                 out += '        <DD>' + row[4] + '\n'
 2884             count += 1
 2885 
 2886         out += '    </DL><p>\n</DL><p>'
 2887 
 2888     return {'data': out, 'count': count}
 2889 
 2890 
 2891 def get_firefox_profile_name(path):
 2892     """List folder and detect default Firefox profile name.
 2893 
 2894     Returns
 2895     -------
 2896     profile : str
 2897         Firefox profile name.
 2898     """
 2899     from configparser import ConfigParser, NoOptionError
 2900 
 2901     profile_path = os.path.join(path, 'profiles.ini')
 2902     if os.path.exists(profile_path):
 2903         config = ConfigParser()
 2904         config.read(profile_path)
 2905 
 2906         install_names = [section for section in config.sections() if section.startswith('Install')]
 2907         for name in install_names:
 2908             try:
 2909                 profile_path = config.get(name, 'default')
 2910                 return profile_path
 2911             except NoOptionError:
 2912                 pass
 2913 
 2914         profiles_names = [section for section in config.sections() if section.startswith('Profile')]
 2915         if not profiles_names:
 2916             return None
 2917         for name in profiles_names:
 2918             try:
 2919                 # If profile is default
 2920                 if config.getboolean(name, 'default'):
 2921                     profile_path = config.get(name, 'path')
 2922                     return profile_path
 2923             except NoOptionError:
 2924                 pass
 2925             try:
 2926                 # alternative way to detect default profile
 2927                 if config.get(name, 'name').lower() == "default":
 2928                     profile_path = config.get(name, 'path')
 2929                     return profile_path
 2930             except NoOptionError:
 2931                 pass
 2932 
 2933         # There is no default profile
 2934         return None
 2935     LOGDBG('get_firefox_profile_name(): {} does not exist'.format(path))
 2936     return None
 2937 
 2938 
 2939 def walk(root):
 2940     """Recursively iterate over JSON.
 2941 
 2942     Parameters
 2943     ----------
 2944     root : JSON element
 2945         Base node of the JSON data.
 2946     """
 2947 
 2948     for element in root['children']:
 2949         if element['type'] == 'url':
 2950             url = element['url']
 2951             title = element['name']
 2952             yield (url, title, None, None, 0, True)
 2953         else:
 2954             walk(element)
 2955 
 2956 
 2957 def import_md(filepath, newtag):
 2958     """Parse bookmark Markdown file.
 2959 
 2960     Parameters
 2961     ----------
 2962     filepath : str
 2963         Path to Markdown file.
 2964     newtag : str
 2965         New tag for bookmarks in Markdown file.
 2966 
 2967     Returns
 2968     -------
 2969     tuple
 2970         Parsed result.
 2971     """
 2972     with open(filepath, mode='r', encoding='utf-8') as infp:
 2973         for line in infp:
 2974             # Supported Markdown format: [title](url)
 2975             # Find position of title end, url start delimiter combo
 2976             index = line.find('](')
 2977             if index != -1:
 2978                 # Find title start delimiter
 2979                 title_start_delim = line[:index].find('[')
 2980                 # Reverse find the url end delimiter
 2981                 url_end_delim = line[index + 2:].rfind(')')
 2982 
 2983                 if title_start_delim != -1 and url_end_delim > 0:
 2984                     # Parse title
 2985                     title = line[title_start_delim + 1:index]
 2986                     # Parse url
 2987                     url = line[index + 2:index + 2 + url_end_delim]
 2988                     if is_nongeneric_url(url):
 2989                         continue
 2990 
 2991                     yield (
 2992                         url, title, delim_wrap(newtag)
 2993                         if newtag else None, None, 0, True
 2994                     )
 2995 
 2996 def import_org(filepath, newtag):
 2997     """Parse bookmark org file.
 2998 
 2999     Parameters
 3000     ----------
 3001     filepath : str
 3002         Path to org file.
 3003     newtag : str
 3004         New tag for bookmarks in org file.
 3005 
 3006     Returns
 3007     -------
 3008     tuple
 3009         Parsed result.
 3010     """
 3011     def get_org_tags(tag_string):
 3012         """Extracts tags from Org
 3013 
 3014         Parameters
 3015         ----------
 3016         tag_string: str
 3017              string of tags in Org-format
 3018 
 3019         Syntax: Org splits tags with colons. If colons are part of a buku-tag, this is indicated by using
 3020                 multiple colons in org. If a buku-tag starts or ends with a colon, this is indicated by a
 3021                 preceding or trailing whitespace
 3022 
 3023         Returns
 3024         -------
 3025         list
 3026             List of tags
 3027         """
 3028         tag_list_raw = [i for i in re.split(r'(?<!\:)\:', tag_string) if i]
 3029         tag_list_cleaned = []
 3030         for i, tag in enumerate(tag_list_raw):
 3031             if tag.startswith(":"):
 3032                 if tag_list_raw[i-1] == ' ':
 3033                     tag_list_cleaned.append(tag.strip())
 3034                 else:
 3035                     new_item = tag_list_cleaned[-1] + tag
 3036                     del tag_list_cleaned[-1]
 3037                     tag_list_cleaned.append(new_item.strip())
 3038             elif tag != ' ':
 3039                 tag_list_cleaned.append(tag.strip())
 3040         return tag_list_cleaned
 3041 
 3042     with open(filepath, mode='r', encoding='utf-8') as infp:
 3043         # Supported Markdown format: * [[url][title]] :tags:
 3044         # Find position of url end, title start delimiter combo
 3045         for line in infp:
 3046             index = line.find('][')
 3047             if index != -1:
 3048                 # Find url start delimiter
 3049                 url_start_delim = line[:index].find('[[')
 3050                 # Reverse find title end delimiter
 3051                 title_end_delim = line[index + 2:].rfind(']]')
 3052 
 3053                 if url_start_delim != -1 and title_end_delim > 0:
 3054                     # Parse title
 3055                     title = line[index + 2: index + 2 + title_end_delim]
 3056                     # Parse url
 3057                     url = line[url_start_delim + 2:index]
 3058                     # Parse Tags
 3059                     tags = list(collections.OrderedDict.fromkeys(get_org_tags(line[(index + 4 + title_end_delim):])))
 3060                     tags_string = DELIM.join(tags)
 3061 
 3062                     if is_nongeneric_url(url):
 3063                         continue
 3064 
 3065                     if newtag:
 3066                         if newtag.lower() not in tags:
 3067                             tags_string = (newtag + DELIM) + tags_string
 3068 
 3069                     yield (
 3070                         url, title, delim_wrap(tags_string)
 3071                         if newtag else None, None, 0, True
 3072                     )
 3073 
 3074 def import_firefox_json(json, add_bookmark_folder_as_tag=False, unique_tag=None):
 3075     """Open Firefox JSON export file and import data.
 3076     Ignore 'SmartBookmark'  and 'Separator'  entries.
 3077 
 3078     Needed/used fields out of the JSON schema of the bookmarks:
 3079 
 3080     title              : the name/title of the entry
 3081     tags               : ',' separated tags for the bookmark entry
 3082     typeCode           : 1 - uri, 2 - subfolder, 3 - separator
 3083     annos/{name,value} : following annotation entries are used
 3084         name : Places/SmartBookmark            : identifies smart folder, ignored
 3085         name : bookmarkPropereties/description :  detailed bookmark entry description
 3086     children           : for subfolders, recurse into the child entries
 3087 
 3088     Parameters
 3089     ----------
 3090     path : str
 3091         Path to Firefox JSON bookmarks file.
 3092     unique_tag : str
 3093         Timestamp tag in YYYYMonDD format.
 3094     add_bookmark_folder_as_tag : bool
 3095         True if bookmark parent folder should be added as tags else False.
 3096     """
 3097 
 3098     class TypeCode(Enum):
 3099         """ Format
 3100             typeCode
 3101                 1 : uri        (type=text/x-moz-place)
 3102                 2 : subfolder  (type=text/x-moz-container)
 3103                 3 : separator  (type=text/x-moz-separator)
 3104         """
 3105         uri = 1
 3106         folder = 2
 3107         separator = 3
 3108 
 3109     def is_smart(entry):
 3110         result = False
 3111         try:
 3112             d = [anno for anno in entry['annos'] if anno['name'] == "Places/SmartBookmark"]
 3113             result = bool(len(d))
 3114         except Exception:
 3115             result = False
 3116 
 3117         return result
 3118 
 3119     def extract_desc(entry):
 3120         try:
 3121             d = [
 3122                 anno for anno in entry['annos']
 3123                 if anno['name'] == "bookmarkProperties/description"
 3124             ]
 3125             return d[0]['value']
 3126         except Exception:
 3127             LOGDBG("ff_json: No description found for entry: {} {}".format(entry['uri'], entry['title']))
 3128             return ""
 3129 
 3130     def extract_tags(entry):
 3131         tags = []
 3132         try:
 3133             tags = entry['tags'].split(',')
 3134         except Exception:
 3135             LOGDBG("ff_json: No tags found for entry: {} {}".format(entry['uri'], entry['title']))
 3136 
 3137         return tags
 3138 
 3139     def iterate_children(parent_folder, entry_list):
 3140         for bm_entry in entry_list:
 3141             entry_title = bm_entry['title'] if 'title' in bm_entry else "<no title>"
 3142 
 3143             try:
 3144                 typeCode = bm_entry['typeCode']
 3145             except Exception:
 3146                 LOGDBG("ff_json: item without typeCode found, ignoring: {}".format(entry_title))
 3147                 continue
 3148 
 3149             LOGDBG("ff_json: processing typeCode '{}', title '{}'".format(typeCode, entry_title))
 3150             if TypeCode.uri.value == typeCode:
 3151                 try:
 3152                     if is_smart(bm_entry):
 3153                         LOGDBG("ff_json: SmartBookmark found, ignoring: {}".format(entry_title))
 3154                         continue
 3155 
 3156                     if is_nongeneric_url(bm_entry['uri']):
 3157                         LOGDBG("ff_json: Non-Generic URL found, ignoring: {}".format(entry_title))
 3158                         continue
 3159 
 3160                     desc = extract_desc(bm_entry)
 3161                     bookmark_tags = extract_tags(bm_entry)
 3162 
 3163                     # if parent_folder is not "None"
 3164                     if add_bookmark_folder_as_tag and parent_folder:
 3165                         bookmark_tags.append(parent_folder)
 3166 
 3167                     if unique_tag:
 3168                         bookmark_tags.append(unique_tag)
 3169 
 3170                     formatted_tags = [DELIM + tag for tag in bookmark_tags]
 3171                     tags = parse_tags(formatted_tags)
 3172 
 3173                     LOGDBG("ff_json: Entry found: {}, {}, {}, {} " .format(bm_entry['uri'], entry_title, tags, desc))
 3174                     yield (bm_entry['uri'], entry_title, tags, desc, 0, True, False)
 3175 
 3176                 except Exception as e:
 3177                     LOGERR("ff_json: Error parsing entry '{}' Exception '{}'".format(entry_title, e))
 3178 
 3179             elif TypeCode.folder.value == typeCode:
 3180 
 3181                 # ignore special bookmark folders
 3182                 if 'root' in bm_entry and bm_entry['root'] in IGNORE_FF_BOOKMARK_FOLDERS:
 3183                     LOGDBG("ff_json: ignoring root folder: {}" .format(entry_title))
 3184                     entry_title = None
 3185 
 3186                 if "children" in bm_entry:
 3187                     yield from iterate_children(entry_title, bm_entry['children'])
 3188                 else:
 3189                     # if any of the properties does not exist, bail out silently
 3190                     LOGDBG("ff_json: No 'children' found in bookmark folder - skipping: {}".format(entry_title))
 3191 
 3192             elif TypeCode.separator.value == typeCode:
 3193                 # ignore separator
 3194                 pass
 3195             else:
 3196                 LOGDBG("ff_json: Unknown typeCode found : {}".format(typeCode))
 3197 
 3198     if "children" in json:
 3199         main_entry_list = json['children']
 3200     else:
 3201         LOGDBG("ff_json: No children in Root entry found")
 3202         return []
 3203 
 3204     yield from iterate_children(None, main_entry_list)
 3205 
 3206 
 3207 def import_html(html_soup, add_parent_folder_as_tag, newtag):
 3208     """Parse bookmark HTML.
 3209 
 3210     Parameters
 3211     ----------
 3212     html_soup : BeautifulSoup object
 3213         BeautifulSoup representation of bookmark HTML.
 3214     add_parent_folder_as_tag : bool
 3215         True if bookmark parent folders should be added as tags else False.
 3216     newtag : str
 3217         A new unique tag to add to imported bookmarks.
 3218 
 3219     Returns
 3220     -------
 3221     tuple
 3222         Parsed result.
 3223     """
 3224 
 3225     # compatibility
 3226     soup = html_soup
 3227 
 3228     for tag in soup.findAll('a'):
 3229         # Extract comment from <dd> tag
 3230         try:
 3231             if is_nongeneric_url(tag['href']):
 3232                 continue
 3233         except KeyError:
 3234             continue
 3235 
 3236         desc = None
 3237         comment_tag = tag.findNextSibling('dd')
 3238 
 3239         if comment_tag:
 3240             desc = comment_tag.find(text=True, recursive=False)
 3241 
 3242         # add parent folder as tag
 3243         if add_parent_folder_as_tag:
 3244             # could be its folder or not
 3245             possible_folder = tag.find_previous('h3')
 3246             # get list of tags within that folder
 3247             tag_list = tag.parent.parent.find_parent('dl')
 3248 
 3249             if ((possible_folder) and possible_folder.parent in list(tag_list.parents)):
 3250                 # then it's the folder of this bookmark
 3251                 if tag.has_attr('tags'):
 3252                     tag['tags'] += (DELIM + possible_folder.text)
 3253                 else:
 3254                     tag['tags'] = possible_folder.text
 3255 
 3256         # add unique tag if opted
 3257         if newtag:
 3258             if tag.has_attr('tags'):
 3259                 tag['tags'] += (DELIM + newtag)
 3260             else:
 3261                 tag['tags'] = newtag
 3262 
 3263         yield (
 3264             tag['href'], tag.string,
 3265             parse_tags([tag['tags']]) if tag.has_attr('tags') else None,
 3266             desc if not desc else desc.strip(), 0, True, False
 3267         )
 3268 
 3269 
 3270 def is_bad_url(url):
 3271     """Check if URL is malformed.
 3272 
 3273     .. Note:: This API is not bulletproof but works in most cases.
 3274 
 3275     Parameters
 3276     ----------
 3277     url : str
 3278         URL to scan.
 3279 
 3280     Returns
 3281     -------
 3282     bool
 3283         True if URL is malformed, False otherwise.
 3284     """
 3285 
 3286     # Get the netloc token
 3287     try:
 3288         netloc = parse_url(url).netloc
 3289         if not netloc:
 3290             # Try of prepend '//' and get netloc
 3291             netloc = parse_url('//' + url).netloc
 3292             if not netloc:
 3293                 return True
 3294     except LocationParseError as e:
 3295         LOGERR('%s, URL: %s', e, url)
 3296         return True
 3297 
 3298     LOGDBG('netloc: %s', netloc)
 3299 
 3300     # netloc cannot start or end with a '.'
 3301     if netloc.startswith('.') or netloc.endswith('.'):
 3302         return True
 3303 
 3304     # netloc should have at least one '.'
 3305     if netloc.rfind('.') < 0:
 3306         return True
 3307 
 3308     return False
 3309 
 3310 
 3311 def is_nongeneric_url(url):
 3312     """Returns True for URLs which are non-http and non-generic.
 3313 
 3314     Parameters
 3315     ----------
 3316     url : str
 3317         URL to scan.
 3318 
 3319     Returns
 3320     -------
 3321     bool
 3322         True if URL is a non-generic URL, False otherwise.
 3323     """
 3324 
 3325     ignored_prefix = [
 3326         'about:',
 3327         'apt:',
 3328         'chrome://',
 3329         'file://',
 3330         'place:',
 3331     ]
 3332 
 3333     for prefix in ignored_prefix:
 3334         if url.startswith(prefix):
 3335             return True
 3336 
 3337     return False
 3338 
 3339 
 3340 def is_ignored_mime(url):
 3341     """Check if URL links to ignored MIME.
 3342 
 3343     .. Note:: Only a 'HEAD' request is made for these URLs.
 3344 
 3345     Parameters
 3346     ----------
 3347     url : str
 3348         URL to scan.
 3349 
 3350     Returns
 3351     -------
 3352     bool
 3353         True if URL links to ignored MIME, False otherwise.
 3354     """
 3355 
 3356     for mime in SKIP_MIMES:
 3357         if url.lower().endswith(mime):
 3358             LOGDBG('matched MIME: %s', mime)
 3359             return True
 3360 
 3361     return False
 3362 
 3363 
 3364 def is_unusual_tag(tagstr):
 3365     """Identify unusual tags with word to comma ratio > 3.
 3366 
 3367     Parameters
 3368     ----------
 3369     tagstr : str
 3370         tag string to check.
 3371 
 3372     Returns
 3373     -------
 3374     bool
 3375         True if valid tag else False.
 3376     """
 3377 
 3378     if not tagstr:
 3379         return False
 3380 
 3381     nwords = len(tagstr.split())
 3382     ncommas = tagstr.count(',') + 1
 3383 
 3384     if nwords / ncommas > 3:
 3385         return True
 3386 
 3387     return False
 3388 
 3389 
 3390 def parse_decoded_page(page):
 3391     """Fetch title, description and keywords from decoded HTML page.
 3392 
 3393     Parameters
 3394     ----------
 3395     page : str
 3396         Decoded HTML page.
 3397 
 3398     Returns
 3399     -------
 3400     tuple
 3401         (title, description, keywords).
 3402     """
 3403 
 3404     title = None
 3405     desc = None
 3406     keys = None
 3407 
 3408     soup = BeautifulSoup(page, 'html5lib')
 3409 
 3410     try:
 3411         title = soup.find('title').text.strip().replace('\n', ' ')
 3412         if title:
 3413             title = re.sub(r'\s{2,}', ' ', title)
 3414     except Exception as e:
 3415         LOGDBG(e)
 3416 
 3417     description = (soup.find('meta', attrs={'name':'description'}) or
 3418                    soup.find('meta', attrs={'name':'Description'}) or
 3419                    soup.find('meta', attrs={'property':'description'}) or
 3420                    soup.find('meta', attrs={'property':'Description'}) or
 3421                    soup.find('meta', attrs={'name':'og:description'}) or
 3422                    soup.find('meta', attrs={'name':'og:Description'}) or
 3423                    soup.find('meta', attrs={'property':'og:description'}) or
 3424                    soup.find('meta', attrs={'property':'og:Description'}))
 3425     try:
 3426         if description:
 3427             desc = description.get('content').strip()
 3428             if desc:
 3429                 desc = re.sub(r'\s{2,}', ' ', desc)
 3430     except Exception as e:
 3431         LOGDBG(e)
 3432 
 3433     keywords = (soup.find('meta', attrs={'name':'keywords'}) or
 3434                 soup.find('meta', attrs={'name':'Keywords'}))
 3435     try:
 3436         if keywords:
 3437             keys = keywords.get('content').strip().replace('\n', ' ')
 3438             keys = re.sub(r'\s{2,}', ' ', keys)
 3439             if is_unusual_tag(keys):
 3440                 if keys not in (title, desc):
 3441                     LOGDBG('keywords to description: %s', keys)
 3442                     if desc:
 3443                         desc = desc + '\n## ' + keys
 3444                     else:
 3445                         desc = '* ' + keys
 3446 
 3447                 keys = None
 3448     except Exception as e:
 3449         LOGDBG(e)
 3450 
 3451     LOGDBG('title: %s', title)
 3452     LOGDBG('desc : %s', desc)
 3453     LOGDBG('keys : %s', keys)
 3454 
 3455     return (title, desc, keys)
 3456 
 3457 
 3458 def get_data_from_page(resp):
 3459     """Detect HTTP response encoding and invoke parser with decoded data.
 3460 
 3461     Parameters
 3462     ----------
 3463     resp : HTTP response
 3464         Response from GET request.
 3465 
 3466     Returns
 3467     -------
 3468     tuple
 3469         (title, description, keywords).
 3470     """
 3471 
 3472     try:
 3473         soup = BeautifulSoup(resp.data, 'html.parser')
 3474     except Exception as e:
 3475         LOGERR('get_data_from_page(): %s', e)
 3476 
 3477     try:
 3478         charset = None
 3479 
 3480         if soup.meta and soup.meta.get('charset') is not None:
 3481             charset = soup.meta.get('charset')
 3482         elif 'content-type' in resp.headers:
 3483             _, params = cgi.parse_header(resp.headers['content-type'])
 3484             if params.get('charset') is not None:
 3485                 charset = params.get('charset')
 3486 
 3487         if not charset and soup:
 3488             meta_tag = soup.find('meta', attrs={'http-equiv': 'Content-Type'})
 3489             if meta_tag:
 3490                 _, params = cgi.parse_header(meta_tag.attrs['content'])
 3491                 charset = params.get('charset', charset)
 3492 
 3493         if charset:
 3494             LOGDBG('charset: %s', charset)
 3495             title, desc, keywords = parse_decoded_page(resp.data.decode(charset, errors='replace'))
 3496         else:
 3497             title, desc, keywords = parse_decoded_page(resp.data.decode(errors='replace'))
 3498 
 3499         return (title, desc, keywords)
 3500     except Exception as e:
 3501         LOGERR(e)
 3502         return (None, None, None)
 3503 
 3504 
 3505 def gen_headers():
 3506     """Generate headers for network connection."""
 3507 
 3508     global MYHEADERS, MYPROXY
 3509 
 3510     MYHEADERS = {
 3511         'Accept-Encoding': 'gzip,deflate',
 3512         'User-Agent': USER_AGENT,
 3513         'Accept': '*/*',
 3514         'Cookie': '',
 3515         'DNT': '1'
 3516     }
 3517 
 3518     MYPROXY = os.environ.get('https_proxy')
 3519     if MYPROXY:
 3520         try:
 3521             url = parse_url(MYPROXY)
 3522         except Exception as e:
 3523             LOGERR(e)
 3524             return
 3525 
 3526         # Strip username and password (if present) and update headers
 3527         if url.auth:
 3528             MYPROXY = MYPROXY.replace(url.auth + '@', '')
 3529             auth_headers = make_headers(basic_auth=url.auth)
 3530             MYHEADERS.update(auth_headers)
 3531 
 3532         LOGDBG('proxy: [%s]', MYPROXY)
 3533 
 3534 
 3535 def get_PoolManager():
 3536     """Creates a pool manager with proxy support, if applicable.
 3537 
 3538     Returns
 3539     -------
 3540     ProxyManager or PoolManager
 3541         ProxyManager if https_proxy is defined, PoolManager otherwise.
 3542     """
 3543     ca_certs = os.getenv('BUKU_CA_CERTS', default=certifi.where())
 3544     if MYPROXY:
 3545         return urllib3.ProxyManager(MYPROXY, num_pools=1, headers=MYHEADERS, timeout=15,
 3546                                     cert_reqs='CERT_REQUIRED', ca_certs=ca_certs)
 3547 
 3548     return urllib3.PoolManager(
 3549         num_pools=1,
 3550         headers=MYHEADERS,
 3551         timeout=15,
 3552         cert_reqs='CERT_REQUIRED',
 3553         ca_certs=ca_certs)
 3554 
 3555 
 3556 def network_handler(
 3557         url: str,
 3558         http_head: Optional[bool] = False
 3559 ) -> Tuple[Optional[str], Optional[str], Optional[str], int, int]:
 3560     """Handle server connection and redirections.
 3561 
 3562     Parameters
 3563     ----------
 3564     url : str
 3565         URL to fetch.
 3566     http_head : bool
 3567         If True, send only HTTP HEAD request. Default is False.
 3568 
 3569     Returns
 3570     -------
 3571     tuple
 3572         (title, description, tags, recognized mime, bad url).
 3573     """
 3574 
 3575     page_title = None
 3576     page_desc = None
 3577     page_keys = None
 3578     exception = False
 3579 
 3580     if is_nongeneric_url(url) or is_bad_url(url):
 3581         return (None, None, None, 0, 1)
 3582 
 3583     if is_ignored_mime(url) or http_head:
 3584         method = 'HEAD'
 3585     else:
 3586         method = 'GET'
 3587 
 3588     if not MYHEADERS:
 3589         gen_headers()
 3590 
 3591     try:
 3592         manager = get_PoolManager()
 3593 
 3594         while True:
 3595             resp = manager.request(method, url, retries=Retry(redirect=10))
 3596 
 3597             if resp.status == 200:
 3598                 if method == 'GET':
 3599                     page_title, page_desc, page_keys = get_data_from_page(resp)
 3600             elif resp.status == 403 and url.endswith('/'):
 3601                 # HTTP response Forbidden
 3602                 # Handle URLs in the form of https://www.domain.com/
 3603                 # which fail when trying to fetch resource '/'
 3604                 # retry without trailing '/'
 3605 
 3606                 LOGDBG('Received status 403: retrying...')
 3607                 # Remove trailing /
 3608                 url = url[:-1]
 3609                 resp.close()
 3610                 continue
 3611             else:
 3612                 LOGERR('[%s] %s', resp.status, resp.reason)
 3613 
 3614             if resp:
 3615                 resp.close()
 3616 
 3617             break
 3618     except Exception as e:
 3619         LOGERR('network_handler(): %s', e)
 3620         exception = True
 3621     finally:
 3622         if manager:
 3623             manager.clear()
 3624         if exception:
 3625             return (None, None, None, 0, 0)
 3626         if method == 'HEAD':
 3627             return ('', '', '', 1, 0)
 3628         if page_title is None:
 3629             return ('', page_desc, page_keys, 0, 0)
 3630 
 3631         return (page_title, page_desc, page_keys, 0, 0)
 3632 
 3633 
 3634 def parse_tags(keywords=[]):
 3635     """Format and get tag string from tokens.
 3636 
 3637     Parameters
 3638     ----------
 3639     keywords : list, optional
 3640         List of tags to parse. Default is empty list.
 3641 
 3642     Returns
 3643     -------
 3644     str
 3645         Comma-delimited string of tags.
 3646     DELIM : str
 3647         If no keywords, returns the delimiter.
 3648     None
 3649         If keywords is None.
 3650     """
 3651 
 3652     if keywords is None:
 3653         return None
 3654 
 3655     if not keywords or len(keywords) < 1 or not keywords[0]:
 3656         return DELIM
 3657 
 3658     tags = DELIM
 3659 
 3660     # Cleanse and get the tags
 3661     tagstr = ' '.join(keywords)
 3662     marker = tagstr.find(DELIM)
 3663 
 3664     while marker >= 0:
 3665         token = tagstr[0:marker]
 3666         tagstr = tagstr[marker + 1:]
 3667         marker = tagstr.find(DELIM)
 3668         token = token.strip()
 3669         if token == '':
 3670             continue
 3671 
 3672         tags += token + DELIM
 3673 
 3674     tagstr = tagstr.strip()
 3675     if tagstr != '':
 3676         tags += tagstr + DELIM
 3677 
 3678     LOGDBG('keywords: %s', keywords)
 3679     LOGDBG('parsed tags: [%s]', tags)
 3680 
 3681     if tags == DELIM:
 3682         return tags
 3683 
 3684     # original tags in lower case
 3685     orig_tags = tags.lower().strip(DELIM).split(DELIM)
 3686 
 3687     # Create list of unique tags and sort
 3688     unique_tags = sorted(set(orig_tags))
 3689 
 3690     # Wrap with delimiter
 3691     return delim_wrap(DELIM.join(unique_tags))
 3692 
 3693 
 3694 def prep_tag_search(tags: str) -> Tuple[List[str], Optional[str], Optional[str]]:
 3695     """Prepare list of tags to search and determine search operator.
 3696 
 3697     Parameters
 3698     ----------
 3699     tags : str
 3700         String list of tags to search.
 3701 
 3702     Returns
 3703     -------
 3704     tuple
 3705         (list of formatted tags to search,
 3706          a string indicating query search operator (either OR or AND),
 3707          a regex string of tags or None if ' - ' delimiter not in tags).
 3708     """
 3709 
 3710     exclude_only = False
 3711 
 3712     # tags may begin with `- ` if only exclusion list is provided
 3713     if tags.startswith('- '):
 3714         tags = ' ' + tags
 3715         exclude_only = True
 3716 
 3717     # tags may start with `+ ` etc., tricky test case
 3718     if tags.startswith(('+ ', ', ')):
 3719         tags = tags[2:]
 3720 
 3721     # tags may end with ` -` etc., tricky test case
 3722     if tags.endswith((' -', ' +', ' ,')):
 3723         tags = tags[:-2]
 3724 
 3725     # tag exclusion list can be separated by comma (,), so split it first
 3726     excluded_tags = None
 3727     if ' - ' in tags:
 3728         tags, excluded_tags = tags.split(' - ', 1)
 3729 
 3730         excluded_taglist = [delim_wrap(re.escape(t.strip())) for t in excluded_tags.split(',')]
 3731         # join with pipe to construct regex string
 3732         excluded_tags = '|'.join(excluded_taglist)
 3733 
 3734     if exclude_only:
 3735         search_operator = 'OR'
 3736         tags_ = ['']
 3737     else:
 3738         # do not allow combination of search logics in tag inclusion list
 3739         if ' + ' in tags and ',' in tags:
 3740             return [], None, None
 3741 
 3742         search_operator = 'OR'
 3743         tag_delim = ','
 3744         if ' + ' in tags:
 3745             search_operator = 'AND'
 3746             tag_delim = ' + '
 3747 
 3748         tags_ = [delim_wrap(t.strip()) for t in tags.split(tag_delim)]
 3749 
 3750     return tags_, search_operator, excluded_tags
 3751 
 3752 
 3753 def gen_auto_tag():
 3754     """Generate a tag in Year-Month-Date format.
 3755 
 3756     Returns
 3757     -------
 3758     str
 3759         New tag as YYYYMonDD.
 3760     """
 3761 
 3762     t = time.localtime()
 3763     return '%d%s%02d' % (t.tm_year, calendar.month_abbr[t.tm_mon], t.tm_mday)
 3764 
 3765 
 3766 def edit_at_prompt(obj, nav, suggest=False):
 3767     """Edit and add or update a bookmark.
 3768 
 3769     Parameters
 3770     ----------
 3771     obj : BukuDb instance
 3772         A valid instance of BukuDb class.
 3773     nav : str
 3774         Navigation command argument passed at prompt by user.
 3775     suggest : bool, optional
 3776         If True, suggest similar tags on new bookmark addition.
 3777     """
 3778 
 3779     if nav == 'w':
 3780         editor = get_system_editor()
 3781         if not is_editor_valid(editor):
 3782             return
 3783     elif is_int(nav[2:]):
 3784         obj.edit_update_rec(int(nav[2:]))
 3785         return
 3786     else:
 3787         editor = nav[2:]
 3788 
 3789     result = edit_rec(editor, '', None, DELIM, None)
 3790     if result is not None:
 3791         url, title, tags, desc = result
 3792         if suggest:
 3793             tags = obj.suggest_similar_tag(tags)
 3794         obj.add_rec(url, title, tags, desc)
 3795 
 3796 
 3797 def show_taglist(obj):
 3798     """Additional prompt to show unique tag list.
 3799 
 3800     Parameters
 3801     ----------
 3802     obj : BukuDb instance
 3803         A valid instance of BukuDb class.
 3804     """
 3805 
 3806     unique_tags, dic = obj.get_tag_all()
 3807     if not unique_tags:
 3808         count = 0
 3809         print('0 tags')
 3810     else:
 3811         count = 1
 3812         for tag in unique_tags:
 3813             print('%6d. %s (%d)' % (count, tag, dic[tag]))
 3814             count += 1
 3815         print()
 3816 
 3817 
 3818 def prompt(obj, results, noninteractive=False, deep=False, listtags=False, suggest=False, num=10):
 3819     """Show each matching result from a search and prompt.
 3820 
 3821     Parameters
 3822     ----------
 3823     obj : BukuDb instance
 3824         A valid instance of BukuDb class.
 3825     results : list
 3826         Search result set from a DB query.
 3827     noninteractive : bool, optional
 3828         If True, does not seek user input. Shows all results. Default is False.
 3829     deep : bool, optional
 3830         Use deep search. Default is False.
 3831     listtags : bool, optional
 3832         If True, list all tags.
 3833     suggest : bool, optional
 3834         If True, suggest similar tags on edit and add bookmark.
 3835     num : int, optional
 3836         Number of results to show per page. Default is 10.
 3837     """
 3838 
 3839     if not isinstance(obj, BukuDb):
 3840         LOGERR('Not a BukuDb instance')
 3841         return
 3842 
 3843     new_results = bool(results)
 3844     nav = ''
 3845     cur_index = next_index = count = 0
 3846 
 3847     if listtags:
 3848         show_taglist(obj)
 3849 
 3850     if noninteractive:
 3851         try:
 3852             for row in results:
 3853                 count += 1
 3854                 print_single_rec(row, count)
 3855         except Exception:
 3856             pass
 3857         finally:
 3858             return
 3859 
 3860     while True:
 3861         if new_results or nav == 'n':
 3862             count = 0
 3863 
 3864             if results:
 3865                 total_results = len(results)
 3866                 cur_index = next_index
 3867                 if cur_index < total_results:
 3868                     next_index = min(cur_index + num, total_results)
 3869                     print()
 3870                     for row in results[cur_index:next_index]:
 3871                         count += 1
 3872                         print_single_rec(row, count)
 3873                 else:
 3874                     print('No more results')
 3875             else:
 3876                 print('0 results')
 3877 
 3878         try:
 3879             nav = read_in(PROMPTMSG)
 3880             if not nav:
 3881                 nav = read_in(PROMPTMSG)
 3882                 if not nav:
 3883                     # Quit on double enter
 3884                     break
 3885             nav = nav.strip()
 3886         except EOFError:
 3887             return
 3888 
 3889         # show the next set of results from previous search
 3890         if nav == 'n':
 3891             continue
 3892 
 3893         # search ANY match with new keywords
 3894         if nav.startswith('s '):
 3895             results = obj.searchdb(nav[2:].split(), False, deep)
 3896             new_results = True
 3897             cur_index = next_index = 0
 3898             continue
 3899 
 3900         # search ALL match with new keywords
 3901         if nav.startswith('S '):
 3902             results = obj.searchdb(nav[2:].split(), True, deep)
 3903             new_results = True
 3904             cur_index = next_index = 0
 3905             continue
 3906 
 3907         # regular expressions search with new keywords
 3908         if nav.startswith('r '):
 3909             results = obj.searchdb(nav[2:].split(), True, regex=True)
 3910             new_results = True
 3911             cur_index = next_index = 0
 3912             continue
 3913 
 3914         # tag search with new keywords
 3915         if nav.startswith('t '):
 3916             results = obj.search_by_tag(nav[2:])
 3917             new_results = True
 3918             cur_index = next_index = 0
 3919             continue
 3920 
 3921         # quit with 'q'
 3922         if nav == 'q':
 3923             return
 3924 
 3925         # No new results fetched beyond this point
 3926         new_results = False
 3927 
 3928         # toggle deep search with 'd'
 3929         if nav == 'd':
 3930             deep = not deep
 3931             if deep:
 3932                 print('deep search on')
 3933             else:
 3934                 print('deep search off')
 3935 
 3936             continue
 3937 
 3938         # Toggle GUI browser with 'O'
 3939         if nav == 'O':
 3940             browse.override_text_browser = not browse.override_text_browser
 3941             print('text browser override toggled')
 3942             continue
 3943 
 3944         # Show help with '?'
 3945         if nav == '?':
 3946             ExtendedArgumentParser.prompt_help(sys.stdout)
 3947             continue
 3948 
 3949         # Edit and add or update
 3950         if nav == 'w' or nav.startswith('w '):
 3951             edit_at_prompt(obj, nav, suggest)
 3952             continue
 3953 
 3954         # Append or overwrite tags
 3955         if nav.startswith('g '):
 3956             unique_tags, dic = obj.get_tag_all()
 3957             _count = obj.set_tag(nav[2:], unique_tags)
 3958             if _count == -1:
 3959                 print('Invalid input')
 3960             elif _count == -2:
 3961                 try:
 3962                     tagid_list = nav[2:].split()
 3963                     tagstr = obj.get_tagstr_from_taglist(tagid_list, unique_tags)
 3964                     tagstr = tagstr.strip(DELIM)
 3965                     results = obj.search_by_tag(tagstr)
 3966                     new_results = True
 3967                     cur_index = next_index = 0
 3968                 except Exception:
 3969                     print('Invalid input')
 3970             else:
 3971                 print('%d updated' % _count)
 3972             continue
 3973 
 3974         # Print bookmarks by DB index
 3975         if nav.startswith('p '):
 3976             id_list = nav[2:].split()
 3977             try:
 3978                 for id in id_list:
 3979                     if is_int(id):
 3980                         obj.print_rec(int(id))
 3981                     elif '-' in id:
 3982                         vals = [int(x) for x in id.split('-')]
 3983                         obj.print_rec(0, vals[0], vals[-1], True)
 3984                     else:
 3985                         print('Invalid input')
 3986             except ValueError:
 3987                 print('Invalid input')
 3988             continue
 3989 
 3990         # Browse bookmarks by DB index
 3991         if nav.startswith('o '):
 3992             id_list = nav[2:].split()
 3993             try:
 3994                 for id in id_list:
 3995                     if is_int(id):
 3996                         obj.browse_by_index(int(id))
 3997                     elif '-' in id:
 3998                         vals = [int(x) for x in id.split('-')]
 3999                         obj.browse_by_index(0, vals[0], vals[-1], True)
 4000                     else:
 4001                         print('Invalid input')
 4002             except ValueError:
 4003                 print('Invalid input')
 4004             continue
 4005 
 4006         # Copy URL to clipboard
 4007         if nav.startswith('c ') and nav[2:].isdigit():
 4008             index = int(nav[2:]) - 1
 4009             if index < 0 or index >= count:
 4010                 print('No matching index')
 4011                 continue
 4012             copy_to_clipboard(content=results[index + cur_index][1].encode('utf-8'))
 4013             continue
 4014 
 4015         # open all results and re-prompt with 'a'
 4016         if nav == 'a':
 4017             for index in range(cur_index, next_index):
 4018                 browse(results[index][1])
 4019             continue
 4020 
 4021         # list tags with 't'
 4022         if nav == 't':
 4023             show_taglist(obj)
 4024             continue
 4025 
 4026         toggled = False
 4027         # Open in GUI browser
 4028         if nav.startswith('O '):
 4029             if not browse.override_text_browser:
 4030                 browse.override_text_browser = True
 4031                 toggled = True
 4032             nav = nav[2:]
 4033 
 4034         # iterate over white-space separated indices
 4035         for nav in nav.split():
 4036             if is_int(nav):
 4037                 index = int(nav) - 1
 4038                 if index < 0 or index >= count:
 4039                     print('No matching index %s' % nav)
 4040                     continue
 4041                 browse(results[index + cur_index][1])
 4042             elif '-' in nav:
 4043                 try:
 4044                     vals = [int(x) for x in nav.split('-')]
 4045                     if vals[0] > vals[-1]:
 4046                         vals[0], vals[-1] = vals[-1], vals[0]
 4047 
 4048                     for _id in range(vals[0]-1, vals[-1]):
 4049                         if 0 <= _id < count:
 4050                             browse(results[_id + cur_index][1])
 4051                         else:
 4052                             print('No matching index %d' % (_id + 1))
 4053                 except ValueError:
 4054                     print('Invalid input')
 4055                     break
 4056             else:
 4057                 print('Invalid input')
 4058                 break
 4059 
 4060         if toggled:
 4061             browse.override_text_browser = False
 4062 
 4063 
 4064 def copy_to_clipboard(content):
 4065     """Copy content to clipboard
 4066 
 4067     Parameters
 4068     ----------
 4069     content : str
 4070         Content to be copied to clipboard
 4071     """
 4072 
 4073     # try copying the url to clipboard using native utilities
 4074     copier_params = []
 4075     if sys.platform.startswith(('linux', 'freebsd', 'openbsd')):
 4076         if shutil.which('xsel') is not None:
 4077             copier_params = ['xsel', '-b', '-i']
 4078         elif shutil.which('xclip') is not None:
 4079             copier_params = ['xclip', '-selection', 'clipboard']
 4080         # If we're using Termux (Android) use its 'termux-api'
 4081         # add-on to set device clipboard.
 4082         elif shutil.which('termux-clipboard-set') is not None:
 4083             copier_params = ['termux-clipboard-set']
 4084     elif sys.platform == 'darwin':
 4085         copier_params = ['pbcopy']
 4086     elif sys.platform == 'win32':
 4087         copier_params = ['clip']
 4088 
 4089     if copier_params:
 4090         Popen(copier_params, stdin=PIPE, stdout=DEVNULL, stderr=DEVNULL).communicate(content)
 4091         return
 4092 
 4093     # If native clipboard utilities are absent, try to use terminal multiplexers
 4094     # tmux
 4095     if os.getenv('TMUX_PANE'):
 4096         copier_params = ['tmux', 'set-buffer']
 4097         Popen(
 4098             copier_params + [content],
 4099             stdin=DEVNULL,
 4100             stdout=DEVNULL,
 4101             stderr=DEVNULL
 4102         ).communicate()
 4103         print('URL copied to tmux buffer.')
 4104         return
 4105 
 4106     # GNU Screen paste buffer
 4107     if os.getenv('STY'):
 4108         copier_params = ['screen', '-X', 'readbuf', '-e', 'utf8']
 4109         tmpfd, tmppath = tempfile.mkstemp()
 4110         try:
 4111             with os.fdopen(tmpfd, 'wb') as fp:
 4112                 fp.write(content)
 4113             copier_params.append(tmppath)
 4114             Popen(copier_params, stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL).communicate()
 4115         finally:
 4116             os.unlink(tmppath)
 4117         return
 4118 
 4119     print('Failed to locate suitable clipboard utility')
 4120     return
 4121 
 4122 
 4123 def print_rec_with_filter(records, field_filter=0):
 4124     """Print records filtered by field.
 4125 
 4126     User determines which fields in the records to display
 4127     by using the --format option.
 4128 
 4129     Parameters
 4130     ----------
 4131     records : list or sqlite3.Cursor object
 4132         List of bookmark records to print
 4133     field_filter : int
 4134         Integer indicating which fields to print.
 4135     """
 4136 
 4137     try:
 4138         if field_filter == 0:
 4139             for row in records:
 4140                 print_single_rec(row)
 4141         elif field_filter == 1:
 4142             for row in records:
 4143                 print('%s\t%s' % (row[0], row[1]))
 4144         elif field_filter == 2:
 4145             for row in records:
 4146                 print('%s\t%s\t%s' % (row[0], row[1], row[3][1:-1]))
 4147         elif field_filter == 3:
 4148             for row in records:
 4149                 print('%s\t%s' % (row[0], row[2]))
 4150         elif field_filter == 4:
 4151             for row in records:
 4152                 print('%s\t%s\t%s\t%s' % (row[0], row[1], row[2], row[3][1:-1]))
 4153         elif field_filter == 5:
 4154             for row in records:
 4155                 print('%s\t%s\t%s' % (row[0], row[2], row[3][1:-1]))
 4156         elif field_filter == 10:
 4157             for row in records:
 4158                 print(row[1])
 4159         elif field_filter == 20:
 4160             for row in records:
 4161                 print('%s\t%s' % (row[1], row[3][1:-1]))
 4162         elif field_filter == 30:
 4163             for row in records:
 4164                 print(row[2])
 4165         elif field_filter == 40:
 4166             for row in records:
 4167                 print('%s\t%s\t%s' % (row[1], row[2], row[3][1:-1]))
 4168         elif field_filter == 50:
 4169             for row in records:
 4170                 print('%s\t%s' % (row[2], row[3][1:-1]))
 4171     except BrokenPipeError:
 4172         sys.stdout = os.fdopen(1)
 4173         sys.exit(1)
 4174 
 4175 
 4176 def print_single_rec(row, idx=0):  # NOQA
 4177     """Print a single DB record.
 4178 
 4179     Handles both search results and individual record.
 4180 
 4181     Parameters
 4182     ----------
 4183     row : tuple
 4184         Tuple representing bookmark record data.
 4185     idx : int, optional
 4186         Search result index. If 0, print with DB index.
 4187         Default is 0.
 4188     """
 4189 
 4190     str_list = []
 4191 
 4192     # Start with index and title
 4193     if idx != 0:
 4194         id_title_res = ID_STR % (idx, row[2] if row[2] else 'Untitled', row[0])
 4195     else:
 4196         id_title_res = ID_DB_STR % (row[0], row[2] if row[2] else 'Untitled')
 4197         # Indicate if record is immutable
 4198         if row[5] & 1:
 4199             id_title_res = MUTE_STR % (id_title_res)
 4200         else:
 4201             id_title_res += '\n'
 4202 
 4203     str_list.append(id_title_res)
 4204     str_list.append(URL_STR % (row[1]))
 4205     if row[4]:
 4206         str_list.append(DESC_STR % (row[4]))
 4207     if row[3] != DELIM:
 4208         str_list.append(TAG_STR % (row[3][1:-1]))
 4209 
 4210     try:
 4211         print(''.join(str_list))
 4212     except BrokenPipeError:
 4213         sys.stdout = os.fdopen(1)
 4214         sys.exit(1)
 4215 
 4216 def write_string_to_file(content: str, filepath: str):
 4217     """Writes given content to file
 4218 
 4219     Parameters
 4220     ----------
 4221     content : str
 4222     filepath : str
 4223 
 4224     Returns
 4225     -------
 4226     None
 4227     """
 4228     try:
 4229         with open(filepath, 'w', encoding='utf-8') as f:
 4230             f.write(content)
 4231     except Exception as e:
 4232         LOGERR(e)
 4233 
 4234 def format_json(resultset, single_record=False, field_filter=0):
 4235     """Return results in JSON format.
 4236 
 4237     Parameters
 4238     ----------
 4239     resultset : list
 4240         Search results from DB query.
 4241     single_record : bool, optional
 4242         If True, indicates only one record. Default is False.
 4243 
 4244     Returns
 4245     -------
 4246     json
 4247         Record(s) in JSON format.
 4248     """
 4249 
 4250     if single_record:
 4251         marks = {}
 4252         for row in resultset:
 4253             if field_filter == 1:
 4254                 marks['uri'] = row[1]
 4255             elif field_filter == 2:
 4256                 marks['uri'] = row[1]
 4257                 marks['tags'] = row[3][1:-1]
 4258             elif field_filter == 3:
 4259                 marks['title'] = row[2]
 4260             elif field_filter == 4:
 4261                 marks['uri'] = row[1]
 4262                 marks['tags'] = row[3][1:-1]
 4263                 marks['title'] = row[2]
 4264             else:
 4265                 marks['index'] = row[0]
 4266                 marks['uri'] = row[1]
 4267                 marks['title'] = row[2]
 4268                 marks['description'] = row[4]
 4269                 marks['tags'] = row[3][1:-1]
 4270     else:
 4271         marks = []
 4272         for row in resultset:
 4273             if field_filter == 1:
 4274                 record = {'uri': row[1]}
 4275             elif field_filter == 2:
 4276                 record = {'uri': row[1], 'tags': row[3][1:-1]}
 4277             elif field_filter == 3:
 4278                 record = {'title': row[2]}
 4279             elif field_filter == 4:
 4280                 record = {'uri': row[1], 'title': row[2], 'tags': row[3][1:-1]}
 4281             else:
 4282                 record = {
 4283                     'index': row[0],
 4284                     'uri': row[1],
 4285                     'title': row[2],
 4286                     'description': row[4],
 4287                     'tags': row[3][1:-1]
 4288                 }
 4289 
 4290             marks.append(record)
 4291 
 4292     return json.dumps(marks, sort_keys=True, indent=4)
 4293 
 4294 
 4295 def is_int(string):
 4296     """Check if a string is a digit.
 4297 
 4298     string : str
 4299         Input string to check.
 4300 
 4301     Returns
 4302     -------
 4303     bool
 4304         True on success, False on exception.
 4305     """
 4306 
 4307     try:
 4308         int(string)
 4309         return True
 4310     except Exception:
 4311         return False
 4312 
 4313 
 4314 def browse(url):
 4315     """Duplicate stdin, stdout and open URL in default browser.
 4316 
 4317     .. Note:: Duplicates stdin and stdout in order to
 4318               suppress showing errors on the terminal.
 4319 
 4320     Parameters
 4321     ----------
 4322     url : str
 4323         URL to open in browser.
 4324 
 4325     Attributes
 4326     ----------
 4327     suppress_browser_output : bool
 4328         True if a text based browser is detected.
 4329         Must be initialized (as applicable) to use the API.
 4330     override_text_browser : bool
 4331         If True, tries to open links in a GUI based browser.
 4332     """
 4333 
 4334     if not parse_url(url).scheme:
 4335         # Prefix with 'http://' if no scheme
 4336         # Otherwise, opening in browser fails anyway
 4337         # We expect http to https redirection
 4338         # will happen for https-only websites
 4339         LOGERR('Scheme missing in URI, trying http')
 4340         url = 'http://' + url
 4341 
 4342     browser = webbrowser.get()
 4343     if browse.override_text_browser:
 4344         browser_output = browse.suppress_browser_output
 4345         for name in [b for b in webbrowser._tryorder if b not in TEXT_BROWSERS]:
 4346             browser = webbrowser.get(name)
 4347             LOGDBG(browser)
 4348 
 4349             # Found a GUI browser, suppress browser output
 4350             browse.suppress_browser_output = True
 4351             break
 4352 
 4353     if browse.suppress_browser_output:
 4354         _stderr = os.dup(2)
 4355         os.close(2)
 4356         _stdout = os.dup(1)
 4357         if "microsoft" not in platform.uname()[3].lower():
 4358             os.close(1)
 4359         fd = os.open(os.devnull, os.O_RDWR)
 4360         os.dup2(fd, 2)
 4361         os.dup2(fd, 1)
 4362     try:
 4363         if sys.platform != 'win32':
 4364             browser.open(url, new=2)
 4365         else:
 4366             # On Windows, the webbrowser module does not fork.
 4367             # Use threads instead.
 4368             def browserthread():
 4369                 webbrowser.open(url, new=2)
 4370 
 4371             t = threading.Thread(target=browserthread)
 4372             t.start()
 4373     except Exception as e:
 4374         LOGERR('browse(): %s', e)
 4375     finally:
 4376         if browse.suppress_browser_output:
 4377             os.close(fd)
 4378             os.dup2(_stderr, 2)
 4379             os.dup2(_stdout, 1)
 4380 
 4381     if browse.override_text_browser:
 4382         browse.suppress_browser_output = browser_output
 4383 
 4384 
 4385 def check_upstream_release():
 4386     """Check and report the latest upstream release version."""
 4387 
 4388     global MYPROXY
 4389 
 4390     if MYPROXY is None:
 4391         gen_headers()
 4392 
 4393     ca_certs = os.getenv('BUKU_CA_CERTS', default=certifi.where())
 4394     if MYPROXY:
 4395         manager = urllib3.ProxyManager(
 4396             MYPROXY,
 4397             num_pools=1,
 4398             headers=MYHEADERS,
 4399             cert_reqs='CERT_REQUIRED',
 4400             ca_certs=ca_certs
 4401         )
 4402     else:
 4403         manager = urllib3.PoolManager(num_pools=1,
 4404                                       headers={'User-Agent': USER_AGENT},
 4405                                       cert_reqs='CERT_REQUIRED',
 4406                                       ca_certs=ca_certs)
 4407 
 4408     try:
 4409         r = manager.request(
 4410             'GET',
 4411             'https://api.github.com/repos/jarun/buku/releases?per_page=1',
 4412             headers={'User-Agent': USER_AGENT}
 4413         )
 4414     except Exception as e:
 4415         LOGERR(e)
 4416         return
 4417 
 4418     if r.status == 200:
 4419         latest = json.loads(r.data.decode(errors='replace'))[0]['tag_name']
 4420         if latest == 'v' + __version__:
 4421             print('This is the latest release')
 4422         else:
 4423             print('Latest upstream release is %s' % latest)
 4424     else:
 4425         LOGERR('[%s] %s', r.status, r.reason)
 4426 
 4427     manager.clear()
 4428 
 4429 
 4430 def regexp(expr, item):
 4431     """Perform a regular expression search.
 4432 
 4433     Parameters
 4434     ----------
 4435     expr : regex
 4436         Regular expression to search for.
 4437     item : str
 4438         Item on which to perform regex search.
 4439 
 4440     Returns
 4441     -------
 4442     bool
 4443         True if result of search is not None, else False.
 4444     """
 4445 
 4446     if expr is None or item is None:
 4447         LOGDBG('expr: [%s], item: [%s]', expr, item)
 4448         return False
 4449 
 4450     return re.search(expr, item, re.IGNORECASE) is not None
 4451 
 4452 
 4453 def delim_wrap(token):
 4454     """Returns token string wrapped in delimiters.
 4455 
 4456     Parameters
 4457     ----------
 4458     token : str
 4459         String item to wrap with DELIM.
 4460 
 4461     Returns
 4462     -------
 4463     str
 4464         Token string wrapped by DELIM.
 4465     """
 4466 
 4467     if token is None or token.strip() == '':
 4468         return DELIM
 4469 
 4470     if token[0] != DELIM:
 4471         token = DELIM + token
 4472 
 4473     if token[-1] != DELIM:
 4474         token = token + DELIM
 4475 
 4476     return token
 4477 
 4478 
 4479 def read_in(msg):
 4480     """A wrapper to handle input() with interrupts disabled.
 4481 
 4482     Parameters
 4483     ----------
 4484     msg : str
 4485         String to pass to to input().
 4486     """
 4487 
 4488     disable_sigint_handler()
 4489     message = None
 4490     try:
 4491         message = input(msg)
 4492     except KeyboardInterrupt:
 4493         print('Interrupted.')
 4494 
 4495     enable_sigint_handler()
 4496     return message
 4497 
 4498 
 4499 def sigint_handler(signum, frame):
 4500     """Custom SIGINT handler.
 4501 
 4502     .. Note:: Neither signum nor frame are used in
 4503               this custom handler. However, they are
 4504               required parameters for signal handlers.
 4505 
 4506     Parameters
 4507     ----------
 4508     signum : int
 4509         Signal number.
 4510     frame : frame object or None.
 4511     """
 4512 
 4513     global INTERRUPTED
 4514 
 4515     INTERRUPTED = True
 4516     print('\nInterrupted.', file=sys.stderr)
 4517 
 4518     # Do a hard exit from here
 4519     os._exit(1)
 4520 
 4521 DEFAULT_HANDLER = signal.signal(signal.SIGINT, sigint_handler)
 4522 
 4523 
 4524 def disable_sigint_handler():
 4525     """Disable signint handler."""
 4526     signal.signal(signal.SIGINT, DEFAULT_HANDLER)
 4527 
 4528 
 4529 def enable_sigint_handler():
 4530     """Enable sigint handler."""
 4531     signal.signal(signal.SIGINT, sigint_handler)
 4532 
 4533 # ---------------------
 4534 # Editor mode functions
 4535 # ---------------------
 4536 
 4537 
 4538 def get_system_editor():
 4539     """Returns default system editor is $EDITOR is set."""
 4540 
 4541     return os.environ.get('EDITOR', 'none')
 4542 
 4543 
 4544 def is_editor_valid(editor):
 4545     """Check if the editor string is valid.
 4546 
 4547     Parameters
 4548     ----------
 4549     editor : str
 4550         Editor string.
 4551 
 4552     Returns
 4553     -------
 4554     bool
 4555         True if string is valid, else False.
 4556     """
 4557 
 4558     if editor == 'none':
 4559         LOGERR('EDITOR is not set')
 4560         return False
 4561 
 4562     if editor == '0':
 4563         LOGERR('Cannot edit index 0')
 4564         return False
 4565 
 4566     return True
 4567 
 4568 
 4569 def to_temp_file_content(url, title_in, tags_in, desc):
 4570     """Generate temporary file content string.
 4571 
 4572     Parameters
 4573     ----------
 4574     url : str
 4575         URL to open.
 4576     title_in : str
 4577         Title to add manually.
 4578     tags_in : str
 4579         Comma-separated tags to add manually.
 4580     desc : str
 4581         String description.
 4582 
 4583     Returns
 4584     -------
 4585     str
 4586         Lines as newline separated string.
 4587 
 4588     Raises
 4589     ------
 4590     AttributeError
 4591         when tags_in is None.
 4592     """
 4593 
 4594     strings = [('# Lines beginning with "#" will be stripped.\n'
 4595                 '# Add URL in next line (single line).'), ]
 4596 
 4597     # URL
 4598     if url is not None:
 4599         strings += (url,)
 4600 
 4601     # TITLE
 4602     strings += (('# Add TITLE in next line (single line). '
 4603                  'Leave blank to web fetch, "-" for no title.'),)
 4604     if title_in is None:
 4605         title_in = ''
 4606     elif title_in == '':
 4607         title_in = '-'
 4608     strings += (title_in,)
 4609 
 4610     # TAGS
 4611     strings += ('# Add comma-separated TAGS in next line (single line).',)
 4612     strings += (tags_in.strip(DELIM),) if not None else ''
 4613 
 4614     # DESC
 4615     strings += ('# Add COMMENTS in next line(s). Leave blank to web fetch, "-" for no comments.',)
 4616     if desc is None:
 4617         strings += ('\n',)
 4618     elif desc == '':
 4619         strings += ('-',)
 4620     else:
 4621         strings += (desc,)
 4622     return '\n'.join(strings)
 4623 
 4624 
 4625 def parse_temp_file_content(content):
 4626     """Parse and return temporary file content.
 4627 
 4628     Parameters
 4629     ----------
 4630     content : str
 4631         String of content.
 4632 
 4633     Returns
 4634     -------
 4635     tuple
 4636         (url, title, tags, comments)
 4637 
 4638         url: URL to open
 4639         title: string title to add manually
 4640         tags: string of comma-separated tags to add manually
 4641         comments: string description
 4642     """
 4643 
 4644     content = content.split('\n')
 4645     content = [c for c in content if not c or c[0] != '#']
 4646     if not content or content[0].strip() == '':
 4647         print('Edit aborted')
 4648         return None
 4649 
 4650     url = content[0]
 4651     title = None
 4652     if len(content) > 1:
 4653         title = content[1]
 4654 
 4655     if title == '':
 4656         title = None
 4657     elif title == '-':
 4658         title = ''
 4659 
 4660     tags = DELIM
 4661     if len(content) > 2:
 4662         tags = parse_tags([content[2]])
 4663 
 4664     comments = []
 4665     if len(content) > 3:
 4666         comments = list(content[3:])
 4667         # need to remove all empty line that are at the end
 4668         # and not those in the middle of the text
 4669         for i in range(len(comments) - 1, -1, -1):
 4670             if comments[i].strip() != '':
 4671                 break
 4672 
 4673         if i == -1:
 4674             comments = []
 4675         else:
 4676             comments = comments[0:i+1]
 4677 
 4678     comments = '\n'.join(comments)
 4679     if comments == '':
 4680         comments = None
 4681     elif comments == '-':
 4682         comments = ''
 4683 
 4684     return url, title, tags, comments
 4685 
 4686 
 4687 def edit_rec(editor, url, title_in, tags_in, desc):
 4688     """Edit a bookmark record.
 4689 
 4690     Parameters
 4691     ----------
 4692     editor : str
 4693         Editor to open.
 4694     URL : str
 4695         URL to open.
 4696     title_in : str
 4697         Title to add manually.
 4698     tags_in : str
 4699         Comma-separated tags to add manually.
 4700     desc : str
 4701         Bookmark description.
 4702 
 4703     Returns
 4704     -------
 4705     tuple
 4706         Parsed results from parse_temp_file_content().
 4707     """
 4708 
 4709     temp_file_content = to_temp_file_content(url, title_in, tags_in, desc)
 4710 
 4711     fd, tmpfile = tempfile.mkstemp(prefix='buku-edit-')
 4712     os.close(fd)
 4713 
 4714     try:
 4715         with open(tmpfile, 'w+', encoding='utf-8') as fp:
 4716             fp.write(temp_file_content)
 4717             fp.flush()
 4718             LOGDBG('Edited content written to %s', tmpfile)
 4719 
 4720         cmd = editor.split(' ')
 4721         cmd += (tmpfile,)
 4722         subprocess.call(cmd)
 4723 
 4724         with open(tmpfile, 'r', encoding='utf-8') as f:
 4725             content = f.read()
 4726 
 4727         os.remove(tmpfile)
 4728     except FileNotFoundError:
 4729         if os.path.exists(tmpfile):
 4730             os.remove(tmpfile)
 4731             LOGERR('Cannot open editor')
 4732         else:
 4733             LOGERR('Cannot open tempfile')
 4734         return None
 4735 
 4736     parsed_content = parse_temp_file_content(content)
 4737     return parsed_content
 4738 
 4739 
 4740 def setup_logger(LOGGER):
 4741     """Setup logger with color.
 4742 
 4743     Parameters
 4744     ----------
 4745     LOGGER : logger object
 4746         Logger to colorize.
 4747     """
 4748 
 4749     def decorate_emit(fn):
 4750         def new(*args):
 4751             levelno = args[0].levelno
 4752 
 4753             if levelno == logging.DEBUG:
 4754                 color = '\x1b[35m'
 4755             elif levelno == logging.ERROR:
 4756                 color = '\x1b[31m'
 4757             elif levelno == logging.WARNING:
 4758                 color = '\x1b[33m'
 4759             elif levelno == logging.INFO:
 4760                 color = '\x1b[32m'
 4761             elif levelno == logging.CRITICAL:
 4762                 color = '\x1b[31m'
 4763             else:
 4764                 color = '\x1b[0m'
 4765 
 4766             args[0].msg = '{}[{}]\x1b[0m {}'.format(color, args[0].levelname, args[0].msg)
 4767             return fn(*args)
 4768         return new
 4769 
 4770     sh = logging.StreamHandler()
 4771     sh.emit = decorate_emit(sh.emit)
 4772     LOGGER.addHandler(sh)
 4773 
 4774 
 4775 def piped_input(argv, pipeargs=None):
 4776     """Handle piped input.
 4777 
 4778     Parameters
 4779     ----------
 4780     pipeargs : str
 4781     """
 4782     if not sys.stdin.isatty():
 4783         pipeargs += argv
 4784         print('waiting for input')
 4785         for s in sys.stdin:
 4786             pipeargs += s.split()
 4787 
 4788 
 4789 def setcolors(args):
 4790     """Get colors from user and separate into 'result' list for use in arg.colors.
 4791 
 4792     Parameters
 4793     ----------
 4794     args : str
 4795         Color string.
 4796     """
 4797     Colors = collections.namedtuple('Colors', ' ID_srch, ID_STR, URL_STR, DESC_STR, TAG_STR')
 4798     colors = Colors(*[COLORMAP[c] for c in args])
 4799     id_col = colors.ID_srch
 4800     id_str_col = colors.ID_STR
 4801     url_col = colors.URL_STR
 4802     desc_col = colors.DESC_STR
 4803     tag_col = colors.TAG_STR
 4804     result = [id_col, id_str_col, url_col, desc_col, tag_col]
 4805     return result
 4806 
 4807 # main starts here
 4808 def main():
 4809     """Main."""
 4810     global ID_STR, ID_DB_STR, MUTE_STR, URL_STR, DESC_STR, TAG_STR, PROMPTMSG
 4811 
 4812     title_in = None
 4813     tags_in = None
 4814     desc_in = None
 4815     pipeargs = []
 4816     colorstr_env = os.getenv('BUKU_COLORS')
 4817 
 4818     try:
 4819         piped_input(sys.argv, pipeargs)
 4820     except KeyboardInterrupt:
 4821         pass
 4822 
 4823     # If piped input, set argument vector
 4824     if pipeargs:
 4825         sys.argv = pipeargs
 4826 
 4827     # Setup custom argument parser
 4828     argparser = ExtendedArgumentParser(
 4829         description='''Bookmark manager like a text-based mini-web.
 4830 
 4831 POSITIONAL ARGUMENTS:
 4832       KEYWORD              search keywords''',
 4833         formatter_class=argparse.RawTextHelpFormatter,
 4834         usage='''buku [OPTIONS] [KEYWORD [KEYWORD ...]]''',
 4835         add_help=False
 4836     )
 4837     hide = argparse.SUPPRESS
 4838 
 4839     argparser.add_argument('keywords', nargs='*', metavar='KEYWORD', help=hide)
 4840 
 4841     # ---------------------
 4842     # GENERAL OPTIONS GROUP
 4843     # ---------------------
 4844 
 4845     general_grp = argparser.add_argument_group(
 4846         title='GENERAL OPTIONS',
 4847         description='''    -a, --add URL [tag, ...]
 4848                          bookmark URL with comma-separated tags
 4849     -u, --update [...]   update fields of an existing bookmark
 4850                          accepts indices and ranges
 4851                          refresh title and desc if no edit options
 4852                          if no arguments:
 4853                          - update results when used with search
 4854                          - otherwise refresh all titles and desc
 4855     -w, --write [editor|index]
 4856                          open editor to edit a fresh bookmark
 4857                          edit last bookmark, if index=-1
 4858                          to specify index, EDITOR must be set
 4859     -d, --delete [...]   remove bookmarks from DB
 4860                          accepts indices or a single range
 4861                          if no arguments:
 4862                          - delete results when used with search
 4863                          - otherwise delete all bookmarks
 4864     -h, --help           show this information and exit
 4865     -v, --version        show the program version and exit''')
 4866     addarg = general_grp.add_argument
 4867     addarg('-a', '--add', nargs='+', help=hide)
 4868     addarg('-u', '--update', nargs='*', help=hide)
 4869     addarg('-w', '--write', nargs='?', const=get_system_editor(), help=hide)
 4870     addarg('-d', '--delete', nargs='*', help=hide)
 4871     addarg('-h', '--help', action='store_true', help=hide)
 4872     addarg('-v', '--version', action='version', version=__version__, help=hide)
 4873 
 4874     # ------------------
 4875     # EDIT OPTIONS GROUP
 4876     # ------------------
 4877 
 4878     edit_grp = argparser.add_argument_group(
 4879         title='EDIT OPTIONS',
 4880         description='''    --url keyword        bookmark link
 4881     --tag [+|-] [...]    comma-separated tags
 4882                          clear bookmark tagset, if no arguments
 4883                          '+' appends to, '-' removes from tagset
 4884     --title [...]        bookmark title; if no arguments:
 4885                          -a: do not set title, -u: clear title
 4886     -c, --comment [...]  notes or description of the bookmark
 4887                          clears description, if no arguments
 4888     --immutable N        disable web-fetch during auto-refresh
 4889                          N=0: mutable (default), N=1: immutable''')
 4890     addarg = edit_grp.add_argument
 4891     addarg('--url', nargs=1, help=hide)
 4892     addarg('--tag', nargs='*', help=hide)
 4893     addarg('--title', nargs='*', help=hide)
 4894     addarg('-c', '--comment', nargs='*', help=hide)
 4895     addarg('--immutable', type=int, default=-1, choices={0, 1}, help=hide)
 4896 
 4897     # --------------------
 4898     # SEARCH OPTIONS GROUP
 4899     # --------------------
 4900 
 4901     search_grp = argparser.add_argument_group(
 4902         title='SEARCH OPTIONS',
 4903         description='''    -s, --sany [...]     find records with ANY matching keyword
 4904                          this is the default search option
 4905     -S, --sall [...]     find records matching ALL the keywords
 4906                          special keywords -
 4907                          "blank": entries with empty title/tag
 4908                          "immutable": entries with locked title
 4909     --deep               match substrings ('pen' matches 'opens')
 4910     -r, --sreg expr      run a regex search
 4911     -t, --stag [tag [,|+] ...] [- tag, ...]
 4912                          search bookmarks by tags
 4913                          use ',' to find entries matching ANY tag
 4914                          use '+' to find entries matching ALL tags
 4915                          excludes entries with tags after ' - '
 4916                          list all tags, if no search keywords
 4917     -x, --exclude [...]  omit records matching specified keywords''')
 4918     addarg = search_grp.add_argument
 4919     addarg('-s', '--sany', nargs='*', help=hide)
 4920     addarg('-S', '--sall', nargs='*', help=hide)
 4921     addarg('-r', '--sreg', nargs='*', help=hide)
 4922     addarg('--deep', action='store_true', help=hide)
 4923     addarg('-t', '--stag', nargs='*', help=hide)
 4924     addarg('-x', '--exclude', nargs='*', help=hide)
 4925 
 4926     # ------------------------
 4927     # ENCRYPTION OPTIONS GROUP
 4928     # ------------------------
 4929 
 4930     crypto_grp = argparser.add_argument_group(
 4931         title='ENCRYPTION OPTIONS',
 4932         description='''    -l, --lock [N]       encrypt DB in N (default 8) # iterations
 4933     -k, --unlock [N]     decrypt DB in N (default 8) # iterations''')
 4934     addarg = crypto_grp.add_argument
 4935     addarg('-k', '--unlock', nargs='?', type=int, const=8, help=hide)
 4936     addarg('-l', '--lock', nargs='?', type=int, const=8, help=hide)
 4937 
 4938     # ----------------
 4939     # POWER TOYS GROUP
 4940     # ----------------
 4941 
 4942     power_grp = argparser.add_argument_group(
 4943         title='POWER TOYS',
 4944         description='''    --ai                 auto-import from Firefox/Chrome/Chromium
 4945     -e, --export file    export bookmarks to Firefox format HTML
 4946                          export Markdown, if file ends with '.md'
 4947                          format: [title](url) <!-- TAGS -->
 4948                          export Orgfile, if file ends with '.org'
 4949                          format: *[[url][title]] :tags:
 4950                          export buku DB, if file ends with '.db'
 4951                          combines with search results, if opted
 4952     -i, --import file    import bookmarks based on file extension
 4953                          supports 'html', 'json', 'md', 'org', 'db'
 4954     -p, --print [...]    show record details by indices, ranges
 4955                          print all bookmarks, if no arguments
 4956                          -n shows the last n results (like tail)
 4957     -f, --format N       limit fields in -p or JSON search output
 4958                          N=1: URL; N=2: URL, tag; N=3: title;
 4959                          N=4: URL, title, tag; N=5: title, tag;
 4960                          N0 (10, 20, 30, 40, 50) omits DB index
 4961     -j, --json [file]    JSON formatted output for -p and search.
 4962                          prints to stdout if no arguments are provided.
 4963                          otherwise writes to given file
 4964     --colors COLORS      set output colors in five-letter string
 4965     --nc                 disable color output
 4966     -n, --count N        show N results per page (default 10)
 4967     --np                 do not show the prompt, run and exit
 4968     -o, --open [...]     browse bookmarks by indices and ranges
 4969                          open a random bookmark, if no arguments
 4970     --oa                 browse all search results immediately
 4971     --replace old new    replace old tag with new tag everywhere
 4972                          delete old tag, if new tag not specified
 4973     --shorten index|URL  fetch shortened url from tny.im service
 4974     --expand index|URL   expand a tny.im shortened url
 4975     --cached index|URL   browse a cached page from Wayback Machine
 4976     --suggest            show similar tags when adding bookmarks
 4977     --tacit              reduce verbosity
 4978     --threads N          max network connections in full refresh
 4979                          default N=4, min N=1, max N=10
 4980     -V                   check latest upstream version available
 4981     -z, --debug          show debug information and verbose logs''')
 4982     addarg = power_grp.add_argument
 4983     addarg('--ai', action='store_true', help=hide)
 4984     addarg('-e', '--export', nargs=1, help=hide)
 4985     addarg('-i', '--import', nargs=1, dest='importfile', help=hide)
 4986     addarg('-p', '--print', nargs='*', help=hide)
 4987     addarg('-f', '--format', type=int, default=0, choices={1, 2, 3, 4, 5, 10, 20, 30, 40, 50}, help=hide)
 4988     addarg('-j', '--json', nargs='?', default=None, const='', help=hide)
 4989     addarg('--colors', dest='colorstr', type=argparser.is_colorstr, metavar='COLORS', help=hide)
 4990     addarg('--nc', action='store_true', help=hide)
 4991     addarg('-n', '--count', nargs='?', const=10, type=int, default=0, help=hide)
 4992     addarg('--np', action='store_true', help=hide)
 4993     addarg('-o', '--open', nargs='*', help=hide)
 4994     addarg('--oa', action='store_true', help=hide)
 4995     addarg('--replace', nargs='+', help=hide)
 4996     addarg('--shorten', nargs=1, help=hide)
 4997     addarg('--expand', nargs=1, help=hide)
 4998     addarg('--cached', nargs=1, help=hide)
 4999     addarg('--suggest', action='store_true', help=hide)
 5000     addarg('--tacit', action='store_true', help=hide)
 5001     addarg('--threads', type=int, default=4, choices=range(1, 11), help=hide)
 5002     addarg('-V', dest='upstream', action='store_true', help=hide)
 5003     addarg('-z', '--debug', action='store_true', help=hide)
 5004     # Undocumented APIs
 5005     # Fix uppercase tags allowed in releases before v2.7
 5006     addarg('--fixtags', action='store_true', help=hide)
 5007     # App-use only, not for manual usage
 5008     addarg('--db', nargs=1, help=hide)
 5009 
 5010     # Parse the arguments
 5011     args = argparser.parse_args()
 5012 
 5013     # Show help and exit if help requested
 5014     if args.help:
 5015         argparser.print_help(sys.stdout)
 5016         sys.exit(0)
 5017 
 5018     # By default, buku uses ANSI colors. As Windows does not really use them,
 5019     # we'd better check for known working console emulators first. Currently,
 5020     # only ConEmu is supported. If the user does not use ConEmu, colors are
 5021     # disabled unless --colors or %BUKU_COLORS% is specified.
 5022     if sys.platform == 'win32' and os.environ.get('ConemuDir') is None:
 5023         if args.colorstr is None and colorstr_env is not None:
 5024             args.nc = True
 5025 
 5026     # Handle color output preference
 5027     if args.nc:
 5028         logging.basicConfig(format='[%(levelname)s] %(message)s')
 5029     else:
 5030         # Set colors
 5031         if colorstr_env is not None:
 5032             # Someone set BUKU_COLORS.
 5033             colorstr = colorstr_env
 5034         elif args.colorstr is not None:
 5035             colorstr = args.colorstr
 5036         else:
 5037             colorstr = 'oKlxm'
 5038 
 5039         ID = setcolors(colorstr)[0] + '%d. ' + COLORMAP['x']
 5040         ID_DB_dim = COLORMAP['z'] + '[%s]\n' + COLORMAP['x']
 5041         ID_STR = ID + setcolors(colorstr)[1] + '%s ' + COLORMAP['x'] + ID_DB_dim
 5042         ID_DB_STR = ID + setcolors(colorstr)[1] + '%s' + COLORMAP['x']
 5043         MUTE_STR = '%s \x1b[2m(L)\x1b[0m\n'
 5044         URL_STR = COLORMAP['j'] + '   > ' + setcolors(colorstr)[2] + '%s\n' + COLORMAP['x']
 5045         DESC_STR = COLORMAP['j'] + '   + ' + setcolors(colorstr)[3] + '%s\n' + COLORMAP['x']
 5046         TAG_STR = COLORMAP['j'] + '   # ' + setcolors(colorstr)[4] + '%s\n' + COLORMAP['x']
 5047 
 5048         # Enable color in logs
 5049         setup_logger(LOGGER)
 5050 
 5051         # Enable prompt with reverse video
 5052         PROMPTMSG = '\001\x1b[7\002mbuku (? for help)\001\x1b[0m\002 '
 5053 
 5054     # Enable browser output in case of a text based browser
 5055     if os.getenv('BROWSER') in TEXT_BROWSERS:
 5056         browse.suppress_browser_output = False
 5057     else:
 5058         browse.suppress_browser_output = True
 5059 
 5060     # Overriding text browsers is disabled by default
 5061     browse.override_text_browser = False
 5062 
 5063     # Fallback to prompt if no arguments
 5064     if len(sys.argv) == 1:
 5065         bdb = BukuDb()
 5066         prompt(bdb, None)
 5067         bdb.close_quit(0)
 5068 
 5069     # Set up debugging
 5070     if args.debug:
 5071         LOGGER.setLevel(logging.DEBUG)
 5072         LOGDBG('buku v%s', __version__)
 5073         LOGDBG('Python v%s', ('%d.%d.%d' % sys.version_info[:3]))
 5074     else:
 5075         logging.disable(logging.WARNING)
 5076         urllib3.disable_warnings()
 5077 
 5078     # Handle encrypt/decrypt options at top priority
 5079     if args.lock is not None:
 5080         BukuCrypt.encrypt_file(args.lock)
 5081     elif args.unlock is not None:
 5082         BukuCrypt.decrypt_file(args.unlock)
 5083 
 5084     # Set up title
 5085     if args.title is not None:
 5086         if args.title:
 5087             title_in = ' '.join(args.title)
 5088         else:
 5089             title_in = ''
 5090 
 5091     # Set up tags
 5092     if args.tag is not None:
 5093         if args.tag:
 5094             tags_in = args.tag
 5095         else:
 5096             tags_in = [DELIM, ]
 5097 
 5098     # Set up comment
 5099     if args.comment is not None:
 5100         if args.comment:
 5101             desc_in = ' '.join(args.comment)
 5102         else:
 5103             desc_in = ''
 5104 
 5105     # Initialize the database and get handles, set verbose by default
 5106     bdb = BukuDb(
 5107         args.json,
 5108         args.format,
 5109         not args.tacit,
 5110         dbfile=args.db[0] if args.db is not None else None,
 5111         colorize=not args.nc
 5112     )
 5113 
 5114     # Editor mode
 5115     if args.write is not None:
 5116         if not is_editor_valid(args.write):
 5117             bdb.close_quit(1)
 5118 
 5119         if is_int(args.write):
 5120             if not bdb.edit_update_rec(int(args.write), args.immutable):
 5121                 bdb.close_quit(1)
 5122         elif args.add is None:
 5123             # Edit and add a new bookmark
 5124             # Parse tags into a comma-separated string
 5125             if tags_in:
 5126                 if tags_in[0] == '+':
 5127                     tags = '+' + parse_tags(tags_in[1:])
 5128                 elif tags_in[0] == '-':
 5129                     tags = '-' + parse_tags(tags_in[1:])
 5130                 else:
 5131                     tags = parse_tags(tags_in)
 5132             else:
 5133                 tags = DELIM
 5134 
 5135             result = edit_rec(args.write, '', title_in, tags, desc_in)
 5136             if result is not None:
 5137                 url, title_in, tags, desc_in = result
 5138                 if args.suggest:
 5139                     tags = bdb.suggest_similar_tag(tags)
 5140                 bdb.add_rec(url, title_in, tags, desc_in, args.immutable)
 5141 
 5142     # Add record
 5143     if args.add is not None:
 5144         if args.url is not None and args.update is None:
 5145             LOGERR('Bookmark a single URL at a time')
 5146             bdb.close_quit(1)
 5147 
 5148         # Parse tags into a comma-separated string
 5149         tags = DELIM
 5150         keywords = args.add
 5151         if tags_in is not None:
 5152             if tags_in[0] == '+':
 5153                 if len(tags_in) > 1:
 5154                     # The case: buku -a url tag1, tag2 --tag + tag3, tag4
 5155                     tags_in = tags_in[1:]
 5156                     # In case of add, args.add may have URL followed by tags
 5157                     # Add delimiter as url+tags may not end with one
 5158                     keywords = args.add + [DELIM] + tags_in
 5159             else:
 5160                 keywords = args.add + [DELIM] + tags_in
 5161 
 5162         if len(keywords) > 1:  # args.add is URL followed by optional tags
 5163             tags = parse_tags(keywords[1:])
 5164 
 5165         url = args.add[0]
 5166         edit_aborted = False
 5167 
 5168         if args.write and not is_int(args.write):
 5169             result = edit_rec(args.write, url, title_in, tags, desc_in)
 5170             if result is not None:
 5171                 url, title_in, tags, desc_in = result
 5172             else:
 5173                 edit_aborted = True
 5174 
 5175         if edit_aborted is False:
 5176             if args.suggest:
 5177                 tags = bdb.suggest_similar_tag(tags)
 5178             bdb.add_rec(url, title_in, tags, desc_in, args.immutable)
 5179 
 5180     # Search record
 5181     search_results = None
 5182     search_opted = True
 5183     tags_search = bool(args.stag is not None and len(args.stag))
 5184     exclude_results = bool(args.exclude is not None and len(args.exclude))
 5185 
 5186     if args.sany is not None:
 5187         if len(args.sany):
 5188             LOGDBG('args.sany')
 5189             # Apply tag filtering, if opted
 5190             if tags_search:
 5191                 search_results = bdb.search_keywords_and_filter_by_tags(
 5192                     args.sany, False, args.deep, False, args.stag)
 5193             else:
 5194                 # Search URLs, titles, tags for any keyword
 5195                 search_results = bdb.searchdb(args.sany, False, args.deep)
 5196 
 5197             if exclude_results:
 5198                 search_results = bdb.exclude_results_from_search(
 5199                     search_results,
 5200                     args.exclude,
 5201                     args.deep
 5202                 )
 5203         else:
 5204             LOGERR('no keyword')
 5205     elif args.sall is not None:
 5206         if len(args.sall):
 5207             LOGDBG('args.sall')
 5208             # Apply tag filtering, if opted
 5209             if tags_search:
 5210                 search_results = bdb.search_keywords_and_filter_by_tags(
 5211                     args.sall,
 5212                     True,
 5213                     args.deep,
 5214                     False,
 5215                     args.stag
 5216                 )
 5217             else:
 5218                 # Search URLs, titles, tags with all keywords
 5219                 search_results = bdb.searchdb(args.sall, True, args.deep)
 5220 
 5221             if exclude_results:
 5222                 search_results = bdb.exclude_results_from_search(
 5223                     search_results,
 5224                     args.exclude,
 5225                     args.deep
 5226                 )
 5227         else:
 5228             LOGERR('no keyword')
 5229     elif args.sreg is not None:
 5230         if len(args.sreg):
 5231             LOGDBG('args.sreg')
 5232             # Apply tag filtering, if opted
 5233             if tags_search:
 5234                 search_results = bdb.search_keywords_and_filter_by_tags(
 5235                     args.sreg,
 5236                     False,
 5237                     False,
 5238                     True,
 5239                     args.stag
 5240                 )
 5241             else:
 5242                 # Run a regular expression search
 5243                 search_results = bdb.searchdb(args.sreg, regex=True)
 5244 
 5245             if exclude_results:
 5246                 search_results = bdb.exclude_results_from_search(
 5247                     search_results,
 5248                     args.exclude,
 5249                     args.deep
 5250                 )
 5251         else:
 5252             LOGERR('no expression')
 5253     elif len(args.keywords):
 5254         LOGDBG('args.keywords')
 5255         # Apply tag filtering, if opted
 5256         if tags_search:
 5257             search_results = bdb.search_keywords_and_filter_by_tags(
 5258                 args.keywords,
 5259                 False,
 5260                 args.deep,
 5261                 False,
 5262                 args.stag
 5263             )
 5264         else:
 5265             # Search URLs, titles, tags for any keyword
 5266             search_results = bdb.searchdb(args.keywords, False, args.deep)
 5267 
 5268         if exclude_results:
 5269             search_results = bdb.exclude_results_from_search(
 5270                 search_results,
 5271                 args.exclude,
 5272                 args.deep
 5273             )
 5274     elif args.stag is not None:
 5275         if len(args.stag):
 5276             LOGDBG('args.stag')
 5277             # Search bookmarks by tag
 5278             search_results = bdb.search_by_tag(' '.join(args.stag))
 5279             if exclude_results:
 5280                 search_results = bdb.exclude_results_from_search(
 5281                     search_results,
 5282                     args.exclude,
 5283                     args.deep
 5284                 )
 5285         else:
 5286             # Use sub prompt to list all tags
 5287             prompt(bdb, None, args.np, listtags=True, suggest=args.suggest)
 5288     elif args.exclude is not None:
 5289         LOGERR('No search criteria to exclude results from')
 5290     else:
 5291         search_opted = False
 5292 
 5293     # Add cmdline search options to readline history
 5294     if search_opted and len(args.keywords):
 5295         try:
 5296             readline.add_history(' '.join(args.keywords))
 5297         except Exception:
 5298             pass
 5299 
 5300     if search_results:
 5301         oneshot = args.np
 5302         update_search_results = False
 5303 
 5304         # Open all results in browser right away if args.oa
 5305         # is specified. The has priority over delete/update.
 5306         # URLs are opened first and updated/deleted later.
 5307         if args.oa:
 5308             for row in search_results:
 5309                 browse(row[1])
 5310 
 5311         if (
 5312                 (args.export is not None) or
 5313                 (args.delete is not None and not args.delete) or
 5314                 (args.update is not None and not args.update)):
 5315             oneshot = True
 5316 
 5317         if args.json is None and not args.format:
 5318             num = 10 if not args.count else args.count
 5319             prompt(bdb, search_results, oneshot, args.deep, num=num)
 5320         elif args.json is None:
 5321             print_rec_with_filter(search_results, field_filter=args.format)
 5322         elif args.json:
 5323             write_string_to_file(format_json(search_results, field_filter=args.format), args.json)
 5324         else:
 5325             # Printing in JSON format is non-interactive
 5326             print(format_json(search_results, field_filter=args.format))
 5327 
 5328         # Export the results, if opted
 5329         if args.export is not None:
 5330             bdb.exportdb(args.export[0], search_results)
 5331 
 5332         # In case of search and delete/update,
 5333         # prompt should be non-interactive
 5334         # delete gets priority over update
 5335         if args.delete is not None and not args.delete:
 5336             bdb.delete_resultset(search_results)
 5337         elif args.update is not None and not args.update:
 5338             update_search_results = True
 5339 
 5340     # Update record
 5341     if args.update is not None:
 5342         if args.url is not None:
 5343             url_in = args.url[0]
 5344         else:
 5345             url_in = ''
 5346 
 5347         # Parse tags into a comma-separated string
 5348         if tags_in:
 5349             if tags_in[0] == '+':
 5350                 tags = '+' + parse_tags(tags_in[1:])
 5351             elif tags_in[0] == '-':
 5352                 tags = '-' + parse_tags(tags_in[1:])
 5353             else:
 5354                 tags = parse_tags(tags_in)
 5355         else:
 5356             tags = None
 5357 
 5358         # No arguments to --update, update all
 5359         if not args.update:
 5360             # Update all records only if search was not opted
 5361             if not search_opted:
 5362                 bdb.update_rec(0, url_in, title_in, tags, desc_in, args.immutable, args.threads)
 5363             elif update_search_results and search_results is not None:
 5364                 if not args.tacit:
 5365                     print('Updated results:\n')
 5366 
 5367                 pos = len(search_results) - 1
 5368                 while pos >= 0:
 5369                     idx = search_results[pos][0]
 5370                     bdb.update_rec(
 5371                         idx,
 5372                         url_in,
 5373                         title_in,
 5374                         tags,
 5375                         desc_in,
 5376                         args.immutable,
 5377                         args.threads
 5378                     )
 5379 
 5380                     # Commit at every 200th removal
 5381                     if pos % 200 == 0:
 5382                         bdb.conn.commit()
 5383 
 5384                     pos -= 1
 5385         else:
 5386             for idx in args.update:
 5387                 if is_int(idx):
 5388                     bdb.update_rec(
 5389                         int(idx),
 5390                         url_in,
 5391                         title_in,
 5392                         tags,
 5393                         desc_in,
 5394                         args.immutable,
 5395                         args.threads
 5396                     )
 5397                 elif '-' in idx:
 5398                     try:
 5399                         vals = [int(x) for x in idx.split('-')]
 5400                         if vals[0] > vals[1]:
 5401                             vals[0], vals[1] = vals[1], vals[0]
 5402 
 5403                         # Update only once if range starts from 0 (all)
 5404                         if vals[0] == 0:
 5405                             bdb.update_rec(
 5406                                 0,
 5407                                 url_in,
 5408                                 title_in,
 5409                                 tags,
 5410                                 desc_in,
 5411                                 args.immutable,
 5412                                 args.threads
 5413                             )
 5414                         else:
 5415                             for _id in range(vals[0], vals[1] + 1):
 5416                                 bdb.update_rec(
 5417                                     _id,
 5418                                     url_in,
 5419                                     title_in,
 5420                                     tags,
 5421                                     desc_in,
 5422                                     args.immutable,
 5423                                     args.threads
 5424                                 )
 5425                                 if INTERRUPTED:
 5426                                     break
 5427                     except ValueError:
 5428                         LOGERR('Invalid index or range to update')
 5429                         bdb.close_quit(1)
 5430 
 5431                 if INTERRUPTED:
 5432                     break
 5433 
 5434     # Delete record
 5435     if args.delete is not None:
 5436         if not args.delete:
 5437             # Attempt delete-all only if search was not opted
 5438             if not search_opted:
 5439                 bdb.cleardb()
 5440         elif len(args.delete) == 1 and '-' in args.delete[0]:
 5441             try:
 5442                 vals = [int(x) for x in args.delete[0].split('-')]
 5443                 if len(vals) == 2:
 5444                     bdb.delete_rec(0, vals[0], vals[1], True)
 5445             except ValueError:
 5446                 LOGERR('Invalid index or range to delete')
 5447                 bdb.close_quit(1)
 5448         else:
 5449             ids = []
 5450             # Select the unique indices
 5451             for idx in args.delete:
 5452                 if idx not in ids:
 5453                     ids += (idx,)
 5454 
 5455             try:
 5456                 # Index delete order - highest to lowest
 5457                 ids.sort(key=lambda x: int(x), reverse=True)
 5458                 for idx in ids:
 5459                     bdb.delete_rec(int(idx))
 5460             except ValueError:
 5461                 LOGERR('Invalid index or range or combination')
 5462                 bdb.close_quit(1)
 5463 
 5464     # Print record
 5465     if args.print is not None:
 5466         if not args.print:
 5467             if args.count:
 5468                 search_results = bdb.list_using_id()
 5469                 prompt(bdb, search_results, args.np, False, num=args.count)
 5470             else:
 5471                 bdb.print_rec(0)
 5472         else:
 5473             if args.count:
 5474                 search_results = bdb.list_using_id(args.print)
 5475                 prompt(bdb, search_results, args.np, False, num=args.count)
 5476             else:
 5477                 try:
 5478                     for idx in args.print:
 5479                         if is_int(idx):
 5480                             bdb.print_rec(int(idx))
 5481                         elif '-' in idx:
 5482                             vals = [int(x) for x in idx.split('-')]
 5483                             bdb.print_rec(0, vals[0], vals[-1], True)
 5484 
 5485                 except ValueError:
 5486                     LOGERR('Invalid index or range to print')
 5487                     bdb.close_quit(1)
 5488 
 5489     # Replace a tag in DB
 5490     if args.replace is not None:
 5491         if len(args.replace) == 1:
 5492             bdb.delete_tag_at_index(0, args.replace[0])
 5493         else:
 5494             bdb.replace_tag(args.replace[0], args.replace[1:])
 5495 
 5496     # Export bookmarks
 5497     if args.export is not None and not search_opted:
 5498         bdb.exportdb(args.export[0])
 5499 
 5500     # Import bookmarks
 5501     if args.importfile is not None:
 5502         bdb.importdb(args.importfile[0], args.tacit)
 5503 
 5504     # Import bookmarks from browser
 5505     if args.ai:
 5506         bdb.auto_import_from_browser()
 5507 
 5508     # Open URL in browser
 5509     if args.open is not None:
 5510         if not args.open:
 5511             bdb.browse_by_index(0)
 5512         else:
 5513             try:
 5514                 for idx in args.open:
 5515                     if is_int(idx):
 5516                         bdb.browse_by_index(int(idx))
 5517                     elif '-' in idx:
 5518                         vals = [int(x) for x in idx.split('-')]
 5519                         bdb.browse_by_index(0, vals[0], vals[-1], True)
 5520             except ValueError:
 5521                 LOGERR('Invalid index or range to open')
 5522                 bdb.close_quit(1)
 5523 
 5524     # Shorten URL
 5525     if args.shorten:
 5526         if is_int(args.shorten[0]):
 5527             shorturl = bdb.tnyfy_url(index=int(args.shorten[0]))
 5528         else:
 5529             shorturl = bdb.tnyfy_url(url=args.shorten[0])
 5530 
 5531         if shorturl:
 5532             print(shorturl)
 5533 
 5534     # Expand URL
 5535     if args.expand:
 5536         if is_int(args.expand[0]):
 5537             url = bdb.tnyfy_url(index=int(args.expand[0]), shorten=False)
 5538         else:
 5539             url = bdb.tnyfy_url(url=args.expand[0], shorten=False)
 5540 
 5541         if url: