"Fossies" - the Fresh Open Source Software Archive

Member "glusterfs-8.2/tools/glusterfind/src/changelogdata.py" (16 Sep 2020, 16079 Bytes) of package /linux/misc/glusterfs-8.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "changelogdata.py" see the Fossies "Dox" file reference documentation.

    1 # -*- coding: utf-8 -*-
    2 
    3 # Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com/>
    4 # This file is part of GlusterFS.
    5 #
    6 # This file is licensed to you under your choice of the GNU Lesser
    7 # General Public License, version 3 or any later version (LGPLv3 or
    8 # later), or the GNU General Public License, version 2 (GPLv2), in all
    9 # cases as published by the Free Software Foundation.
   10 
   11 import sqlite3
   12 import os
   13 
   14 from utils import RecordType, unquote_plus_space_newline
   15 from utils import output_path_prepare
   16 
   17 
   18 class OutputMerger(object):
   19     """
   20     Class to merge the output files collected from
   21     different nodes
   22     """
   23     def __init__(self, db_path, all_dbs):
   24         self.conn = sqlite3.connect(db_path)
   25         self.cursor = self.conn.cursor()
   26         self.cursor_reader = self.conn.cursor()
   27         query = "DROP TABLE IF EXISTS finallist"
   28         self.cursor.execute(query)
   29 
   30         query = """
   31         CREATE TABLE finallist(
   32           id     INTEGER PRIMARY KEY AUTOINCREMENT,
   33           ts     VARCHAR,
   34           type   VARCHAR,
   35           gfid   VARCHAR,
   36           path1  VARCHAR,
   37           path2  VARCHAR,
   38           UNIQUE (type, path1, path2) ON CONFLICT IGNORE
   39         )
   40         """
   41         self.cursor.execute(query)
   42 
   43         # If node database exists, read each db and insert into
   44         # final table. Ignore if combination of TYPE PATH1 PATH2
   45         # already exists
   46         for node_db in all_dbs:
   47             if os.path.exists(node_db):
   48                 conn = sqlite3.connect(node_db)
   49                 cursor = conn.cursor()
   50                 query = """
   51                 SELECT   ts, type, gfid, path1, path2
   52                 FROM     gfidpath
   53                 WHERE    path1 != ''
   54                 ORDER BY id ASC
   55                 """
   56                 for row in cursor.execute(query):
   57                     self.add_if_not_exists(row[0], row[1], row[2],
   58                                            row[3], row[4])
   59 
   60         self.conn.commit()
   61 
   62     def add_if_not_exists(self, ts, ty, gfid, path1, path2=""):
   63         # Adds record to finallist only if not exists
   64         query = """
   65         INSERT INTO finallist(ts, type, gfid, path1, path2)
   66         VALUES(?, ?, ?, ?, ?)
   67         """
   68         self.cursor.execute(query, (ts, ty, gfid, path1, path2))
   69 
   70     def get(self):
   71         query = """SELECT type, path1, path2 FROM finallist
   72         ORDER BY ts ASC, id ASC"""
   73         return self.cursor_reader.execute(query)
   74 
   75     def get_failures(self):
   76         query = """
   77         SELECT   gfid
   78         FROM     finallist
   79         WHERE path1 = '' OR (path2 = '' AND type = 'RENAME')
   80         """
   81         return self.cursor_reader.execute(query)
   82 
   83 
   84 class ChangelogData(object):
   85     def __init__(self, dbpath, args):
   86         self.conn = sqlite3.connect(dbpath)
   87         self.cursor = self.conn.cursor()
   88         self.cursor_reader = self.conn.cursor()
   89         self._create_table_gfidpath()
   90         self._create_table_pgfid()
   91         self._create_table_inodegfid()
   92         self.args = args
   93         self.path_sep = "/"
   94 
   95     def _create_table_gfidpath(self):
   96         drop_table = "DROP TABLE IF EXISTS gfidpath"
   97         self.cursor.execute(drop_table)
   98 
   99         create_table = """
  100         CREATE TABLE gfidpath(
  101             id     INTEGER PRIMARY KEY AUTOINCREMENT,
  102             ts     VARCHAR,
  103             type   VARCHAR,
  104             gfid   VARCHAR(40),
  105             pgfid1 VARCHAR(40) DEFAULT '',
  106             bn1    VARCHAR(500) DEFAULT '',
  107             pgfid2 VARCHAR(40) DEFAULT '',
  108             bn2    VARCHAR(500) DEFAULT '',
  109             path1  VARCHAR DEFAULT '',
  110             path2  VARCHAR DEFAULT ''
  111         )
  112         """
  113         self.cursor.execute(create_table)
  114 
  115         create_index = """
  116         CREATE INDEX gfid_index ON gfidpath(gfid);
  117         """
  118         self.cursor.execute(create_index)
  119 
  120     def _create_table_inodegfid(self):
  121         drop_table = "DROP TABLE IF EXISTS inodegfid"
  122         self.cursor.execute(drop_table)
  123 
  124         create_table = """
  125         CREATE TABLE inodegfid(
  126             inode     INTEGER PRIMARY KEY,
  127             gfid      VARCHAR(40),
  128             converted INTEGER DEFAULT 0,
  129             UNIQUE    (inode, gfid) ON CONFLICT IGNORE
  130         )
  131         """
  132         self.cursor.execute(create_table)
  133 
  134     def _create_table_pgfid(self):
  135         drop_table = "DROP TABLE IF EXISTS pgfid"
  136         self.cursor.execute(drop_table)
  137 
  138         create_table = """
  139         CREATE TABLE pgfid(
  140             pgfid  VARCHAR(40) PRIMARY KEY,
  141             UNIQUE (pgfid) ON CONFLICT IGNORE
  142         )
  143         """
  144         self.cursor.execute(create_table)
  145 
  146     def _get(self, tablename, filters):
  147         # SELECT * FROM <TABLENAME> WHERE <CONDITION>
  148         params = []
  149         query = "SELECT * FROM %s WHERE 1=1" % tablename
  150 
  151         for key, value in filters.items():
  152             query += " AND %s = ?" % key
  153             params.append(value)
  154 
  155         return self.cursor_reader.execute(query, params)
  156 
  157     def _get_distinct(self, tablename, distinct_field, filters):
  158         # SELECT DISTINCT <COL> FROM <TABLENAME> WHERE <CONDITION>
  159         params = []
  160         query = "SELECT DISTINCT %s FROM %s WHERE 1=1" % (distinct_field,
  161                                                           tablename)
  162 
  163         for key, value in filters.items():
  164             query += " AND %s = ?" % key
  165             params.append(value)
  166 
  167         return self.cursor_reader.execute(query, params)
  168 
  169     def _delete(self, tablename, filters):
  170         # DELETE FROM <TABLENAME> WHERE <CONDITIONS>
  171         query = "DELETE FROM %s WHERE 1=1" % tablename
  172         params = []
  173 
  174         for key, value in filters.items():
  175             query += " AND %s = ?" % key
  176             params.append(value)
  177 
  178         self.cursor.execute(query, params)
  179 
  180     def _add(self, tablename, data):
  181         # INSERT INTO <TABLENAME>(<col1>, <col2>..) VALUES(?,?..)
  182         query = "INSERT INTO %s(" % tablename
  183         fields = []
  184         params = []
  185         for key, value in data.items():
  186             fields.append(key)
  187             params.append(value)
  188 
  189         values_substitute = len(fields)*["?"]
  190         query += "%s) VALUES(%s)" % (",".join(fields),
  191                                      ",".join(values_substitute))
  192         self.cursor.execute(query, params)
  193 
  194     def _update(self, tablename, data, filters):
  195         # UPDATE <TABLENAME> SET col1 = ?,.. WHERE col1=? AND ..
  196         params = []
  197         update_fields = []
  198         for key, value in data.items():
  199             update_fields.append("%s = ?" % key)
  200             params.append(value)
  201 
  202         query = "UPDATE %s SET %s WHERE 1 = 1" % (tablename,
  203                                                   ", ".join(update_fields))
  204 
  205         for key, value in filters.items():
  206             query += " AND %s = ?" % key
  207             params.append(value)
  208 
  209         self.cursor.execute(query, params)
  210 
  211     def _exists(self, tablename, filters):
  212         if not filters:
  213             return False
  214 
  215         query = "SELECT COUNT(1) FROM %s WHERE 1=1" % tablename
  216         params = []
  217 
  218         for key, value in filters.items():
  219             query += " AND %s = ?" % key
  220             params.append(value)
  221 
  222         self.cursor.execute(query, params)
  223         row = self.cursor.fetchone()
  224         return True if row[0] > 0 else False
  225 
  226     def gfidpath_add(self, changelogfile, ty, gfid, pgfid1="", bn1="",
  227                      pgfid2="", bn2="", path1="", path2=""):
  228         self._add("gfidpath", {
  229             "ts": changelogfile.split(".")[-1],
  230             "type": ty,
  231             "gfid": gfid,
  232             "pgfid1": pgfid1,
  233             "bn1": bn1,
  234             "pgfid2": pgfid2,
  235             "bn2": bn2,
  236             "path1": path1,
  237             "path2": path2
  238         })
  239 
  240     def gfidpath_update(self, data, filters):
  241         self._update("gfidpath", data, filters)
  242 
  243     def gfidpath_delete(self, filters):
  244         self._delete("gfidpath", filters)
  245 
  246     def gfidpath_exists(self, filters):
  247         return self._exists("gfidpath", filters)
  248 
  249     def gfidpath_get(self, filters={}):
  250         return self._get("gfidpath", filters)
  251 
  252     def gfidpath_get_distinct(self, distinct_field, filters={}):
  253         return self._get_distinct("gfidpath", distinct_field, filters)
  254 
  255     def pgfid_add(self, pgfid):
  256         self._add("pgfid", {
  257             "pgfid": pgfid
  258         })
  259 
  260     def pgfid_update(self, data, filters):
  261         self._update("pgfid", data, filters)
  262 
  263     def pgfid_get(self, filters={}):
  264         return self._get("pgfid", filters)
  265 
  266     def pgfid_get_distinct(self, distinct_field, filters={}):
  267         return self._get_distinct("pgfid", distinct_field, filters)
  268 
  269     def pgfid_exists(self, filters):
  270         return self._exists("pgfid", filters)
  271 
  272     def inodegfid_add(self, inode, gfid, converted=0):
  273         self._add("inodegfid", {
  274             "inode": inode,
  275             "gfid": gfid,
  276             "converted": converted
  277         })
  278 
  279     def inodegfid_update(self, data, filters):
  280         self._update("inodegfid", data, filters)
  281 
  282     def inodegfid_get(self, filters={}):
  283         return self._get("inodegfid", filters)
  284 
  285     def inodegfid_get_distinct(self, distinct_field, filters={}):
  286         return self._get_distinct("inodegfid", distinct_field, filters)
  287 
  288     def inodegfid_exists(self, filters):
  289         return self._exists("inodegfid", filters)
  290 
  291     def append_path1(self, path, inode):
  292         # || is for concatenate in SQL
  293         query = """UPDATE gfidpath SET path1 = path1 || ',' || ?
  294         WHERE gfid IN (SELECT gfid FROM inodegfid WHERE inode = ?)"""
  295         self.cursor.execute(query, (path, inode))
  296 
  297     def gfidpath_set_path1(self, path1, pgfid1):
  298         # || is for concatenate in SQL
  299         if path1 == "":
  300             update_str1 = "? || bn1"
  301             update_str2 = "? || bn2"
  302         else:
  303             update_str1 = "? || '{0}' || bn1".format(self.path_sep)
  304             update_str2 = "? || '{0}' || bn2".format(self.path_sep)
  305 
  306         query = """UPDATE gfidpath SET path1 = %s
  307         WHERE pgfid1 = ?""" % update_str1
  308         self.cursor.execute(query, (path1, pgfid1))
  309 
  310         # Set Path2 if pgfid1 and pgfid2 are same
  311         query = """UPDATE gfidpath SET path2 = %s
  312         WHERE pgfid2 = ?""" % update_str2
  313         self.cursor.execute(query, (path1, pgfid1))
  314 
  315     def gfidpath_set_path2(self, path2, pgfid2):
  316         # || is for concatenate in SQL
  317         if path2 == "":
  318             update_str = "? || bn2"
  319         else:
  320             update_str = "? || '{0}' || bn2".format(self.path_sep)
  321 
  322         query = """UPDATE gfidpath SET path2 = %s
  323         WHERE pgfid2 = ?""" % update_str
  324         self.cursor.execute(query, (path2, pgfid2))
  325 
  326     def when_create_mknod_mkdir(self, changelogfile, data):
  327         # E <GFID> <MKNOD|CREATE|MKDIR> <MODE> <USER> <GRP> <PGFID>/<BNAME>
  328         # Add the Entry to DB
  329         pgfid1, bn1 = data[6].split("/", 1)
  330 
  331         if self.args.no_encode:
  332             bn1 = unquote_plus_space_newline(bn1).strip()
  333 
  334         self.gfidpath_add(changelogfile, RecordType.NEW, data[1], pgfid1, bn1)
  335 
  336     def when_rename(self, changelogfile, data):
  337         # E <GFID> RENAME <OLD_PGFID>/<BNAME> <PGFID>/<BNAME>
  338         pgfid1, bn1 = data[3].split("/", 1)
  339         pgfid2, bn2 = data[4].split("/", 1)
  340 
  341         if self.args.no_encode:
  342             bn1 = unquote_plus_space_newline(bn1).strip()
  343             bn2 = unquote_plus_space_newline(bn2).strip()
  344 
  345         if self.gfidpath_exists({"gfid": data[1], "type": "NEW",
  346                                  "pgfid1": pgfid1, "bn1": bn1}):
  347             # If <OLD_PGFID>/<BNAME> is same as CREATE, Update
  348             # <NEW_PGFID>/<BNAME> in NEW.
  349             self.gfidpath_update({"pgfid1": pgfid2, "bn1": bn2},
  350                                  {"gfid": data[1], "type": "NEW",
  351                                   "pgfid1": pgfid1, "bn1": bn1})
  352         elif self.gfidpath_exists({"gfid": data[1], "type": "RENAME",
  353                                    "pgfid2": pgfid1, "bn2": bn1}):
  354             # If we are renaming file back to original name then just
  355             # delete the entry since it will effectively be a no-op
  356             if self.gfidpath_exists({"gfid": data[1], "type": "RENAME",
  357                                      "pgfid2": pgfid1, "bn2": bn1,
  358                                      "pgfid1": pgfid2, "bn1": bn2}):
  359                 self.gfidpath_delete({"gfid": data[1], "type": "RENAME",
  360                                       "pgfid2": pgfid1, "bn2": bn1})
  361             else:
  362                 # If <OLD_PGFID>/<BNAME> is same as <PGFID2>/<BN2>
  363                 # (may be previous RENAME)
  364                 # then UPDATE <NEW_PGFID>/<BNAME> as <PGFID2>/<BN2>
  365                 self.gfidpath_update({"pgfid2": pgfid2, "bn2": bn2},
  366                                      {"gfid": data[1], "type": "RENAME",
  367                                       "pgfid2": pgfid1, "bn2": bn1})
  368         else:
  369             # Else insert as RENAME
  370             self.gfidpath_add(changelogfile, RecordType.RENAME, data[1],
  371                               pgfid1, bn1, pgfid2, bn2)
  372 
  373         if self.gfidpath_exists({"gfid": data[1], "type": "MODIFY"}):
  374             # If MODIFY exists already for that GFID, remove it and insert
  375             # again so that MODIFY entry comes after RENAME entry
  376             # Output will have MODIFY <NEWNAME>
  377             self.gfidpath_delete({"gfid": data[1], "type": "MODIFY"})
  378             self.gfidpath_add(changelogfile, RecordType.MODIFY, data[1])
  379 
  380     def when_link_symlink(self, changelogfile, data):
  381         # E <GFID> <LINK|SYMLINK> <PGFID>/<BASENAME>
  382         # Add as New record in Db as Type NEW
  383         pgfid1, bn1 = data[3].split("/", 1)
  384         if self.args.no_encode:
  385             bn1 = unquote_plus_space_newline(bn1).strip()
  386 
  387         self.gfidpath_add(changelogfile, RecordType.NEW, data[1], pgfid1, bn1)
  388 
  389     def when_data_meta(self, changelogfile, data):
  390         # If GFID row exists, Ignore else Add to Db
  391         if not self.gfidpath_exists({"gfid": data[1], "type": "NEW"}) and \
  392            not self.gfidpath_exists({"gfid": data[1], "type": "MODIFY"}):
  393             self.gfidpath_add(changelogfile, RecordType.MODIFY, data[1])
  394 
  395     def when_unlink_rmdir(self, changelogfile, data):
  396         # E <GFID> <UNLINK|RMDIR> <PGFID>/<BASENAME>
  397         pgfid1, bn1 = data[3].split("/", 1)
  398 
  399         if self.args.no_encode:
  400             bn1 = unquote_plus_space_newline(bn1).strip()
  401 
  402         deleted_path = data[4] if len(data) == 5 else ""
  403         if deleted_path != "":
  404             deleted_path = unquote_plus_space_newline(deleted_path)
  405             deleted_path = output_path_prepare(deleted_path, self.args)
  406 
  407         if self.gfidpath_exists({"gfid": data[1], "type": "NEW",
  408                                  "pgfid1": pgfid1, "bn1": bn1}):
  409             # If path exists in table as NEW with same GFID
  410             # Delete that row
  411             self.gfidpath_delete({"gfid": data[1], "type": "NEW",
  412                                   "pgfid1": pgfid1, "bn1": bn1})
  413         else:
  414             # Else Record as DELETE
  415             self.gfidpath_add(changelogfile, RecordType.DELETE, data[1],
  416                               pgfid1, bn1, path1=deleted_path)
  417 
  418         # Update path1 as deleted_path if pgfid1 and bn1 is same as deleted
  419         self.gfidpath_update({"path1": deleted_path}, {"gfid": data[1],
  420                                                        "pgfid1": pgfid1,
  421                                                        "bn1": bn1})
  422 
  423         # Update path2 as deleted_path if pgfid2 and bn2 is same as deleted
  424         self.gfidpath_update({"path2": deleted_path}, {
  425             "type": RecordType.RENAME,
  426             "gfid": data[1],
  427             "pgfid2": pgfid1,
  428             "bn2": bn1})
  429 
  430         # If deleted directory is parent for somebody
  431         query1 = """UPDATE gfidpath SET path1 = ? || '{0}' || bn1
  432         WHERE pgfid1 = ? AND path1 != ''""".format(self.path_sep)
  433         self.cursor.execute(query1, (deleted_path, data[1]))
  434 
  435         query1 = """UPDATE gfidpath SET path2 = ? || '{0}' || bn1
  436         WHERE pgfid2 = ? AND path2 != ''""".format(self.path_sep)
  437         self.cursor.execute(query1, (deleted_path, data[1]))
  438 
  439     def commit(self):
  440         self.conn.commit()