# Copyright (C) 2007 Insecure.Com LLC. # # Author: Guilherme Polo # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA from umitCore.I18N import _ from umitDB.Connection import ConnectDB from umitDB.Store import RawStore from umitDB.Retrieve import InventoryRetrieve class BadParams(Exception): pass class ConnectToUpdate(ConnectDB, InventoryRetrieve, RawStore): """ You should use this when doing inventory changes update exclusively in database. """ def __init__(self, database): ConnectDB.__init__(self, database) InventoryRetrieve.__init__(self, self.conn, self.cursor) RawStore.__init__(self, self.conn, self.cursor) class UpdateChanges: """ Updates/creates list of changes for Inventory. """ def __init__(self, invdb, fk_inventory=None): self.invdb = invdb self.fk_inventory = fk_inventory if fk_inventory: self.do_update(fk_inventory) def do_update(self, inv_id): """ Build changes found in scans related to an Inventory id and updates everything necessary. """ addresses = [ ] # get scan ids and finish time for especified inventory finish_data = self.invdb.get_finish_data_for_inventory_from_db(inv_id) # retrieve host addresses for especified inventory id for scan in finish_data: # retrieve host ids for each scan for host in self.invdb.get_hosts_id_for_scan_from_db(scan[0]): # retrieve ipv4 address for host addr = self.invdb.get_ipv4_for_host_from_db(host[0]) if addr in addresses: continue addresses.append(addr) # generate changes list for each address in especified inventory for addr in addresses: # get all host pks for especified inventory and for each address # in addresses base_data = self.invdb.get_hosts_base_data_for_inventory_from_db( addr, inv_id) if not base_data: print ("Inventory id %d has no data for host " "address '%s' yet.") % (inv_id, addr) continue # dict where all data will be stored data_dict = { } # get host and scan ids scan_ids = [item[0] for item in finish_data] host_ids = [item[0] for item in base_data] indexes = [ ] count = 0 # get indexes where there is host data for indx, item in enumerate(scan_ids): try: host_ids[count] except IndexError: continue else: indexes.append(scan_ids.index(host_ids[count])) count += 1 down_time = self._check_for_downtime(finish_data, base_data) for item in down_time[1]: data_dict[item] = ("availability", _("Host down"), down_time[0][item], -1, -1) # load initial data for doing comparison date = finish_data[indexes[0]][1] hostA = base_data[0] self.invdb.use_dict_cursor() pdata1 = self.invdb.get_portid_and_state_for_host_from_db(hostA[1]) self.invdb.use_standard_cursor() fpinfo1 = self.invdb.get_fingerprint_info_for_host_from_db(hostA[1]) osmatch1 = self.invdb.get_osmatch_for_host_from_db(hostA[1]) osclasses1 = self.invdb.get_osclasses_for_host_from_db(hostA[1]) pcdata1 = self.invdb.get_portid_and_fks_for_host_from_db(hostA[1]) host_count = 1 # now, load following data and compare against hostA for indx, item in enumerate(base_data[host_count:]): hostB = item # load data to compare against hostA self.invdb.use_dict_cursor() pdata2 = self.invdb.get_portid_and_state_for_host_from_db( hostB[1]) self.invdb.use_standard_cursor() fpinfo2 = self.invdb.get_fingerprint_info_for_host_from_db( hostB[1]) osmatch2 = self.invdb.get_osmatch_for_host_from_db(hostB[1]) osclasses2 = self.invdb.get_osclasses_for_host_from_db(hostB[1]) pcdata2 = self.invdb.get_portid_and_fks_for_host_from_db( hostB[1]) # compare old data against new data category, diff_text = self._compare_data(pdata2, fpinfo2, osmatch2, osclasses2, pcdata2, pdata1, fpinfo1, osmatch1, osclasses1, pcdata1) # store in data_dict current result data_dict[hostA[0]] = (category, diff_text, date, hostA[1], hostB[1]) # swap data pdata1 = pdata2 pcdata1 = pcdata2 fpinfo1 = fpinfo2 osmatch1 = osmatch2 osclasses1 = osclasses2 hostA = hostB # get next date date = finish_data[indexes[indx+1]][1] # now load the first entry data_dict[hostA[0]] = ('inventory', _("Host added to the Inventory."), date, hostA[1], hostA[1]) self._insert_changes(data_dict, addr, inv_id) def _insert_changes(self, data_dict, addr_id, inventory_id): """ Insert changes in database, this was collected at do_update method. """ # sort dict keys in descendent order dict_keys = data_dict.keys() dict_keys.sort() dict_keys.reverse() # insert data into database fk_address = self.invdb.get_address_id_for_address_from_db(addr_id) for key in dict_keys: affected = data_dict[key][0] text = data_dict[key][1] date = data_dict[key][2] new_hostid = data_dict[key][3] old_hostid = data_dict[key][4] # check if category 'affected' already exists on database fk_category = self.invdb.get_inventory_change_category_id(affected) if not fk_category: # didn't exist, create it now self.invdb.insert_inventory_change_category_db(affected) fk_category = self.invdb.get_id_for("inventory_change_category") # check if comparison is already in database # (this should have been done at earlier stage, but for now # it is being done here) ret = self.invdb.get_inventory_comparison(old_hostid, new_hostid, date, inventory_id) if not ret: # need to insert new comparison self.invdb.insert_inventory_comparison_db(old_hostid, new_hostid, date, text, inventory_id, fk_category, fk_address) def _compare_data(self, pdata2, fpinfo2, osmatch2, osclasses2, pcdata2, pdata1, fpinfo1, osmatch1, osclasses1, pcdata1): """ Compare two sets of data, checking how it changes. """ host_diff = '' common_text = '' ports_only = None # ports diff only fp_only = None # fingerprint diff only # compare pdataNs if pdata1 != pdata2: host_diff = self._ports_diff(pdata2, pdata1) fp_only = False ports_only = True # compare pcdataNs (pc here stands for port complete, wich means # we will be dealing with other port infos beyond id and states, # like we did right above. old_ports = { } new_ports = { } self.invdb.use_dict_cursor() for pd in pcdata2: fpd = self.invdb.get_port_data_for_pdata_from_db(pd[2], pd[3], pd[1]) old_ports[pd[0]] = fpd for pd in pcdata1: fpd = self.invdb.get_port_data_for_pdata_from_db(pd[2], pd[3], pd[1]) new_ports[pd[0]] = fpd self.invdb.use_standard_cursor() info_changes = [ ] for key, value in old_ports.items(): if key in new_ports: if value != new_ports[key]: info_changes.append(key) if info_changes: if not ports_only: ports_only = True verb, plural = self.conjugate(info_changes, False) ports_str = ', '.join([str(p) for p in info_changes]) this_port_diff = ' '.join([ _("Port") + plural, ports_str, verb, _("changed info")]) host_diff = ''.join([host_diff, this_port_diff]) # compare fpinfoNs # dont consider uptime and lastboot in fingerprint (will probably # not consider others too) if fpinfo1 and (fpinfo1[2:] != fpinfo2[2:]): space = host_diff and ' ' or '' common_text = space + _("Fingerprint, ") ports_only = False if fp_only is None: fp_only = True # compare osmatchNs if osmatch1 != osmatch2: space = host_diff and ' ' or '' common_text = ''.join([common_text, _("OS Match, ")]) fp_only = False ports_only = False # compare osclassesNs if osclasses1 != osclasses2: space = host_diff and ' ' or '' if len(common_text) == len("Fingerprint, "): common_text = ''.join([common_text[:-2], " "]) if common_text: common_text = ''.join([common_text, _("and OS Classes")]) else: common_text = ''.join([common_text, _("OS Classes")]) fp_only = False ports_only = False if common_text: if fp_only or len(common_text) == len(" Fingerprint, "): common_text = common_text[:-2] host_diff = ''.join([host_diff, common_text, _(" changed.")]) # check diff if host_diff: if ports_only: affected = 'ports' elif fp_only: affected = 'fingerprint' else: affected = 'several' else: # Nothing here means "Almost nothing", there could be # changes in extraports for example. affected = 'nothing' host_diff = _("No noticeables changes since last sucessfull scan.") return (affected, host_diff) def conjugate(self, alist, toBe=True): """ Do conjugation based on alist size. It expects that alist is not empty. """ if toBe: verbs = ('are', 'is') else: verbs = ('have', 'has') if len(alist) > 1: verb = verbs[0] plural = _("s") else: verb = verbs[1] plural = _("") return verb, plural def _ports_diff(self, old, new): """ Return a prettier difference between pdata. """ closed_text = _("closed") open_text = _("open") now_text = _("now") port_text = _("Port") and_text = _("and") # first build dict where portid is the key old_dict = { } new_dict = { } for d in old: old_dict[d['portid']] = d['state'] for d in new: new_dict[d['portid']] = d['state'] # check for port changes now closed_ports = [ ] open_ports = [ ] for key, value in old_dict.items(): if key in new_dict: new_value = new_dict[key] if value != new_value: print value, new_value, 'differs but Im not doing nothing' else: closed_ports.append(key) for key, value in new_dict.items(): if key in old_dict: old_value = old_dict[key] if value != old_value: print value, old_value, 'differs but Im not doing nothing' else: open_ports.append(key) text = '' if closed_ports: verb, plural = self.conjugate(closed_ports) closed_ports = ', '.join([str(p) for p in closed_ports]) closed_ports = "%s%s %s %s %s %s." % (port_text, plural, closed_ports, verb, closed_text, now_text) if open_ports: verb, plural = self.conjugate(open_ports) open_ports = ', '.join([str(p) for p in open_ports]) open_ports = "%s%s %s %s %s %s." % (port_text, plural, open_ports, verb, open_text, now_text) if open_ports and closed_ports: text = ' '.join([open_ports[:-1 -len(now_text) -1], and_text, closed_ports]) elif open_ports: text = open_ports elif closed_ports: text = closed_ports return text def _check_for_downtime(self, all_scans, host_scans): """ Return scan id associated to a date showing in what scans a host was down. """ down_d = { } down_order = [ ] d = 0 for item in all_scans: try: host_scans[d][0] except IndexError: down_order.append(item[0]) down_d[item[0]] = item[1] else: if item[0] == host_scans[d][0]: d += 1 else: down_order.append(item[0]) down_d[item[0]] = item[1] return (down_d, down_order) class ChangesRetrieve(InventoryRetrieve): """ Retrieves changes from database in many ways. """ def __init__(self, conn, cursor): InventoryRetrieve.__init__(self, conn, cursor) def get_categories_id_name(self): """ Return all category_id, category_name from database. """ self.cursor.execute("SELECT * FROM inventory_change_category") data = self.cursor.fetchall() return data def get_categories_name(self): """ Return all categories name from database. """ self.cursor.execute("SELECT name FROM inventory_change_category") ctg = self.cursor.fetchall() return ctg def get_category_name_by_id(self, cid): """ Return category name with especified id. """ name = self.cursor.execute("SELECT name FROM " "inventory_change_category WHERE pk=?", (cid, )).fetchone() if name: return name[0] def get_category_id_by_name(self, name): """ Return category id with especified name. """ cid = self.cursor.execute("SELECT pk FROM inventory_change_category " "WHERE name=?", (name, )).fetchone()[0] return cid def timerange_changes_data_generic(self, start, end, category, inventory, hostaddr): """ Selects what method to use to grab changes data. """ # start and end should always be present if not start or not end: raise BadParams("You should especify range start and range end") # check for missing args for inventory and hostaddr if inventory and not hostaddr or not inventory and hostaddr: raise BadParams("You should especify a hostaddr and inventory") if category: return self.timerange_changes_categoryid_data(category, start, end, inventory, hostaddr) else: return self.timerange_changes_data(start, end, inventory, hostaddr) def timerange_changes_data(self, start, end, fk_inventory=None, fk_address=None): """ Retrieve changes data in a timerange. """ if fk_inventory: data = self.cursor.execute("SELECT fk_category, " "short_descritpion, entry_date, fk_inventory, fk_address, " "old_hostid, new_hostid FROM _inventory_changes " "WHERE fk_inventory=? AND fk_address=? AND entry_date >= ? " "AND entry_date < ? ORDER BY entry_date DESC", (fk_inventory, fk_address, start, end)).fetchall() else: data = self.cursor.execute("SELECT fk_category, " "short_description, entry_date, fk_inventory, fk_address, " "old_hostid, new_hostid FROM _inventory_changes " "WHERE entry_date >= ? AND entry_date < ? " "ORDER BY entry_date DESC", (start, end)).fetchall() return data def timerange_changes_categoryid_data(self, fk_category, start, end, fk_inventory=None, fk_address=None): """ Retrieve changes data in a timerange for an especific category id. """ if fk_inventory: data = self.cursor.execute("SELECT fk_category, " "short_description, entry_date, fk_inventory, fk_address, " "old_hostid, new_hostid FROM _inventory_changes " "WHERE fk_inventory=? AND fk_address=? AND fk_category=? AND " "entry_date >= ? AND entry_date < ? ORDER BY entry_date DESC", (fk_inventory, fk_address, fk_category, start, end)).fetchall() else: data = self.cursor.execute("SELECT fk_category, " "short_description, entry_date, fk_inventory, fk_address, " "old_hostid, new_hostid FROM _inventory_changes " "WHERE fk_category=? AND entry_date >= ? AND entry_date < ? " "ORDER BY entry_date DESC", (fk_category, start, end)).fetchall() return data def timerange_changes_categoryname_data(self, category, start, end): """ Retrieve changes data in a timerange for an especific category name. """ data = self.cursor.execute("SELECT fk_category, short_description, " "entry_date, fk_inventory, fk_address, old_hostid, new_hostid " "FROM _inventory_changes JOIN inventory_change_category as icc ON " "(_inventory_changes.fk_category = icc.pk) " "WHERE icc.name=? AND entry_date >= ? AND entry_date < ? " "ORDER BY entry_date DESC", (category, start, end)).fetchall() return data def timerange_changes_count_generic(self, start, end, category, inventory, hostaddr): """ Selects what method to use to grab changes count. """ # start and end should always be present if not start or not end: raise BadParams("You should especify range start and range end") # check for missing args for inventory and hostaddr if inventory and not hostaddr or not inventory and hostaddr: raise BadParams("You should especify a hostaddr and inventory") if category: return self.timerange_changes_categoryid_count(category, start, end, inventory, hostaddr) else: return self.timerange_changes_count(start, end, inventory, hostaddr) def timerange_changes_count(self, start, end, fk_inventory=None, fk_hostaddr=None): """ Get number of changes in a timerange. """ if fk_inventory: count = self.cursor.execute("SELECT pk FROM _inventory_changes " "WHERE fk_inventory=? AND fk_address=? AND entry_date >= ? " "AND entry_date < ?", (fk_inventory, fk_hostaddr, start, end)).fetchall() else: count = self.cursor.execute("SELECT pk FROM _inventory_changes " "WHERE entry_date >= ? AND entry_date < ?", (start, end)).fetchall() return len(count) def timerange_changes_categoryid_count(self, fk_category, start, end, fk_inventory=None, fk_hostaddr=None): """ Get number of changes in a timerange for an especific category id. """ if fk_inventory: self.cursor.execute("SELECT pk FROM _inventory_changes WHERE " "fk_category=? AND fk_inventory=? AND fk_address=? AND " "entry_date >= ? AND entry_date < ?", (fk_category, fk_inventory, fk_hostaddr, start, end)) else: self.cursor.execute("SELECT pk FROM _inventory_changes WHERE " "fk_category=? AND entry_date >= ? AND entry_date < ?", (fk_category, start, end)) count = self.cursor.fetchall() return len(count) def timerange_changes_categoryname_count(self, category, start, end): """ Get number of changes in a timerange for an especific category name. """ # not being used count = self.cursor.execute("SELECT pk FROM _inventory_changes JOIN " "inventory_change_category as icc ON " "(_inventory_changes.fk_category = icc.pk) " "WHERE icc.name=? AND entry_date > ? AND entry_date < ?", (category, start, end)).fetchall() return len(count)