root/branch/ggpolo/umitDB/Store.py @ 1431

Revision 1431, 14.8 kB (checked in by ggpolo, 6 years ago)

Huge speed gain on XML insertion into databaseng, including insertion on Inventory and calculating changes.
Line 
1# Copyright (C) 2007 Insecure.Com LLC.
2#
3# Author:  Guilherme Polo <ggpolo@gmail.com>
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program; if not, write to the Free Software
17# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
18# USA
19
20from umitDB.Utils import empty
21from umitDB.Utils import debug
22   
23"""
24Missing methods for:
25    Traceroute insertion.
26"""
27
28class RawStore:
29    """
30    Store data into database.
31    """
32   
33    def __init__(self, conn, cursor):
34        """
35        Expects a conn and cursor from database connection.   
36        """
37        self.conn = conn
38        self.cursor = cursor
39
40
41    def insert_scan_db(self, scan_d):
42        """
43        Creates new record in scan with data from scan dict.
44        """
45        debug("Inserting new scan into database")
46
47        self.cursor.execute("INSERT INTO scan (args, start, startstr, finish, \
48                    finishstr, xmloutputversion, xmloutput, verbose, \
49                    debugging, hosts_up, hosts_down, fk_scanner) VALUES \
50                    (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (scan_d["args"],
51                    scan_d["start"], scan_d["startstr"], scan_d["finish"],
52                    scan_d["finishstr"], scan_d["xmloutputversion"],
53                    scan_d["xmloutput"], scan_d["verbose"], 
54                    scan_d["debugging"], scan_d["hosts_up"], 
55                    scan_d["hosts_down"], scan_d["scanner"]))
56
57   
58    def insert_scaninfo_db(self, scaninfo):
59        """
60        Creates new record in scaninfo with data from scaninfo dict.
61        """
62        debug("Inserting new scaninfo into database")
63
64        self.cursor.execute("INSERT INTO scaninfo (numservices, services, \
65                        fk_scan, fk_scan_type, fk_protocol) VALUES \
66                        (?, ?, ?, ?, ?)", (scaninfo["numservices"], 
67                        scaninfo["services"], scaninfo["fk_scan"],
68                        scaninfo["type"], scaninfo["protocol"]))
69
70
71    def insert_scan_type_db(self, scan_name):
72        """
73        Insert new record in scan_type.
74        """
75        debug("Inserting new scan_type into database")
76       
77        self.cursor.execute("INSERT INTO scan_type (name) VALUES \
78                                 (?)", (scan_name, ))
79
80
81    def insert_scanner_db(self, scanner_name, scanner_version):
82        """
83        Creates new record in scanner.
84        """
85        debug("Inserting new scanner into database")
86       
87        self.cursor.execute("INSERT INTO scanner (name, version) VALUES \
88                                 (?, ?)", (scanner_name, scanner_version))
89       
90
91    def insert_port_db(self, portid, service_info_id, protocol_id, 
92                                      port_state_id):
93        """
94        Creates new record in port.
95        """
96        debug("Inserting new port into database")
97       
98        self.cursor.execute("INSERT INTO port (portid, fk_service_info, \
99                       fk_protocol, fk_port_state) VALUES (?, ?, ?, ?)",
100                       (portid, service_info_id, protocol_id,
101                        port_state_id))
102           
103   
104    def insert_port_state_db(self, port_state):
105        """
106        Creates new record in port_state.
107        """
108        debug("Inserting new port_state into database")
109       
110        self.cursor.execute("INSERT INTO port_state (state) VALUES \
111                                (?)", (port_state, ))
112   
113   
114    def insert_protocol_db(self, protocol):
115        """
116        Creates new record in protocol.
117        """
118        debug("Inserting new protocol into database")
119       
120        self.cursor.execute("INSERT INTO protocol (name) VALUES \
121                                 (?)", (protocol, ))
122       
123   
124    def insert_host_port_db(self, fk_host, fk_port):
125        """
126        Creates new record in _host_port based on fk_host and fk_port.
127        """
128        debug("Inserting new _host_port into database")
129       
130        self.cursor.execute("INSERT INTO _host_port (fk_host, fk_port) \
131                    VALUES (?, ?)", (fk_host, fk_port))
132       
133
134    def insert_extraports_db(self, count, fk_host, fk_port_state):
135        """
136        Creates new record in extraports.
137        """
138        debug("Inserting new extraports into database")
139       
140        self.cursor.execute("INSERT INTO extraports (count, fk_host, \
141                            fk_port_state) VALUES (?, ?, ?)",
142                            (count, fk_host, fk_port_state))
143
144
145    def insert_portused_db(self, portid, fk_port_state, fk_protocol, 
146                           fk_host):
147        """
148        Create new record in portused.
149        """
150        debug("Inserting new portused into database")
151       
152        self.cursor.execute("INSERT INTO portused (portid, fk_port_state, \
153                        fk_protocol, fk_host) VALUES (?, ?, ?, ?)",
154                        (portid, fk_port_state, fk_protocol, fk_host))
155
156
157    def insert_osclass_db(self, osclass_accuracy, fk_osgen, fk_osfamily, 
158                          fk_osvendor, fk_ostype, fk_host):
159        """
160        Create new record in osclass.
161        """
162        debug("Inserting new osclass into database")
163       
164        self.cursor.execute("INSERT INTO osclass (accuracy, fk_osgen, \
165                        fk_osfamily, fk_osvendor, fk_ostype, fk_host) VALUES \
166                        (?, ?, ?, ?, ?, ?)", (osclass_accuracy, fk_osgen,
167                        fk_osfamily, fk_osvendor, fk_ostype, fk_host))
168
169   
170    def insert_osmatch_db(self, host, osmatch):
171        """
172        Create new record in osmatch with data from osmatch dict.
173        """
174        debug("Inserting new osmatch into database")
175       
176        osmatch["line"] = empty() # ToFix: Parser isnt storing this
177        self.cursor.execute("INSERT INTO osmatch (name, accuracy, line, \
178                    fk_host) VALUES (?, ?, ?, ?)", (osmatch["name"], 
179                    osmatch["accuracy"], osmatch["line"], host))
180       
181       
182    def insert_osgen_db(self, osgen):
183        """
184        Creates new record in osgen.
185        """
186        debug("Inserting new osgen into database")
187       
188        self.cursor.execute("INSERT INTO osgen (gen) VALUES (?)", (osgen, ))
189
190       
191    def insert_osfamily_db(self, osfamily):
192        """
193        Creates new record in osfamily.
194        """
195        debug("Inserting new osfamily into database")
196       
197        self.cursor.execute("INSERT INTO osfamily (family) VALUES (?)", 
198                        (osfamily, ))
199           
200
201    def insert_osvendor_db(self, osvendor):
202        """
203        Creates new record in osvendor.
204        """
205        debug("Inserting new osvendor into database")
206       
207        self.cursor.execute("INSERT INTO osvendor (vendor) VALUES (?)", 
208                        (osvendor, ))
209
210   
211    def insert_ostype_db(self, ostype):
212        """
213        Creates new record in ostype.
214        """
215        debug("Inserting new ostype into database")
216       
217        self.cursor.execute("INSERT INTO ostype (type) VALUES (?)", 
218                        (ostype, ))
219       
220
221    def insert_host_db(self, host_d):
222        """
223        Create new record in host with data from host dict.
224        """
225        debug("Inserting new host into database")
226
227        self.cursor.execute("INSERT INTO host (distance, fk_scan, \
228                     fk_host_state) VALUES (?, ?, ?)", (host_d["distance"],
229                     host_d["fk_scan"], host_d["fk_host_state"]))
230
231
232    def insert_host_address_db(self, fk_host, fk_address):
233        """
234        Creates new record in _host_address.
235        """
236        debug("Inserting new _host_address into database")
237       
238        self.cursor.execute("INSERT INTO _host_address (fk_host, fk_address) \
239                            VALUES (?, ?)", (fk_host, fk_address))
240
241
242    def insert_host_hostname_db(self, fk_host, fk_hostname):
243        """
244        Creates new record in _host_hostname.
245        """
246        debug("Inserting new _host_hostname into database")
247       
248        self.cursor.execute("INSERT INTO _host_hostname (fk_host, \
249                                fk_hostname) VALUES (?, ?)", (fk_host, 
250                                fk_hostname))
251   
252   
253    def insert_hostname_db(self, hostname):
254        """
255        Insert new record in hostnamed based on data from hostname.
256        """
257        debug("Inserting new hostname into database")
258       
259        self.cursor.execute("INSERT INTO hostname (type, name) VALUES \
260                                 (?, ?)", (hostname["hostname_type"], 
261                                           hostname["hostname"]))
262   
263
264    def insert_fingerprint_info_db(self, fp_d):
265        """
266        Creates new record in fingerprint_info with data from fp_d.
267        """
268        debug("Inserting new fingerprint information for host into database")
269
270        columns = ( "uptime", "lastboot", "tcp_sequence_class", 
271                    "tcp_sequence_index", "tcp_sequence_value",
272                    "tcp_sequence_difficulty", "tcp_ts_sequence_class",
273                    "tcp_ts_sequence_value", "ip_id_sequence_class",
274                    "ip_id_sequence_value", "fk_host" )
275
276        data = [fp_d[column] for column in columns]
277
278        self.cursor.execute("INSERT INTO fingerprint_info (uptime, lastboot, \
279                tcp_sequence_class, tcp_sequence_index, tcp_sequence_value, \
280                tcp_sequence_difficulty, tcp_ts_sequence_class, \
281                tcp_ts_sequence_value, ip_id_sequence_class, \
282                ip_id_sequence_value, fk_host) VALUES (?, ?, ?, ?, ?, ?, ?, \
283                ?, ?, ?, ?)", (data))
284
285   
286    def insert_service_name_db(self, service_name):
287        """
288        Creates new record in service_name.
289        """       
290        debug("Inserting new service_name into database")
291       
292        self.cursor.execute("INSERT INTO service_name (name) VALUES \
293                            (?)", (service_name, ))
294       
295       
296    def insert_service_info_db(self, service_data):
297        """
298        Creates new record in service_info based on service_data
299        """
300        debug("Inserting new service_info into database")
301       
302        self.cursor.execute("INSERT INTO service_info (product, version, \
303                    extrainfo, method, conf, fk_service_name) VALUES (?, ?, ?,\
304                    ?, ?, ?)", service_data)
305       
306   
307    def insert_address_db(self, address_addr, address_type, vendor):
308        """
309        Creates new record on address.
310        """
311        debug("Inserting new address into database")
312       
313        self.cursor.execute("INSERT INTO address (address, type, \
314                        fk_vendor) VALUES (?, ?, ?)", (address_addr,
315                        address_type, vendor))
316
317   
318    def insert_vendor_db(self, vendor_name):
319        """
320        Creates new record in  vendor.
321        """
322        debug("Inserting new vendor into database")
323       
324        self.cursor.execute("INSERT INTO vendor (name) \
325                                         VALUES (?)", (vendor_name, ))
326
327       
328    def insert_host_state_db(self, host_state):
329        """
330        Creates new record in host_state.
331        """
332        debug("Inserting new host_state into database")
333       
334        self.cursor.execute("INSERT INTO host_state (state) \
335                        VALUES (?)", (host_state, ))
336
337
338    def insert_tcp_sequence_db(self, tcpseq_dict):
339        """
340        Creates new record in tcp_sequence based on data from tcpseq_dict.
341        """
342        debug("Inserting new tcp_sequence into database")
343       
344        self.cursor.execute("INSERT INTO tcp_sequence (tcp_index, \
345                        class, difficulty, tcp_values) VALUES (?, ?, ?, ?)", (
346                        tcpseq_dict["index"], tcpseq_dict["class"], 
347                        tcpseq_dict["difficulty"], tcpseq_dict["values"]))
348
349
350    def insert_tcp_ts_sequence_db(self, tcptsseq_dict):
351        """
352        Creates new record in tcp_ts_sequence based on data from tcptsseq
353        dict.
354        """
355        debug("Inserting new tcp_ts_sequence into database")
356       
357        self.cursor.execute("INSERT INTO tcp_ts_sequence (class, \
358                      tcp_ts_values)  VALUES (?, ?)", (tcptsseq_dict["class"], 
359                      tcptsseq_dict["values"]))
360
361
362    def insert_ip_id_sequence_db(self, ipidseq_dict):
363        """
364        Creates new record in ip_id_sequence based on data from ipidseq dict.
365        """
366        debug("Inserting new ip_id_sequence into database")
367       
368        self.cursor.execute("INSERT INTO ip_id_sequence (class, \
369                      ip_id_values)  VALUES (?, ?)", (ipidseq_dict["class"], 
370                      ipidseq_dict["values"]))
371
372
373    def insert_inventory_db(self, inventory):
374        """
375        Creates new record in inventory.
376        """
377        debug("Inserting new inventory into database")
378
379        self.cursor.execute("INSERT INTO inventory (name) VALUES (?)", 
380                    (inventory, ))
381       
382   
383    def insert_inventory_scan_db(self, scan, inventory):
384        """
385        Creates new record in _inventory_scan.
386        """
387        debug("Inserting new _inventory_scan into database")
388
389        self.cursor.execute("INSERT INTO _inventory_scan (fk_scan, \
390                    fk_inventory) VALUES (?, ?)", (scan, inventory))
391       
392       
393    def insert_inventory_change_category_db(self, category):
394        """
395        Creates new record in inventory_change_category.
396        """
397        debug("Inserting new category '%s' into \
398inventory_change_category" % category)
399       
400        self.cursor.execute("INSERT INTO inventory_change_category (name) \
401                             VALUES (?)", (category, ))
402       
403   
404    def insert_inventory_comparison_db(self, old_hid, new_hid, date, 
405                                       short_descr, fk_inventory, fk_category,
406                                       fk_address):
407        """
408        Creates new record in _inventory_changes.
409        """
410        debug("Inserting new change into _inventory_changes")
411       
412        self.cursor.execute("INSERT INTO _inventory_changes (old_hostid, \
413                             new_hostid, entry_date, short_description, \
414                             fk_inventory, fk_category, fk_address) VALUES \
415                             (?, ?, ?, ?, ?, ?, ?)", (old_hid, new_hid, date,
416                                                      short_descr, fk_inventory,
417                                                      fk_category, fk_address))
418
Note: See TracBrowser for help on using the browser.