Changeset 5735

Timestamp:

07/12/10 21:23:58 (2 months ago)

Author:

diogo

Message:

zion: fingerprint matching with database

Location:

zion

Files:

• branches/clann/bind/setup.py (modified) (1 diff)
• branches/clann/bind/som.c (modified) (2 diffs)
• branches/clann/bind/som.h (modified) (2 diffs)
• branches/clann/bind/umit/clann/matrix.so (modified) (previous)
• branches/clann/bind/umit/clann/metric.so (modified) (previous)
• branches/clann/bind/umit/clann/som.so (modified) (previous)
• branches/clann/code/som.c (modified) (3 diffs)
• branches/clann/code/som.h (modified) (3 diffs)
• trunk/umit/scan/zion/gui/ZionScanNotebookPage.py (modified) (13 diffs)
• trunk/umit/zion/core/pmatrix.py (added)
• trunk/umit/zion/core/zion.py (modified) (6 diffs)
• trunk/umit/zion/db/db.sqlite (modified) (previous)

zion/branches/clann/bind/setup.py

@@ -30 +30 @@
                                         '../code/clann.o',
                                         '../code/metric.o',
-                                        '../code/matrix.o'],
+                                        '../code/matrix.o',
+                                        '../code/function.o'],
                        extra_compile_args = ['-Wall', '-ggdb'],
                        sources = ['som.c'])

zion/branches/clann/bind/som.c

@@ -112 +112 @@
      * Convert output
      */
-    return PyCObject_FromVoidPtr(n, (void *) delete_matrix);
+    return PyCObject_FromVoidPtr(n, NULL);
 }
 
@@ -142 +142 @@
 
     som_training(a, n, i);
+
+    /**
+     * Convert output
+     */
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+PyObject*
+classification(PyObject *self, PyObject *args)
+{
+    /**
+     * Convert input
+     */
+    PyObject *s = NULL,
+             *m = NULL;
+    unsigned int metric, method, i, j;
+    float limit_aux;
+    struct matrix ba, pa;
+    clann_type limit;
+
+
+    if (!PyArg_ParseTuple(args, "OOfI", &s, &m, &limit_aux, &method))
+        return NULL;
+
+    limit = (clann_type) limit_aux;
+
+    /**
+     * Call the function
+     */
+    struct matrix *b = (struct matrix *) PyCObject_AsVoidPtr(m);
+    struct som *a = (struct som *) PyCObject_AsVoidPtr(s);
+
+    if(method==1)
+    {
+
+        /* for hausdorff_limit only P and Pi are needed */
+        matrix_initialize(&pa, a->grid.weights.rows, a->grid.weights.cols);
+        matrix_initialize(&ba, b->rows, b->cols - 1);
+                
+        for (i = 0; i < a->grid.weights.rows; i++)
+        {
+            for (j = 0; j < a->grid.weights.cols - 1; j++)
+                *matrix_value(&pa, i, j) = *matrix_value(&a->grid.weights, i, j);
+            *matrix_value(&pa, i, a->grid.weights.cols-1) = a->grid.density[i];
+        }
+
+        for (i = 0; i < b->rows; i++)
+            for (j = 0; j < b->cols - 1; j++)
+                *matrix_value(&ba, i, j) = *matrix_value(b, i, j);
+                
+        metric = metric_hausdorff_limit_symmetric(&pa, &ba, limit);
+    }
+    else
+    {
+        matrix_initialize(&pa, a->grid.weights.rows, a->grid.weights.cols + 2);
+        
+        for (i = 0; i < a->grid.weights.rows; i++)
+            for (j = 0; j < a->grid.weights.cols - 1; j++)
+                *matrix_value(&pa, i, j) = *matrix_value(&(a->grid.weights), i, j);
+            *matrix_value(&pa, i, a->grid.weights.cols-1) = a->grid.density[i];
+            *matrix_value(&pa, i, a->grid.weights.cols) = a->grid.orientation[i];
+            
+        metric = metric_hausdorff_angle_symmetric(&pa, b, limit);
+    }
+    
+    /**
+     * Convert output
+     */
+    return Py_BuildValue("I", metric);
+}
+
+PyObject*
+caracterization(PyObject *self, PyObject *args)
+{
+    /**
+     * Convert input
+     */
+    PyObject *s = NULL,
+             *m = NULL;
+    unsigned int i;
+
+    if (!PyArg_ParseTuple(args, "OOI", &s, &m, &i))
+        return NULL;
+
+    /**
+     * Call the function
+     */
+    struct matrix *n = (struct matrix *) PyCObject_AsVoidPtr(m);
+    struct som *a = (struct som *) PyCObject_AsVoidPtr(s);
+
+    if (i < 1)
+    {
+        PyErr_SetString(PyExc_IndexError,
+                "number of iterations must be positive");
+        return NULL;
+    }
+
+    som_caracterization(a, n, i);
 
     /**

zion/branches/clann/bind/som.h

@@ -63 +63 @@
  *
  */
+static char caracterization__doc__[] = "Start training SOM and fingerpriting";
+
+static PyObject*
+caracterization(PyObject *self, PyObject *args);
+
+/**
+ *
+ */
+static char classification__doc__[] = "Database matching";
+
+static PyObject*
+classification(PyObject *self, PyObject *args);
+
+/**
+ *
+ */
 static PyMethodDef SOMMethods[] =
 {
@@ -69 +85 @@
     {"get", get, METH_VARARGS, get__doc__},
     {"train", train, METH_VARARGS, train__doc__},
+    {"caracterization", caracterization, METH_VARARGS, caracterization__doc__},
+    {"classification", classification, METH_VARARGS, classification__doc__},
     {NULL, NULL, 0, NULL}
 };

zion/branches/clann/code/som.c

@@ -21 +21 @@
 #include "som.h"
 
+clann_type *z,
+           *o,
+           sz = 0.07,
+           so = 0.5,
+           r = 0.07;
+
 
 void
@@ -76 +82 @@
     while (ann->epoch < epochs)
     {
+        printf("epoch: %d\n",ann->epoch);
         clann_shuffle(mess, x->rows);
 
@@ -192 +199 @@
     return 0;
 }
+
+void
+som_caracterization(struct som *ann,
+                   struct matrix *x,
+                   unsigned int epochs)
+{
+    clann_type n_scale[2], z_scale[2];
+    unsigned int i, j;
+    clann_type d, cx, cy, *w, *t, *p, *a, *b;
+
+    ann->grid.orientation = malloc(sizeof(clann_type) * ann->grid.weights.rows);
+    ann->grid.density = malloc(sizeof(clann_type) * ann->grid.weights.rows);
+
+    for (i = 0; i < ann->grid.weights.rows; i++)
+    {
+        ann->grid.orientation[i] = 0;
+        ann->grid.density[i] = 0;
+    }
+
+    n_scale[MIN] = (clann_type) -1.0;
+    n_scale[MAX] = (clann_type)  1.0;
+
+    printf("Training SOM\n");
+    som_training(ann, x, epochs);
+
+    t = malloc(sizeof(clann_type) * ann->grid.weights.rows);
+
+    for (i = 0; i < ann->grid.weights.rows; i++)
+        t[i] = 0;
+
+    /**
+     * Density
+     */
+    printf("Calculating density\n");
+    for (i = 0; i < x->rows; i++)
+    {
+        p = matrix_value(x, i, 0);
+
+        z_scale[MIN] = (clann_type) INT_MAX;
+        z_scale[MAX] = (clann_type) - INT_MAX;
+
+        for (j = 0; j < ann->grid.weights.rows; j++)
+        {
+            w = matrix_value(&ann->grid.weights, j, 0);
+
+            d = CLANN_POW(metric_euclidean(p, w, 2), 2.0);
+            d = function_green_gaussian(&sz, &d);
+
+            t[j] += d;
+
+            if (t[j] < z_scale[MIN])
+                z_scale[MIN] = t[j];
+
+            if (t[j] > z_scale[MAX])
+                z_scale[MAX] = t[j];
+        }
+
+        for (j = 0; j < ann->grid.weights.rows; j++)
+            ann->grid.density[j] = metric_scale(t[j], z_scale, n_scale);
+    }
+
+    /**
+     * Orientation
+     */
+    printf("Calculating orientation\n");
+    for (j = 0; j < ann->grid.weights.rows; j++)
+    {
+        w = matrix_value(&ann->grid.weights, j, 0);
+
+        cx = 0;
+        cy = 0;
+
+        for (i = 0; i < x->rows - 1; i++)
+        {
+            a = matrix_value(x, i, 0);
+            b = matrix_value(x, i + 1, 0);
+
+            d = CLANN_POW(metric_euclidean(a, w, 2), 2.0);
+            d = function_green_gaussian(&so, &d);
+
+            cx += (b[X] - a[X]) * d;
+            cy += (b[Y] - a[Y]) * d;
+        }
+
+        ann->grid.orientation[j] = CLANN_ATAN2(cy, cx);
+    }
+
+    /**
+     * Normalization
+     */
+    printf("Normalization\n");
+    clann_type x_scale[2], y_scale[2];
+
+    x_scale[MIN] = (clann_type) INT_MAX;
+    x_scale[MAX] = (clann_type) - INT_MAX;
+    y_scale[MIN] = (clann_type) INT_MAX;
+    y_scale[MAX] = (clann_type) - INT_MAX;
+
+    for (j = 0; j < ann->grid.weights.rows; j++)
+    {
+        w = matrix_value(&ann->grid.weights, j, 0);
+
+        //if (z[j] < MIN_DENSITY)
+        //    continue;
+
+        if (w[X] < x_scale[MIN])
+            x_scale[MIN] = w[X];
+
+        if (w[X] > x_scale[MAX])
+            x_scale[MAX] = w[X];
+
+        if (w[Y] < y_scale[MIN])
+            y_scale[MIN] = w[Y];
+
+        if (w[Y] > y_scale[MAX])
+            y_scale[MAX] = w[Y];
+    }
+
+    for (j = 0; j < ann->grid.weights.rows; j++)
+    {
+        w = matrix_value(&ann->grid.weights, j, 0);
+
+        w[X] = metric_scale(w[X], x_scale, n_scale);
+        w[Y] = metric_scale(w[Y], y_scale, n_scale);
+    }
+}

zion/branches/clann/code/som.h

@@ -32 +32 @@
 #include "reader.h"
 #include "clann.h"
+#include "function.h"
+
+#define MAX         0
+#define MIN         1
+
+#define X           0
+#define Y           1
+
+#define MIN_DENSITY -0.8
 
 
@@ -42 +51 @@
     unsigned int y_len;
     struct matrix weights;
+    clann_type *density;
+    clann_type *orientation;
 };
 
@@ -83 +94 @@
  *
  */
+inline void
+som_caracterization(struct som *ann,
+                    struct matrix *x,
+                    unsigned int epochs);
+
+/**
+ *
+ */
 inline void 
 som_adjust_weights(struct som *ann,

zion/trunk/umit/scan/zion/gui/ZionScanNotebookPage.py

@@ -23 +23 @@
 import gobject
 import netifaces
+import thread
 
 from higwidgets.higframe import HIGFrameRNet
@@ -43 +44 @@
 
 ICON_DIR = 'share/pixmaps/zion/'
+ICON_DIR_UMIT = 'share/pixmaps/umit/'
 
 PIXBUF_FIREWALL = gtk.gdk.pixbuf_new_from_file(ICON_DIR + 'firewall.png')
 PIXBUF_SYNPROXY = gtk.gdk.pixbuf_new_from_file(ICON_DIR + 'shield.png')
 PIXBUF_HONEYD = gtk.gdk.pixbuf_new_from_file(ICON_DIR + 'honey.png')
+PIXBUF_UNKNOWN = gtk.gdk.pixbuf_new_from_file(ICON_DIR_UMIT + 'unknown_32.png')
+
+SCANNING = _("Scanning")
 
 class ZionHostsView(gtk.Notebook):
@@ -71 +76 @@
 
         self.append_page(self.__scans_page, gtk.Label(_('Scans')))
+        self.append_page(self.__ident_page, gtk.Label(_('Identification')))
         self.append_page(self.__ports_page, gtk.Label(_('Ports')))
-        self.append_page(self.__ident_page, gtk.Label(_('Identification')))
         
         self.__ports_page.add(self.open_ports)
@@ -83 +88 @@
         """
         return self.__ident_page
+    
+    def get_scans_page(self):
+        """
+        """
+        return self.__scans_page
 
 class ZionScansPage(HIGVBox):
@@ -91 +101 @@
         """
         HIGVBox.__init__(self)
+        
+        # Creating widgets
+        self.__create_widgets()
+        
+        # Setting scrolled window
+        self.__set_scrolled_window()
+        
+        # Setting text view
+        self.__set_text_view()
+        
+        # Getting text buffer
+        self.text_buffer = self.text_view.get_buffer()
+        
+        # Adding widgets to the VPaned
+        self._pack_expand_fill(self.scrolled)
+        
+    def __create_widgets (self):
+        # Creating widgets
+        self.scrolled = gtk.ScrolledWindow()
+        self.text_view = gtk.TextView()
+        
+    def __set_scrolled_window(self):
+        # By default the vertical scroller remains at bottom
+        self._scroll_at_bottom = True
+
+        # Seting scrolled window
+        self.scrolled.set_border_width(5)
+        self.scrolled.add(self.text_view)
+        self.scrolled.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
+        
+    def __set_text_view(self):
+        self.text_view.set_wrap_mode(gtk.WRAP_WORD)
+        self.text_view.set_editable(False)
+        
+    def write(self, text):
+        self.text_buffer.insert(self.text_buffer.get_end_iter(), text)
 
 class ZionPortsPage(HIGVBox):
@@ -149 +195 @@
         self.__cell_pixbuf = gtk.CellRendererPixbuf()
 
-        self.__hosts_store = gtk.ListStore(gtk.gdk.Pixbuf,
-                                           gobject.TYPE_STRING)
+        self.__hosts_store = gtk.ListStore(gtk.gdk.Pixbuf, str)
 
         self.__hosts_treeview = gtk.TreeView(self.__hosts_store)
@@ -164 +209 @@
                                                 text=1)
 
-        """self.__hosts_store.append([PIXBUF_FIREWALL,
-            'firewall.example.com\n192.0.2.1'])
-        self.__hosts_store.append([PIXBUF_SYNPROXY,
-            'synproxy.example.com\n192.0.2.2'])
-        self.__hosts_store.append([PIXBUF_HONEYD,
-            'honeyd.example.com\n192.0.2.3'])"""
-
         self.__column_type.set_reorderable(True)
         self.__column_type.set_resizable(False)
@@ -208 +246 @@
         """
         """
-        self.__hosts_store = gtk.ListStore(gtk.gdk.Pixbuf,
-                                           gobject.TYPE_STRING)
+        for i in range(len(self.__hosts_store)):
+            iter = self.__hosts_store.get_iter_root()
+            del(self.__hosts_store[iter])
         
     def add_host(self, name, host_type=None):
@@ -215 +254 @@
         """
         self.__hosts_store.append([host_type,name])
+        pass
 
 class ZionResultsPage(gtk.HPaned):
@@ -355 +395 @@
         
         self.result.clear_port_list()
-                
+        
+        # clear previous hosts in the list
+        self.result.get_hosts_list().clear_hosts()
+        
+        # verify address to scan
         if address.recognize(self.target) == address.Unknown:
             l = probe.get_addr_from_name(self.target)
@@ -361 +405 @@
                 try:
                     z.append_target(host.Host(i, self.target))
+                    host_str = '%s\n%s' % (i, self.target)
                 except:
                     print "Unimplemented support to address: %s." % i
         else:
-            z.append_target(host.Host(self.target))      
-        
+            z.append_target(host.Host(self.target))
+            self.result.get_hosts_list().add_host(self.target)
+            
+        self.result.get_hosts_list().add_host(host_str)
+
+        # configure zion options
         device = get_default_device()
         saddr = get_ip_address(device)
@@ -372 +421 @@
         z.get_option_object().add("--forge-addr",saddr)
         z.run()
+        #z.start()
         
         # update host information
@@ -560 +610 @@
 
     # TODO: read device from options
-    device = "wlan0"
+    device = "eth0"
     #device = netifaces.interfaces()[0]
     return device

zion/trunk/umit/zion/core/zion.py

@@ -24 +24 @@
 import random
 import time
+import thread, threading
+import sqlite3
+import sys
+import json
+import cPickle
 from math import sqrt
 
 from umit.clann import som, matrix
-from umit.zion.core import options, host
+from umit.zion.core import options, host, pmatrix
 from umit.zion.scan import sniff, portscan, forge
 
 FORGE_FILTER = 'src host %s and src port %s and dst host %s and dst port %s'
-AMOUNT_OS_DETECTION = 2000
+AMOUNT_OS_DETECTION = 50
 AMOUNT_HONEYD_DETECTION = 25
 SEND_INTERVAL = 0.1
 SYNPROXY_INTERVAL = 1
-
-class Zion(object):
+METHOD_ALPHA = 1
+METHOD_BETA = 2
+EPOCHS = 500#1800
+ALPHA_LIMIT = 0.1
+
+class Zion(threading.Thread):
     """
     """
@@ -46 +55 @@
         self.__capture_result = []
         self.__attractors = []
+        threading.Thread.__init__ (self)
 
     def get_option_object(self):
@@ -173 +183 @@
         """
         """
-        self.synproxy_detection(self.__target[0])
         if self.__option.has(options.OPTION_HELP):
 
@@ -215 +224 @@
             print 'Creating attractors'
             self.__classification(Rt)
+            
+            print 'Matching'
+            return self.__matching()
                 
         elif self.__option.has(options.OPTION_CAPTURE):
@@ -300 +312 @@
         isn3 = self.__capture_result[0][1]
         
-        print 'isns:'
-        print isn1
-        print isn2
-        print isn3
-        
         if isn1!=isn2 and isn1==isn3:
             return True
@@ -351 +358 @@
         ratio = 2/(max_val-min_val)
         
-        self.__som = som.new(10,(30,30))
+        self.__som = som.new(2,(30,30))
         self.__matrix = matrix.new(len(Rt)-1,2)
-        
+                
         for i in range(len(Rt)-1):
-            x = (Rt[i+1]-min_val)*ratio-1
-            y = (Rt[i]-min_val)*ratio-1
+            x = Rt[i+1]
+            y = Rt[i]
             self.__attractors.append((x, y))
             matrix.set(self.__matrix, i, 0, x)
             matrix.set(self.__matrix, i, 1, y)
 
-        # TODO: confirm how train works
-        som.train(self.__som, self.__matrix, 1800)        
-
+        som.caracterization(self.__som, self.__matrix, EPOCHS)
+    
+    def __matching(self):
+        """
+        Match fingerprint with database
+        """
+        dmin = sys.maxint
+        id_min = None
+        
+        conn = sqlite3.connect('umit/zion/db/db.sqlite')
+        c = conn.cursor()
+        c.execute('SELECT software.pk, s_attractor.fp FROM software INNER JOIN fingerprint ON software.pk = fingerprint.fk_software INNER JOIN s_attractor ON s_attractor.pk = fingerprint.fk_sig1')
+        
+        for fingerprint in c:
+            attractor = cPickle.loads(str(fingerprint[1]))
+            base = attractor.convert()
+            d = som.classification(self.__som, base, ALPHA_LIMIT, METHOD_ALPHA)
+            if d < dmin:
+                dmin = d
+                id_min = fingerprint[0]
+                        
+        details = None
+        if id_min!=None:
+            for vendor_name, os_name, os_version in c.execute('SELECT vendor.name, name.name, name.version FROM software INNER JOIN vendor ON software.fk_vendor = vendor.pk INNER JOIN name ON software.fk_name = name.pk WHERE software.pk = ?',(id_min,)):
+                details = {'vendor_name': vendor_name, 'os_name': os_name, 'os_version': os_version, 'metric': dmin}
+                print 'Vendor name: %s\nOS name: %s\nOS version: %s\nMetric: %d' % (vendor_name, os_name, os_version, dmin)
+        else:
+            print 'no fingerprints available in database'
+        
+        return details
         
     def get_attractors(self):
-        """
-        """
+        """ Return the list of attractors. """
         return self.__attractors
+