Added functions for Aho-Corasick Multi-Pattern Search Engine and

implemented the option for using this algorithm trackers that use 
multiple strings.

errors.mapi: added an error for no memory.


git-svn-id: file:///home/svn/mapi/trunk@750 8d5bb341-7cf1-0310-8cf6-ba355fef3186

src/common/Makefile.am

 ## Process this file with automake to produce Makefile.in
 AM_CPPFLAGS = -I.. -I../lib -I../drivers -D_GNU_SOURCE -D_THREAD_SAFE
 
-noinst_LTLIBRARIES = libflist.la libfhelp.la libmapiipc.la libparseconf.la libprintfstring.la libmapilibhandler.la libmsearch.la
+noinst_LTLIBRARIES = libflist.la libfhelp.la libmapiipc.la libparseconf.la libprintfstring.la libmapilibhandler.la libmsearch.la libacsmx2.la
 libflist_la_SOURCES = flist.c flist.h
 libflist_la_DEPENDENCIES = mapi_errors.h
 libfhelp_la_SOURCES = fhelp.c fhelp.h
@@ -10,6 +10,7 @@ libparseconf_la_SOURCES = parseconf.c parseconf.h
 libprintfstring_la_SOURCES = printfstring.c printfstring.h
 libmapilibhandler_la_SOURCES = mapilibhandler.c mapilibhandler.h
 libmsearch_la_SOURCES = mstring.c mstring.h
+libacsmx2_la_SOURCES = acsmx2.c acsmx2.h
 
 noinst_HEADERS = mapi_errors.h
 

src/common/acsmx2.h

+/*
+**   ACSMX2.H 
+**
+**   Version 2.0
+**
+**   Copyright (C) 2002,2003,2004 Marc Norton
+**   Copyright (C) 2003,2004 Sourcefire,Inc.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef ACSMX2S_H
+#define ACSMX2S_H
+
+#ifdef WIN32
+
+#ifdef inline
+#undef inline
+#endif
+
+#define inline __inline
+
+#else
+#ifdef inline
+#undef inline
+#endif
+
+#define inline 
+#endif
+
+/*
+*   DEFINES and Typedef's
+*/
+#define MAX_ALPHABET_SIZE 256     
+
+/*
+   FAIL STATE for 1,2,or 4 bytes for state transitions
+
+   Uncomment this define to use 32 bit state values
+   #define AC32
+*/
+
+/* #define AC32 */
+
+#ifdef AC32
+
+typedef  unsigned int   acstate_t;
+#define ACSM_FAIL_STATE2  0xffffffff
+
+#else
+
+typedef    unsigned short acstate_t;
+#define ACSM_FAIL_STATE2 0xffff
+
+#endif
+
+/*
+*
+*/
+typedef 
+struct _acsm_pattern2
+{      
+    struct  _acsm_pattern2 *next;
+
+    unsigned char         *patrn;
+    unsigned char         *casepatrn;
+    int      n;
+    int      nocase;
+    int      offset;
+    int      depth;
+    void *   id;
+    int      iid;
+
+} ACSM_PATTERN2;
+
+/*
+*    transition nodes  - either 8 or 12 bytes
+*/
+typedef 
+struct trans_node_s {
+
+  acstate_t    key;           /* The character that got us here - sized to keep structure aligned on 4 bytes */
+                              /* to better the caching opportunities. A value that crosses the cache line */
+                              /* forces an expensive reconstruction, typing this as acstate_t stops that. */
+  acstate_t    next_state;    /*  */
+  struct trans_node_s * next; /* next transition for this state */
+
+} trans_node_t;
+
+
+/*
+*  User specified final storage type for the state transitions
+*/
+enum {
+  ACF_FULL,
+  ACF_SPARSE,
+  ACF_BANDED,
+  ACF_SPARSEBANDS,
+};
+
+/*
+*   User specified machine types
+*
+*   TRIE : Keyword trie
+*   NFA  : 
+*   DFA  : 
+*/
+enum {
+  FSA_TRIE,
+  FSA_NFA,
+  FSA_DFA,
+};
+
+/*
+*   Aho-Corasick State Machine Struct - one per group of pattterns
+*/
+typedef struct {
+  
+	int acsmMaxStates;  
+	int acsmNumStates;  
+
+	ACSM_PATTERN2    * acsmPatterns;
+        acstate_t        * acsmFailState;
+        ACSM_PATTERN2   ** acsmMatchList;
+
+        /* list of transitions in each state, this is used to build the nfa & dfa */
+        /* after construction we convert to sparse or full format matrix and free */
+        /* the transition lists */
+        trans_node_t ** acsmTransTable;
+
+        acstate_t ** acsmNextState;
+        int          acsmFormat;
+        int          acsmSparseMaxRowNodes;
+        int          acsmSparseMaxZcnt;
+        
+	int          acsmNumTrans;
+        int          acsmAlphabetSize;
+        int          acsmFSA;
+
+}ACSM_STRUCT2;
+
+/*
+*   Prototypes
+*/
+ACSM_STRUCT2 * acsmNew2 ();
+int acsmAddPattern2( ACSM_STRUCT2 * p, unsigned char * pat, int n,
+                    int nocase, int offset, int depth, void *  id, int iid );
+int acsmCompile2 ( ACSM_STRUCT2 * acsm );
+int acsmSearch2 ( ACSM_STRUCT2 * acsm,unsigned char * T, int n, 
+		  int (*Match)( void * id, int index, void * data ),
+                  void * data );
+void acsmFree2 ( ACSM_STRUCT2 * acsm );
+
+
+int  acsmSelectFormat2( ACSM_STRUCT2 * acsm, int format );
+int  acsmSelectFSA2( ACSM_STRUCT2 * acsm, int fsa );
+
+void acsmSetMaxSparseBandZeros2( ACSM_STRUCT2 * acsm, int n );
+void acsmSetMaxSparseElements2( ACSM_STRUCT2 * acsm, int n );
+int  acsmSetAlphabetSize2( ACSM_STRUCT2 * acsm, int n );
+void acsmSetVerbose2(int n);
+
+void acsmPrintInfo2( ACSM_STRUCT2 * p);
+
+int acsmPrintDetailInfo2(ACSM_STRUCT2*);
+int acsmPrintSummaryInfo2();
+
+#endif

src/errors.mapi

@@ -134,5 +134,5 @@
 8002 = "Communication error with admission control"
 #define ADMCTRL_AUTH_FAILED 8003
 8003 = "Authorization failed"
-#define MAPID_NOT_AUTHENTICATED 3139
-3139 = "Flow not authenticated"
+#define MAPID_MEM_ALLOCATION_ERROR 3084
+3084 = "Error Allocating Memory"

src/trackflib/Makefile.am

@@ -6,7 +6,7 @@ libdir = $(pkgdatadir)
 # libraries for users)
 lib_LTLIBRARIES = trackflib.la
 trackflib_la_LDFLAGS = -module
-trackflib_la_LIBADD = ../common/libflist.la ../common/libmsearch.la
+trackflib_la_LIBADD = ../common/libflist.la ../common/libmsearch.la ../common/libacsmx2.la
 trackflib_la_SOURCES = \
 dc.c \
 edonkey.c edonkey.h \

src/trackflib/dc.c

@@ -14,6 +14,9 @@
 #include "debug.h"
 #include "mapiipc.h"
 #include "mstring.h"
+#include "acsmx2.h"
+#include "mapi_errors.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <netinet/in.h>
@@ -75,15 +78,22 @@ int dc_string_len[STRING_NUM] = {0};
 ///int isDc(mapidflib_function_instance_t *, unsigned char *, int );
 
 struct mapid_dc {
+#ifndef __WITH_AHO__
 	int *shift[STRING_NUM];
 	int *skip[STRING_NUM];
+#else 
+	ACSM_STRUCT2 *acsm;
+#endif
 	struct list **dclist;
 };
 
 static int dc_init(mapidflib_function_instance_t *instance, MAPI_UNUSED int fd)
 {
 	int i=0;	
-	
+#ifdef __WITH_AHO__
+	char *p;
+#endif
+
 	instance->internal_data = malloc(sizeof(struct mapid_dc));
 	((struct mapid_dc*)instance->internal_data)->dclist = (struct list**)malloc(sizeof(struct list*)*HASHTABLESIZE);
 	memset(((struct mapid_dc*)instance->internal_data)->dclist, 0, (sizeof(struct list*)*HASHTABLESIZE));
@@ -92,18 +102,53 @@ static int dc_init(mapidflib_function_instance_t *instance, MAPI_UNUSED int fd)
 		((struct mapid_dc*)instance->internal_data)->dclist[i]->head = NULL;
 		((struct mapid_dc*)instance->internal_data)->dclist[i]->tail = NULL;
 	}
+#ifndef __WITH_AHO__
 	for(i=0;i<STRING_NUM;i++) {
 		((struct mapid_dc*)instance->internal_data)->shift[i] = make_shift(dc_strings[i],strlen(dc_strings[i]));
 		((struct mapid_dc*)instance->internal_data)->skip[i] = make_skip(dc_strings[i], strlen(dc_strings[i]));
 	}
+#else
+	((struct mapid_dc*)instance->internal_data)->acsm = acsmNew2();
 
+	if(!(((struct mapid_dc*)instance->internal_data)->acsm)) {
+		return MAPID_MEM_ALLOCATION_ERROR;
+	}
+
+	for(i = 0; i< STRING_NUM; i++) {
+		p = dc_strings[i];
+
+		acsmAddPattern2(((struct mapid_dc*)instance->internal_data)->acsm, p, strlen(p), 1, 0, 0, (void*)p, i);
+	}
+	
+	acsmCompile2(((struct mapid_dc*)instance->internal_data)->acsm);
+#endif
 	return 0;
 }	
 
+#ifdef __WITH_AHO__
+
+static int global_index = -1;
+static char *found = NULL;
+
+	int dc_matchFound(void* id, int my_index, MAPI_UNUSED void *data) 
+	{
+  		global_index = my_index;
+		found = (char *)id;
+
+		return my_index;
+	}
+#endif
+
 int isDc(mapidflib_function_instance_t *instance, unsigned char *pkt, unsigned int len)
 {
+#ifndef __WITH_AHO__
 	int i=0;
+#else
+	global_index = -1;
+	found = NULL;
+#endif
 
+#ifndef __WITH_AHO__
 	for(i=0;i<STRING_NUM;i++) {
 		if(len < strlen(dc_strings[i]))
 				continue;
@@ -125,6 +170,12 @@ int isDc(mapidflib_function_instance_t *instance, unsigned char *pkt, unsigned i
 			}
 		}
 	}
+#else 
+	acsmSearch2(((struct mapid_dc*)instance->internal_data)->acsm, pkt, len, dc_matchFound, (void *)0);
+	
+	return global_index;
+
+#endif 
 	return -1;
 
 }
@@ -315,7 +366,11 @@ static int dc_process(mapidflib_function_instance_t *instance,
 		new->sp = sp;
 		new->dp = dp;
 #ifdef __TRACKFLIB_LOGGING__
+	#ifndef __WITH_AHO__
 		write_to_log("DC++", dc_strings[i], iph->protocol, source, sp, dest, dp, p_b, len);
+	#else
+		write_to_log("DC++", found, iph->protocol, source, sp, dest, dp, p_b, len);
+	#endif 
 #endif
 //		printf("DC++ %s %d \n\n",dc_strings[i], iph->protocol);
 		

src/trackflib/gnutella.c

@@ -14,6 +14,9 @@
 #include "debug.h"
 #include "mapiipc.h"
 #include "mstring.h"
+#include "acsmx2.h"
+#include "mapi_errors.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <netinet/in.h>
@@ -61,8 +64,12 @@ struct list{
 char *gnutella_strings[GNUTELLA_STRING_NO]={"GET /uri-res/N2R?urn:sha1:","GNUTELLA CONNECT/","GNUTELLA/", "Server: LimeWire/"};//,"GET /get/"};
 
 struct mapid_gnutella {
+#ifndef __WITH_AHO__
 	int *shift[GNUTELLA_STRING_NO];
 	int *skip[GNUTELLA_STRING_NO];
+#else
+	ACSM_STRUCT2 *acsm;
+#endif 
 	struct list **gnulist;
 };
 
@@ -71,6 +78,9 @@ int isGnutella(mapidflib_function_instance_t *,  unsigned char *, unsigned int )
 static int gnutella_init(mapidflib_function_instance_t *instance, MAPI_UNUSED int fd)
 {
 	int i=0;
+#ifdef __WITH_AHO__
+	char *p;
+#endif
 
 	instance->internal_data = malloc(sizeof(struct mapid_gnutella));
 	((struct mapid_gnutella*)instance->internal_data)->gnulist = (struct list**)malloc(sizeof(struct list *)*HASHTABLESIZE);
@@ -80,18 +90,47 @@ static int gnutella_init(mapidflib_function_instance_t *instance, MAPI_UNUSED in
 		((struct mapid_gnutella*)instance->internal_data)->gnulist[i]->head = NULL;
 		((struct mapid_gnutella*)instance->internal_data)->gnulist[i]->tail = NULL;
 	}
+#ifndef __WITH_AHO__
 	for(i=0;i<GNUTELLA_STRING_NO;i++) {
 		((struct mapid_gnutella*)instance->internal_data)->shift[i] = make_shift(gnutella_strings[i],strlen(gnutella_strings[i]));
 		((struct mapid_gnutella*)instance->internal_data)->skip[i] = make_skip(gnutella_strings[i], strlen(gnutella_strings[i]));
 	}
+#else 
+	((struct mapid_gnutella*)instance->internal_data)->acsm = acsmNew2();
+
+	if(!(((struct mapid_gnutella*)instance->internal_data)->acsm)) {
+		return MAPID_MEM_ALLOCATION_ERROR;
+	}
+	for(i = 0; i < GNUTELLA_STRING_NO; i++) {
+		p = gnutella_strings[i];
+
+		acsmAddPattern2(((struct mapid_gnutella*)instance->internal_data)->acsm, p, strlen(p), 1, 0, 100,(void*)p, i);
+	}
 
+	acsmCompile2(((struct mapid_gnutella*)instance->internal_data)->acsm);
+#endif
 	return 0;
 }	
 
+#ifdef __WITH_AHO__
+
+static int global_index = -1;
+static char *found = NULL;
+
+	int gnutella_matchFound(void* id, int my_index, MAPI_UNUSED void *data) 
+	{
+  		global_index = my_index;
+		found = (char *)id;
+
+		return my_index;
+	}
+#endif
+
 int isGnutella(mapidflib_function_instance_t *instance, unsigned char *pkt, unsigned int len)
 {
+#ifndef __WITH_AHO__
 	int i=0;
-	
+
 	for(i=0;i<GNUTELLA_STRING_NO;i++) {
 		if(len < strlen(gnutella_strings[i]))
 				continue;
@@ -111,6 +150,17 @@ int isGnutella(mapidflib_function_instance_t *instance, unsigned char *pkt, unsi
 			}
 		}
 	}
+
+#else
+	global_index = -1;
+	found = NULL;
+
+	acsmSearch2(((struct mapid_gnutella*)instance->internal_data)->acsm, pkt, len, gnutella_matchFound, (void *)0);
+	
+	return global_index;
+
+#endif
+	
 	return -1;
 
 }
@@ -269,7 +319,11 @@ static int gnutella_process(mapidflib_function_instance_t *instance,
 		new->dp = dp;
 
 #ifdef __TRACKFLIB_LOGGING__
+	#ifndef __WITH_AHO__
 		write_to_log("GNUTELLA", gnutella_strings[i], iph->protocol, source, sp, dest, dp, pkt, len); 
+	#else
+		write_to_log("GNUTELLA", found, iph->protocol, source, sp, dest, dp, pkt, len); 
+	#endif
 #endif
 /*	
 		fprintf(fp, "%d---%s:%d -> ", new->protocol, inet_ntoa(source), sp);

src/trackflib/log.h

 #ifndef __TRACKER_LOG__
 	#define __TRACKER_LOG__
 
-//#define __TRACKFLIB_LOGGING__ 
+#define __TRACKFLIB_LOGGING__ 
 
 #include <sys/socket.h>
 #include <netinet/in.h>

src/trackflib/torrent.c

@@ -6,6 +6,7 @@
 #include <sys/shm.h>
 #include <string.h>
 #include <errno.h>
+
 #include "mapidflib.h"
 #include "mapidlib.h"
 #include "mapidevices.h"
@@ -14,6 +15,9 @@
 #include "debug.h"
 #include "mapiipc.h"
 #include "mstring.h"
+#include "acsmx2.h"
+#include "mapi_errors.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <netinet/in.h>
@@ -41,6 +45,7 @@
 #include "trackflib.h"
 #include "edonkey.h"
 
+
 struct filters {
 	int protocol;
 	unsigned int saddr;
@@ -68,8 +73,12 @@ int torrent_lens[TOR_STRINGS_NO]={20, 100, 100, 20, 20, 50, 50, 50, 50, 50, 50,
 int isTorrent(mapidflib_function_instance_t *, unsigned char *, unsigned int );
 
 struct mapid_torrent {
+#ifndef __WITH_AHO__
 	int *shift[TOR_STRINGS_NO];
 	int *skip[TOR_STRINGS_NO];
+#else
+	ACSM_STRUCT2 *acsm;
+#endif
 	unsigned int search_len[TOR_STRINGS_NO]; 
 	struct list **torlist;
 };
@@ -77,7 +86,9 @@ struct mapid_torrent {
 static int torrent_init(mapidflib_function_instance_t *instance, MAPI_UNUSED int fd)
 {
 	int i=0;
-
+#ifdef __WITH_AHO__
+	char *p;
+#endif
 	instance->internal_data = malloc(sizeof(struct mapid_torrent));
 	((struct mapid_torrent*)instance->internal_data)->torlist = (struct list**)malloc(sizeof(struct list*)*HASHTABLESIZE);
 
@@ -86,22 +97,66 @@ static int torrent_init(mapidflib_function_instance_t *instance, MAPI_UNUSED int
 		((struct mapid_torrent*)instance->internal_data)->torlist[i]->head = NULL;
 		((struct mapid_torrent*)instance->internal_data)->torlist[i]->tail = NULL;
 	}
+
+#ifndef __WITH_AHO__
+	
 	for(i=0;i<TOR_STRINGS_NO;i++) {
 		((struct mapid_torrent*)instance->internal_data)->shift[i] = make_shift(torrent_strings[i],strlen(torrent_strings[i]));
 		((struct mapid_torrent*)instance->internal_data)->skip[i] = make_skip(torrent_strings[i], strlen(torrent_strings[i]));
 		((struct mapid_torrent*)instance->internal_data)->search_len[i] = torrent_lens[i];
 	}
+#else
+	((struct mapid_torrent*)instance->internal_data)->acsm = acsmNew2();
+	
+	if(!(((struct mapid_torrent*)instance->internal_data)->acsm)) {
+		return MAPID_MEM_ALLOCATION_ERROR;
+	}
+	
+	for (i = 1; i < TOR_STRINGS_NO; i++) {
+		p = torrent_strings[i];
+		
+	//	int acsmAddPattern2 (ACSM_STRUCT2 * p, unsigned char *pat, int n, int nocase,
+	//		int offset, int depth, void * id, int iid) 
+		printf("torrent_lens[%d] = %d\n", i, torrent_lens[i]);
+		acsmAddPattern2(((struct mapid_torrent*)instance->internal_data)->acsm, p, strlen(p), 1, 0, torrent_lens[i],(void*)p, i);
+	}
+
+	acsmCompile2(((struct mapid_torrent*)instance->internal_data)->acsm);
+
+#endif
 
 	return 0;
 }	
 
+
+#ifdef __WITH_AHO__
+
+static int global_index = -1;
+static char *found = NULL;
+
+	int torrent_matchFound(void* id, int my_index, MAPI_UNUSED void *data) 
+	{
+		printf("found %s index %d\n", (char *)id, my_index);
+  		global_index = my_index;
+		found = (char *)id;
+
+		return my_index;
+	}
+#endif
+
 int isTorrent(mapidflib_function_instance_t *instance, unsigned char *pkt, unsigned int len)
 {
+#ifndef __WITH_AHO__
 	int i=0;
+#else
+	global_index = -1;
+	found = NULL;
+#endif
 
 	if(pkt[0] == 19 && (memcmp(&pkt[1], torrent_strings[0], 19) == 0))
 		return 0;
 
+#ifndef __WITH_AHO__
 	for(i=1;i<TOR_STRINGS_NO;i++) {
 		if(len < strlen(torrent_strings[i]))
 				continue;
@@ -121,6 +176,17 @@ int isTorrent(mapidflib_function_instance_t *instance, unsigned char *pkt, unsig
 			}
 		}
 	}
+#else 
+
+//	int acsmSearch2(ACSM_STRUCT2 * acsm, unsigned char *Tx, int n,
+//			int (*Match) (void * id, int index, void *data), void *data) 
+
+	acsmSearch2(((struct mapid_torrent*)instance->internal_data)->acsm, pkt, len, torrent_matchFound, (void *)0);
+	
+	return global_index;
+	
+
+#endif
 	return -1;
 
 }
@@ -325,7 +391,17 @@ static int torrent_process(mapidflib_function_instance_t *instance,
 		new->dp = dp;
 		new->byte_counter = 0;
 #ifdef __TRACKFLIB_LOGGING__
+	#ifndef __WITH_AHO__
 		write_to_log("BitTorrent", torrent_strings[i], iph->protocol, source, sp, dest, dp, p, len);
+	#else
+		if(global_index == -1) {
+			write_to_log("BitTorrent", torrent_strings[0], iph->protocol, source, sp, dest, dp, p, len);
+		}
+		else {
+			write_to_log("BitTorrent", found, iph->protocol, source, sp, dest, dp, p, len);
+		}
+		
+	#endif
 #endif
 /*	
 		if(sp == 22 || dp == 22) {	
@@ -379,6 +455,10 @@ static int torrent_cleanup(mapidflib_function_instance_t *instance)
 			  free(tmp);
 		  }
 	  }
+#ifndef __WITH_AHO__
+#else
+	  acsmFree2(((struct mapid_torrent*)instance->internal_data)->acsm);
+#endif
 	free(((struct mapid_torrent*)instance->internal_data)->torlist);
 	free(instance->internal_data);
   }

src/trackflib/trackflib.h

@@ -3,6 +3,7 @@
 
 #define ETHERTYPE_8021Q 0x8100
 
+//#define __WITH_AHO__
 
 #define EDONKEY_COLOR 1
 #define TORRENT_COLOR 2