Commit cc813853 authored by 's avatar
Browse files

Files moved to another repository

git-svn-id: file:///home/svn/mapi/trunk@1436 8d5bb341-7cf1-0310-8cf6-ba355fef3186
parent dee85e94
CC = gcc
CFLAGS = -g -O2 -W -Wall
LDFLAGS = -lmapi -lcurses -lrrd
BINS_PACKETLOSS = packetloss
CFLAGS_APPMON = -g -W -Wall -Wshadow -Wcast-align -Wpointer-arith
LDFLAGS_APPMON = -lmapi -lrrd -lcgic
BINS_APPMON = appmon appform.cgi
.PHONY: clean run
# packetloss
packetloss: packetloss.o pl_conf.o pl_misc.o
$(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
packetloss.o: packetloss.c cgi_headers_packetloss.h pl_conf.h pl_misc.h
$(CC) $(CFLAGS) -c -o $@ $<
pl_conf.o: pl_conf.c pl_conf.h pl_misc.h
$(CC) $(CFLAGS) -c -o $@ $<
pl_misc.o: pl_misc.c pl_misc.h
$(CC) $(CFLAGS) -c -o $@ $<
# appmon
appmon: appmon.o util.o anon_prefix_preserving.o
appform.cgi: appform.o util.o
appmon.o: ../appmon/appmon.c cgi_headers.h
$(CC) $(CFLAGS_APPMON) -c ../appmon/appmon.c -o $@ -include cgi_headers.h
util.o: ../appmon/util.c
$(CC) $(CFLAGS_APPMON) -c $^ -o $@
anon_prefix_preserving.o: ../appmon/anon_prefix_preserving.c
$(CC) $(CFLAGS_APPMON) -c $^ -o $@
appform.o: ../appmon/appform.c
$(CC) $(CFLAGS_APPMON) -c $^ -o $@
@echo 'Cleaning up ...'
rm -rf $(BINS_PACKETLOSS) $(BINS_APPMON) appmon_top.html appmon_top_private.html appmon.cgi packetcount.cgi packetloss.cgi *.log *.o
Packet loss occurs when correctly transmitted packets from a source never arrive at the
intended destination. Packets are usually lost due to congestion, e.g., at the
queue of some router, routing problems, or poor network conditions that may
result to datagram damages.
Packet loss application is a novel real-time, end-to-end packet loss estimation method
based on distributed passive network monitoring. This application does not add any overhead
to the network since it passively monitors the network traffic without injecting any probe packets.
At the same time, it estimates almost in real-time the actual packet loss faced by
the active applications. Moreover, it offers the capability for measuring the
loss rates of particular Grid services, allowing for fine-grained per-application
packet loss estimation, which is important in case different applications on the
same path face different degrees of packet loss.
(for more details see :
+ MAPI should have been configured with the flags
--enable-dimapi , --enable-extraflib and --enable-trackflib (need for appmon application),
enabled for the required functions to be included.
+ Packet loss requires a web server and rrdtool version 1.2 or higher (need for appmon application)
(in Debian, apt-get install apache rrdtool)
+ Packet loss requires librrd, libncurses and libcgi (need for appmon application)
(in Debian, apt-get install librrd2-dev libcgicg1-dev)
+ Compile packet loss using 'make' (executable files packetloss and appmon will be generated)
Server Configuration
- You should configure your webserver to allow cgi script execution.
For Apache, just edit /etc/apache/httpd.conf (or /usr/local/apache/conf/httpd.conf)
and, uncomment the following:
AddHandler cgi-script .cgi
- Also you should define the cgi-bin directory where the server can find the
cgi executables. Simply add:
ScriptAlias /packetloss/cgi-bin/ /var/www/packetloss/cgi-bin/
into the virtual host entry of your apache configuration file.
Web Directory Initialization
- Run the script to set up the necessary files in your web
server's root directory. It takes as parameter the absolute path of the
web server's root directory. The script will create a directory packetloss
and all the necessary files and symbolic links for the web interface.
For example :
If you are using the root folder and the root folder is /var/www, then $WEB_DIR=/var/www.
If you are using the user home directories which may are /home/username/public_html then
will create '$WEB_DIR/packetloss', and the web interface will be accessible
Specifically, for the above paths, will create:
$WEB_DIR/packetloss/packetloss_top.html -> /home/username/mapi/trunk/applications/packetloss/packetloss_top.html
$WEB_DIR/packetloss/cgi-bin/packetloss.cgi -> /home/username/mapi/trunk/applications/packetloss/packetloss.cgi
$WEB_DIR/packetloss/cgi-bin/packetloss3.cgi -> /home/username/mapi/trunk/applications/packetloss/packetloss3.cgi
$WEB_DIR/packetloss/cgi-bin/packetloss24.cgi -> /home/username/mapi/trunk/applications/packetloss/packetloss24.cgi
$WEB_DIR/packetloss/cgi-bin/packetlossWeek.cgi -> /home/username/mapi/trunk/applications/packetloss/packetlossWeek.cgi
$WEB_DIR/packetloss/cgi-bin/packetlossMonth.cgi -> /home/username/mapi/trunk/applications/packetloss/packetlossMonth.cgi
$WEB_DIR/packetloss/cgi-bin/packetlossYear.cgi -> /home/username/mapi/trunk/applications/packetloss/packetlossYear.cgi
$WEB_DIR/packetloss/cgi-bin/packetcount.cgi -> /home/username/mapi/trunk/applications/packetloss/packetcount.cgi
$WEB_DIR/packetloss/cgi-bin/packetcount3.cgi -> /home/username/mapi/trunk/applications/packetloss/packetcount3.cgi
$WEB_DIR/packetloss/cgi-bin/packetcount24.cgi -> /home/username/mapi/trunk/applications/packetloss/packetcount24.cgi
$WEB_DIR/packetloss/cgi-bin/packetcountWeek.cgi -> /home/username/mapi/trunk/applications/packetloss/packetcountWeek.cgi
$WEB_DIR/packetloss/cgi-bin/packetcountMonth.cgi -> /home/username/mapi/trunk/applications/packetloss/packetcountMonth.cgi
$WEB_DIR/packetloss/cgi-bin/packetcountYear.cgi -> /home/username/mapi/trunk/applications/packetloss/packetcountYear.cgi
$WEB_DIR/packetloss/cgi-bin/appmon.cgi -> /home/username/mapi/trunk/applications/packetloss/appmon.cgi
$WEB_DIR/packetloss/cgi-bin/appmon3.cgi -> /home/username/mapi/trunk/applications/packetloss/appmon3.cgi
$WEB_DIR/packetloss/cgi-bin/appmon24.cgi -> /home/username/mapi/trunk/applications/packetloss/appmon24.cgi
$WEB_DIR/packetloss/cgi-bin/appmonWeek.cgi -> /home/username/mapi/trunk/applications/packetloss/appmonWeek.cgi
$WEB_DIR/packetloss/cgi-bin/appmonMonth.cgi -> /home/username/mapi/trunk/applications/packetloss/appmonMonth.cgi
$WEB_DIR/packetloss/cgi-bin/appmonYear.cgi -> /home/username/mapi/trunk/applications/packetloss/appmonYear.cgi
(We assume that mapi has been installed in /home/username/mapi/ directory)
- mapid and mapicommd should be running at the two monitoring sensors (see 'trunk/doc/mapitutor.pdf' for details)
- packet loss takes three arguments: the IP address of each monitoring sensor concatenated with it's interface (e.g eth0) and a pcap filter.
Let's see the following scenario:
We want to measure the packet loss between monitoring sensors with IP addresses and
Their monitoring interface is eth0.
packetloss should be invoked as: ./packetloss "host and host"
- In order to provide tha figure of appmon application, we must execute this application between the two monitoring sensors.
appmon should be invoked as: ./appmon -V -s -u ""
appmon executable file exists in directory /home/username/mapi/trunk/applications/packetloss/
- If packetloss runs on, then the web interface will be accessible at
#ifndef __CGI_HEADERS_H__
#define __CGI_HEADERS_H__
#define BASENAME "appmon"
#define FORM_FILENAME BASENAME"_form.html"
#define TOP_FILENAME BASENAME"_top.html"
#define PTOP_FILENAME BASENAME"_top_private.html"
static char cgiHead[] = "\
<META HTTP-EQUIV=\"Pragma\" CONTENT=\"no-cache\">\n\
<META Http-Equiv=\"Expires\" Content=\"10\">\n\
<META Http-Equiv=\"Refresh\" Content=\"10;url=./appmon.cgi\">\n\
<style type=\"text/css\">\n\
div.invisible {visibility:hidden}\n\
<script language=\"JavaScript\">\n\
<!--var time = null\n\
function move() {\n\
parent.appmon.location = '"CGI_FILENAME"';\n\
} //-->\n\
</HEAD>\n<BODY vlink=\"blue\">\n<div class=\"invisible\">";
static char cgiHead2[] = "\
</div>\n<center><table><tr><td valign=\"top\">\n\
<table align=\"left\">\n\
<tr><td><div class=\"invisible\"><a href=\"packetcount.cgi\">Packet Loss Count</a></div></td></tr>\n\
<tr><td><a href=\"appmon.cgi\">1 hour</a></td></tr>\n\
<tr><td><a href=\"appmon3.cgi\">3 hour</a></td></tr>\n\
<tr><td><a href=\"appmon24.cgi\">1 day</a></td></tr>\n\
<tr><td><a href=\"appmonWeek.cgi\">1 week</a></td></tr>\n\
<tr><td><a href=\"appmonMonth.cgi\">1 month</a></td></tr>\n\
<tr><td><a href=\"appmonYear.cgi\">1 year</a></td></tr>\n\
</table></td><td align=\"right\">\n\
#define IMAGE_NAME "../%s \n %s "
#define RRD_ATRS "--width 500 --height 300\n\--imginfo '<IMG SRC=../%s WIDTH=%lu HEIGHT=%lu>'\n"
#define cgiHead3 "--end now --start end-%s --lazy\n\ --slope-mode --interlaced --vertical-label \"outbound Mbit/s inbound\"\n"
#define BASENAME "packetloss"
#define CGI_PACKETCOUNT_FILENAME "packetcount.cgi"
#define TOP_FILENAME BASENAME"_top.html" // right frame
#define TOP_FILENAME_CSV BASENAME"_status.csv"
static char cgi_head[] = "\
<META HTTP-EQUIV=\"Pragma\" CONTENT=\"no-cache\">\n\
<META Http-Equiv=\"Expires\" Content=\"10\">\n\
<META Http-Equiv=\"Refresh\" Content=\"10;url=";
static char cgi_head2[] = "\
<script language=\"JavaScript\">\n\
<!--var time = null\n\
function move() {\n\
parent.packetloss.location = ";
static char cgi_head3[] = "\
} //-->\n</script>\n</HEAD>\n<BODY vlink=\"blue\">\n<center>\n\
<table><tr><td valign=\"middle\" align=\"left\">\n\
<table align=\"left\">\n\
<tr><td><a href=\"packetcount.cgi\">Packet Loss Count</a></td></tr>\n\
<tr><td><a href=\"packetloss.cgi\">1 hour</a></td></tr>\n\
<tr><td><a href=\"packetloss3.cgi\">3 hour</a></td></tr>\n\
<tr><td><a href=\"packetloss24.cgi\">1 day</a></td></tr>\n\
<tr><td><a href=\"packetlossWeek.cgi\">1 week</a></td></tr>\n\
<tr><td><a href=\"packetlossMonth.cgi\">1 month</a></td></tr>\n\
<tr><td><a href=\"packetlossYear.cgi\">1 year</a></td></tr>\n\
</table></td><td align=\"right\">\n\
static char cgi_pkcount_head[] = "\
<META HTTP-EQUIV=\"Pragma\" CONTENT=\"no-cache\">\n\
<META Http-Equiv=\"Expires\" Content=\"10\">\n\
<META Http-Equiv=\"Refresh\" Content=\"10;url=";
static char cgi_pkcount_head2[] = "\
<script language=\"JavaScript\">\n\
<!--var time = null\n\
function move() {\n\
parent.packetcount.location = ";
static char cgi_pkcount_head3[] = "\
} //-->\n</script>\n</HEAD>\n<BODY vlink=\"blue\">\n<center>\n\
<table><tr><td valign=\"middle\" align=\"left\">\n\
<table align=\"left\">\n\
<tr><td><a href=\"packetloss.cgi\">Packet Loss Ratio</a></td></tr>\n\
<tr><td><a href=\"packetcount.cgi\">1 hour</a></td></tr>\n\
<tr><td><a href=\"packetcount3.cgi\">3 hour</a></td></tr>\n\
<tr><td><a href=\"packetcount24.cgi\">1 day</a></td></tr>\n\
<tr><td><a href=\"packetcountWeek.cgi\">1 week</a></td></tr>\n\
<tr><td><a href=\"packetcountMonth.cgi\">1 month</a></td></tr>\n\
<tr><td><a href=\"packetcountYear.cgi\">1 year</a></td></tr>\n\
</table></td><td align=\"right\">\n\
#define IMAGE_NAME "../%s \n %s "
#define RRD_ATRS "--width 500 --height 300 \n --imginfo '<IMG SRC=../%s WIDTH=%lu HEIGHT=%lu>'\n"
#define cgi_head4 "--end now --start end-%s --lazy \n --slope-mode --interlaced --vertical-label \"Packet Loss Ratio (%%)\"\n"
#define cgi_pkcount_head4 "--end now --start end-%s --lazy \n --slope-mode --interlaced --vertical-label \"Lost Packets\"\n"
// not a cgi header. Need in packetloss GUI (right frame)
static char top_page[] = "\
<script language=\"JavaScript\">\
function toggleDisplay(me,hnr){\
if (parent.hiddenFrame.hidden[hnr]==1){\
else {\
if (parent.hiddenFrame.hidden[hnr]==0){\
function setdisplay(me, hnr) {\
if( parent.hiddenFrame.hidden[hnr] == 1 )\\"none\";\
function appendmydiv(idname, htmltext, hnr) {\
var newhtml = document.createElement(\"div\");\ = idname;\
if( parent.hiddenFrame.hidden[hnr] == 1)\ = 'none';\
else\ = 'inline';\
newhtml.innerHTML = htmltext;\
var prtable = document.getElementById('protocolstable');\
document.body.insertBefore(newhtml, prtable);\
var sensorstatshtml=\"%s\";\
var sensorsstatslink=\"%s\";\
<style type=\"text/css\">\n\
.style1 {font-family: Verdana, Arial, Helvetica, sans-serif}\n\
.style6 {color: #000000}\n\
.style7 {color: #CCCCCC}\n\
.style8 {color: #00CC00}\n\
.style9 {font-size: 15}\n\
<META Http-Equiv=\"Cache-Control\" Content=\"no-cache\">\n\
<META Http-Equiv=\"Pragma\" Content=\"no-cache\">\n\
<META Http-Equiv=\"Expires\" Content=\"0\">\n\
<META Http-Equiv=\"Refresh\" Content=\"10;url=./%s\">\
</head>\n<body onload='appendmydiv(\"sensorsstatslink\",sensorsstatslink,0), appendmydiv(\"sensorstats\",sensorstatshtml,1)'>\n\
<div class=style9 id='bigdiv'>\n\
<table><tr><td width=25%% align=\"center\">\n\
<a href=\"\" style=\"color: white\" target=_new title=\"Large-scale Monitoring of Broadband Internet Infrastructures\">\n\
<img src=\"/packetloss/img/lobster-logo.jpg\"></a></td></tr>\n\
<tr><td width=75%% align=\"center\"><i><b>Distributed Packet Loss Measurement</b></i></td></tr></table><hr>\n";
<script language="JavaScript">
var hidden = new Array(2);
hidden[0] = 0;
hidden[1] = 1;
<title>LOBSTER - Distributed PacketLoss</title>
<frameset cols="60%, 40%, *" framespacing="1" border="1">
<frameset rows="50%, 50%">
<frame src="cgi-bin/packetloss.cgi" name="packetloss">
<frame src="cgi-bin/appmon.cgi" name="appmon">
<frame src="./packetloss_top.html" name="top">
<frame name="hiddenFrame" id="hiddenFrame" src="./hidden.html">
#define _GNU_SOURCE // sched.h
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <stdarg.h>
#include <signal.h>
#include <getopt.h>
#include <string.h>
#include <rrd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <netdb.h>
#include <sched.h>
#include <fcntl.h>
#include <mapi/expiredflowshash.h>
#include <mapi.h>
//#define NEWGUIONLY
#include <ncurses.h>
#include "cgi_headers_packetloss.h"
#include "pl_misc.h"
#include "pl_conf.h"
#define TIMEOUT 500 // time to drop an unmatched flow
#define HASHTABLE_SIZE 131072
#define WITH_NCURSES 0 // set to 1 if you want ncurses, 0 otherwise
#define REFRESH_TIME 10 // 10 seconds base interval with which data will be fed into the RRD
#define SHM_FLOWS_DEVICE_MAX 49998 // (DIMAPI_DATA_SIZE - sizeof(flows_stat) - cca24) / sizeof(struct flow_data))
#define SHM_FLOWS_TOTAL_MAX 250000
// #define TIMESTAMP_TOLERANCE_JOIN 11600000000 // NTP 2.7
#define TIMESTAMP_TOLERANCE 430000000 // NTP 0.1
#define MAX(x, y) ((x) > (y) ? (x) : (y))
#define MIN(x, y) ((x) < (y) ? (x) : (y))
#define DIFF(x, y) ((x) > (y) ? ((x) - (y)) : ((y) - (x)))
#define ALARM(low, middle, high) ((high) ? 3 : (middle) ? 2 : (low) ? 1 : 0)
#define LOGFILE 0 //Log enabled
#define LOGFILENAME_FULL "packetloss.full.log" //Log filename
#define LOGFILENAME_LOST "packetloss.lost.log"
#define LOGFILENAME_UNMATCHED "packetloss.unmatched.log"
struct pl_conf *pl_conf;
struct pl_conf *category;
struct pl_conf *parameter;
struct configuration {
char *pl_conf_filename;
char *packetloss_rrd_filename;
char *packetloss_packets_rrd_filename;
char *packetloss_flows_rrd_filename;
char *packetloss_matchedflows_rrd_filename;
char *packetloss_status_filename;
char *packetloss_starts_filename;
char *packetloss_startlog_filename;
int flows_max;
unsigned long long timestamp_tolerance;
char *bpf_filter; // bpf filter
char **flow;
int still_reconnect; // every loop try to connect?
#ifdef _GNU_SOURCE
int cpu;
struct configuration *configuration;
unsigned int shm_flows;
const unsigned int expired_flows_list_size_max = 0; //1249;
const unsigned int packets_count_min = 2;
int hup = 0;
int reconnect = 0;
int resetstats = 0;
// packetloss states {{{
#define STATES 15
enum stateid {
char *statestr[STATES] = {
"01. Processing command line arguments...\n",
"02. Processing configuration file: \"%s\" ...\n",
"03. Writing to startlog file \"%s\" ...\n",
"04. Creating/opening rrd file \"%s\" ...\n",
"05. Creating mapi flow \"%s\" ...\n",
"06. Applying function %s to mapi flow at %s ...\n",
"07. Connecting to remote flow at %s ...\n",
"08. Writing to starts file \"%s\" ...\n",
"09. Entering measurement loop ...\n",
"10. Retrieving data from remote host %s ...\n",
"11. Storing data received from %s:%s ...\n",
"12. Processing data received from %s:%s (%d records)...\n",
"13. Storing data to rrd \"%s\" ...\n",
"14. Writing to status file \"%s\" ...\n",
"15. Idle for %u sec ...\n"
// }}}
struct hash_bucket {
struct hash_bucket *next;
struct eflow_data data;
unsigned int hsvalue;
unsigned int sensor;
unsigned int sensor2;
unsigned int d; // XXX
struct hash_bucket *assigned;
#define PCT(proto) ((proto.lost) == 0 ? 0 : (float) ((proto.lost) * 100) / (float) (proto.pkts))
#define ROF(part, base) ((base) == 0 ? 0 : (float) (part) / (float) (base))
#define PCTOF(part, base) (ROF(part, base) * 100)
// statistics {{{
struct tstatistics { // need for network load
unsigned long long bytes;
unsigned long long ts;
struct tlcstat {
union {
unsigned long long pkts;
unsigned long long count;
union {
unsigned long long loss;
unsigned long long lost;
struct tlcmstat {
unsigned long long loss;
union {
unsigned long long count;
unsigned long long matched;
struct tcmutstat {
unsigned int count;
unsigned int matched;
unsigned int unmatched;
unsigned int timeouted;
// Variables for known ports
struct pstat {
struct tlcstat mapi;
struct tlcstat https;
struct tlcstat sftp;
struct tlcstat rtsp;
struct tlcstat http;
struct tlcstat dns;
struct tlcstat smtp;
struct tlcstat openvpn;
struct tlcstat telnet;
struct tlcstat ssh;
struct tlcstat ftp;
struct tstats {
struct tcmutstat totalflows;
struct tlcmstat totalpk;
struct tlcmstat totalbytes;
struct pstat pstats;
struct tlcstat totalpk = {{0}, {0}};
struct tlcstat old_totalpk = {{0}, {0}};
struct tlcstat totalbytes = {{0}, {0}};
struct pstat pstats = {{{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}, {{0}, {0}}};
struct t360stat {
unsigned long long loss[360];
unsigned long long count[360];
unsigned long long pkloss;
unsigned long long pkcount;
unsigned long long counter;
struct t180stat {
unsigned long long loss[180];
unsigned long long count[180];
unsigned long long pkloss;
unsigned long long pkcount;
unsigned long long counter;
struct t288stat {
unsigned long long loss[288];
unsigned long long count[288];
unsigned long long pkloss;
unsigned long long pkcount;
unsigned long long counter;
struct t336stat {