Commit a648d21e authored by Otto Jonassen Wittner's avatar Otto Jonassen Wittner
Browse files

Merge branch 'master' into Generalized_server-side

Many updates in master required in branch
parents efa78969 ec843260
Pipeline #49105 passed with stages
in 49 seconds
......@@ -5,15 +5,32 @@ This project holds components required for realizing a measurement probe node i
Components: Scripts to fetch and build configurations for running rude/crude and traceroute.
When client is installed, send ipv4 address, suggested name for location and and coordinates to coordinator
## Hardware requirements for client
Any virtual or real hardware from RPI-4 and up can run this comfortably.
Basic software need about 1GB, so a system with 2GB would be fine.
10GB disk to store one and in emergency up to a couple of days of raw data.
Expect to leave 10-20% og one core to the continuous measurements on a leaf node.
Accurate clock with ntp special configuration is recommended for high time precision.
### Secruity considerations
Probing port on UDP needs to be open to peers. Default is 10001, but is configurable.
Ssh and https is necessary outwards.
Node is autonomous and don't need external ssh access, but central management is easier with access.
## Manual setup on client
~~~~~
sudo apt install ntp
sudo apt install traceroute tcptraceroute
sudo apt install curl libstatistics-linefit-perl libstatistics-basic-perl libdatetime-perl libjson-xs-perl
sudo apt install tcpdump rsync
## Extra packages on ubuntu RPi
sudo apt install libdatetime-perl libjson-perl libstatistics-basic-perl libstatistics-linefit-perl
### ntp configuration
# a nearby ntp server is needed for delay accuracy
sudo apt install ntp
insert in /etc/ntp.conf if in Uninett - else replace server addresses :
server 158.38.0.168 iburst minpoll 4 maxpoll 5
server 128.39.2.22 iburst minpoll 4 maxpoll 5
......@@ -22,7 +39,7 @@ insert in /etc/ntp.conf if in Uninett - else replace server addresses :
apt-get install git build-essential automake autoconf
git clone https://scm.uninett.no/rude/rude.git
cd microdep
cd rude
bash ./build.sh
./configure
make
......@@ -33,7 +50,7 @@ insert in /etc/ntp.conf if in Uninett - else replace server addresses :
sudo cp ~/microdep/bin/get-icmp /usr/local/sbin
add to /etc/sudoers.d/microdep
$USER ALL=(ALL) NOPASSWD: /usr/local/sbin/get-icmp
$USER ALL=(ALL) NOPASSWD: /usr/local/sbin/get-icmp.sh
~~~~~
### Edit ~/microdep/etc/local.cfg :
......
......@@ -17,6 +17,7 @@ logdir=$logpath/$date
PIPE=$logdir/crude.pipe
if test ! -p $PIPE ; then
mkfifo $PIPE
# $root/bin/set-pipe-size $PIPE 1048576 # hangs on writer to connect ..
fi
# kill command and all children
......@@ -40,12 +41,14 @@ else
sleep 3 # allow pipe to be created
fi
if ! pgrep -u $USER -f "bin/crude-spread-queue " >/dev/null; then
#remove the previous scripts before the new crude-spread-ana starts
kill_family crude-ana
kill_family crude-zip
$root/bin/crude-spread-queue "$index" $logdir $PIPE &
if ! pgrep -u $USER -f "bin/crude-spread-select" > /dev/null; then
# make sure that qstream-gapa-ana deos not run already
pkill -f -u $USER bin/crude-ana-gz
pkill -f -u $USER 'gzip .*crude.pipe'
pkill -f -u $USER 'tail .*crude-.*.gz'
if ! pgrep -u $USER -f "bin/crude-spread-select" > /dev/null; then
$root/bin/crude-spread-select $index $logdir $PIPE &
fi
fi
if ! pgrep -u $USER -f "bin/trace.sh " >/dev/null; then
......
#! /bin/bash
# set -e
export PERL5LIB=$HOME/microdep/perl
source $(dirname "${BASH_SOURCE[0]}")/../etc/start.cfg
optadr=
if test -s $root/etc/mp-address.txt; then
optadr="-addresses $root/etc/mp-address.txt"
fi
# try to find a gzip process that stores crude.*gz
crudefile=$( ps -f -u $USER | perl -ne 'if ( /gzip .* (\/.*\/crude-[\d:]+.gz)/){print $1;exit}' )
if test ! -f "$crudefile"; then # sometimes redirected filenames is not shown
crudefile=$( find ~/microdep/data/$date/ -name 'crude-*.gz' | sort | tail -1 )
fi
json=$( echo $crudefile | sed -e s/crude-/gap-ana-/ -e 's/.gz/.json/' )
if test -f "$crudefile" ; then
tail -f -n +0 $crudefile | zcat | perl $root/bin/qstream-gap-ana $optadr \
-index $index -json $json -minloss 5 -head -win 50 -jitter 600 -
else
exit 1
fi
exit 0
......@@ -3,7 +3,7 @@
# set pipe_size so that disk flush does not throttle back
# use non-blocking write and buffer internally
my $debug=1;
my $debug=0;
use Fcntl;
use File::Basename;
......
#!/usr/bin/perl
# spread file input to parallel commands on a measurement point
# using internal buffers/debug
my $debug=0;
use threads;
# use Thread::Queue;
use FileHandle;
use File::Basename;
use Getopt::Long;
use Time::HiRes qw(usleep);
my $pipe_size=2**20 - 1; # 1MB
my (@opt_cmd, $opt_v, $opt_h, $opt_debug);
my $usage="Usage\n$0 --cmd <command> --cmd ...
-pipe_size - buffering output to commands ($pipe_size)
-v - print summary stats at end
$0 index output-directory [data-file]..\n
";
GetOptions('cmd=s'=> \@opt_cmd, 'pipe_size=s'=>\$pipe_size, 'debug=s' => \$opt_debug, 'v'=>\$opt_v, 'h'=>\$opt_h) or die "$usage : $!" ;
die $usage if $opt_h;
my @cmds=();
if ( $#opt_cmd >= 0 ){ # externally spec'd commands
@cmds=@opt_cmd;
} else { # builtin commands
if ( $#ARGV < 1 ){
printf "Usage: $0 index output-directory [data-file]..\n";
exit 1;
}
my $index=shift;
my $lager=shift;
my $bin = dirname($0);
my $root = "$bin/..";
my $ext=`date +%T`;
chomp $ext;
@cmds = (
"perl $bin/qstream-gap-ana -v -minloss 5 -head -win 50 -jitter 600 -addresses $root/etc/mp-address.txt -rtp 5" .
" -json $lager/gap-ana-$ext.json" .
" >> $lager/gap-ana-$ext.txt 2>> $lager/gap-log-$ext.txt",
"gzip >> $lager/crude-$ext.gz" );
# only done at central site - mostly for debug purposes :
# "nice perl $bin/qstream-gap-list >> $lager/gap-list-$ext.txt";
}
$SIG{INT} = 'final_summary';
$SIG{TERM} = 'final_summary';
my @queues=();
my @fh=();
my @blocked=();
my $n_slept=();
my $t_slept=();
my @would_block=();
my @partial_write=();
my @normal_write=();
foreach $i ( 0.. $#cmds ){
$fh[$i] = IO::File->new();
$fh[$i]->open( "|$cmds[$i]" or die "Could not open cmd : $! :\n$cmd");
$fh[$i]->blocking(0);
$queues[$i]=();
set_pipe_size( $fh[$i], $pipe_size); # 1M
}
while(<>){
$nbytes+=length $_;
my $diff= $nbytes - tell STDIN;
printf "diff %d " if $diff !=0;
foreach $i ( 0.. $#cmds ){
push ( @{$queues[$i]}, $_);
$qmax[$i]=$#{$queues[$i]} if ! $qmax[$i] || $qmax[$i] < $#{$queues[$i]};
while ( $#{$queues[$i]} >= 0 ){
if ( $blocked[$i] < 0 ){ # never
printf "blocked index %d for %d\n", $i, $blocked[$i] if $opt_debug && $blocked[$i] > 1;
usleep(100 * $blocked[$i]); # additive increase in sleep_time
$n_slept[$i] ++;
$t_slept[$i] += $blocked[$i];
}
my $line = shift(@{$queues[$i]});
my $rv = syswrite( $fh[$i], $line, length $line);
if (!defined($rv) && $!{EAGAIN}) { # would block
unshift @{$queues[$i]}, $line;
$would_block[$i]++;
$blocked[$i]++;
break;
} elsif ($rv != length $line) { # incomplete write
unshift @{$queues[$i]}, substr( $line, $rv );
$partial_write[$i]++;
$blocked[$i]++;
break;
} else { # successfully wrote
$normal_write[$i]++;
$blocked[$i]=0;
}
}
}
}
summary();
exit (0);
#================================================================================
sub summary{
if ( $opt_v){
foreach $i(0..$#cmds){
printf "summary:\n";
printf "%15s %15s %15s %15s %15s %15s\n", qw /would_block partial_write normal_write n_slept t_slept qmax/;
printf "%15d %15d %15d %15d %15d %15d %s\n", $would_block[$i], $partial_write[$i], $normal_write[$i],
$n_slept[$i], $t_slept[$id], $qmax[$i], $cmds[$i];
}
}
}
sub final_summary{
summary();
exit(1);
}
sub set_pipe_size{
my $fh = shift;
my $pipe_size = shift;
if ($debug and -p $fh ) {
printf "fh %d is pipe\n", $fh;
printf "pipe size was %d\n", fcntl($fh, Fcntl::F_GETPIPE_SZ, 0);
}
my $new = fcntl( $fh, Fcntl::F_SETPIPE_SZ, int($pipe_size))
|| warn "### could not set pipe size $pipe_size for $cmd: $!\n";
warn sprintf "### set-pipe-size for $cmd failed: new size $new, got: %d",
fcntl($fh, Fcntl::F_GETPIPE_SZ, 0)
if $new < $pipe_size;
return $fh;
}
......@@ -2,19 +2,21 @@
# spread file input to parallel commands on a measurement point
# using threads and queueing
my $debug=1;
my $debug=0;
use threads;
use Thread::Queue;
use FileHandle;
use File::Basename;
use Getopt::Long;
my $pipe_size=2**19; # 0.5MB
my (@opt_cmd, $opt_v, $opt_h);
my $usage="Usage\n$0 --cmd <command> --cmd ...
-pipe_size - buffering output to commands ($pipe_size)
-v - print summary stats at end
$0 index output-directory [data-file]..\n
";
GetOptions('cmd=s'=> \@opt_cmd, 'v'=>\$opt_v, 'h'=>\$opt_h) or die "$usage : $!" ;
GetOptions('cmd=s'=> \@opt_cmd, 'pipe_size=s'=>\$pipe_size, 'v'=>\$opt_v, 'h'=>\$opt_h) or die "$usage : $!" ;
die $usage if $opt_h;
......@@ -30,7 +32,6 @@ if ( $#opt_cmd >= 0 ){ # externally spec'd commands
my $index=shift;
my $lager=shift;
my $pipe_size=2**20; # 1MB
my $bin = dirname(__FILE__);
my $root = "$bin/..";
......@@ -89,6 +90,7 @@ sub writer{
my $fh = IO::File->new();
$fh->open( "|$cmd" or die "Could not open cmd : $! :\n$cmd");
set_pipe_size( $fh, $pipe_size);
# $0=$cmd; - changes name of parent process in ps/pkill so don't use
while(1){
......@@ -117,3 +119,18 @@ sub gather_results(){
}
print @lines if $opt_v;
}
sub set_pipe_size{
my $fh = shift;
my $pipe_size = shift;
if ($debug and -p $fh ) {
printf "fh %d is pipe\n", $fh;
printf "pipe size was %d\n", fcntl($fh, Fcntl::F_GETPIPE_SZ, 0);
}
my $new = fcntl( $fh, Fcntl::F_SETPIPE_SZ, int($pipe_size))
|| warn "### could not set pipe size $pipe_size for $cmd: $!\n";
warn sprintf "### set-pipe-size for $cmd failed: new size $new, got: %d",
fcntl($fh, Fcntl::F_GETPIPE_SZ, 0)
if $new < $pipe_size;
return $fh;
}
#!/usr/bin/perl
# spread file input to parallel commands on a measurement point
# using internal buffers/debug
my $debug=0;
use threads;
# use Thread::Queue;
use FileHandle;
use File::Basename;
use IO::Select;
use Getopt::Long;
use Time::HiRes qw(usleep);
my $pipe_size=2**20 - 1; # 1MB
my (@opt_cmd, $opt_v, $opt_h, $opt_debug);
my $usage="Usage\n$0 --cmd <command> --cmd ...
-pipe_size - buffering output to commands ($pipe_size)
-v - print summary stats at end
$0 index output-directory [data-file]..\n
";
GetOptions('cmd=s'=> \@opt_cmd, 'pipe_size=s'=>\$pipe_size, 'debug=s' => \$opt_debug, 'v'=>\$opt_v, 'h'=>\$opt_h) or die "$usage : $!" ;
die $usage if $opt_h;
my @cmds=();
if ( $#opt_cmd >= 0 ){ # externally spec'd commands
@cmds=@opt_cmd;
} else { # builtin commands
if ( $#ARGV < 1 ){
printf "Usage: $0 index output-directory [data-file]..\n";
exit 1;
}
my $index=shift;
my $lager=shift;
my $bin = dirname($0);
my $root = "$bin/..";
my $ext=`date +%T`;
chomp $ext;
@cmds = (
"perl $bin/qstream-gap-ana -v -minloss 5 -head -win 50 -jitter 600 -addresses $root/etc/mp-address.txt -rtp 5" .
" -json $lager/gap-ana-$ext.json" .
" >> $lager/gap-ana-$ext.txt 2>> $lager/gap-log-$ext.txt",
"gzip >> $lager/crude-$ext.gz" );
# only done at central site - mostly for debug purposes :
# "nice perl $bin/qstream-gap-list >> $lager/gap-list-$ext.txt";
}
if ( ! $DB::{single} ){ # drop when in debugger
$SIG{INT} = 'final_summary';
$SIG{TERM} = 'final_summary';
}
my @queues=();
my @fh=();
my @blocked=();
my $n_slept=();
my $t_slept=();
my @would_block=();
my @partial_write=();
my @normal_write=();
my $line_count=0;
foreach $i ( 0.. $#cmds ){
$fh[$i] = IO::File->new();
$fh[$i]->open( "|$cmds[$i]" or die "Could not open cmd : $! :\n$cmd");
$fh[$i]->blocking(0);
$queues[$i]=();
set_pipe_size( $fh[$i], $pipe_size); # 1M
}
push @ARGV, "-" if $#ARGV < 0; # default stdin
foreach $in_file ( @ARGV){
my $in_fh;
if ( $in_file eq "-"){
$in_fh = STDIN;
} else {
open $in_fh, "<", $in_file || die "Could not open $in_file : $!\n" ;
}
my $select = IO::Select->new( $in_fh );
my $remaining_output=0;
my $wait_s=0.01; # 10 ms
my $line;
my $is_pipe= -p $in_fh;
my $eof=0;
$!=0;
until( $eof == 1 and $remaining_output <= 0){
# empty input queue
my $continue = 1;
while ( !$eof and $continue and my @ready_FHs = $select->can_read( $wait_s)) {
foreach my $FH (@ready_FHs) { # only one
if ( $! != 0 ) {
die "Error reading $in_file : $!";
}
sysread $FH, $line, 1000;
if ( length $line == 0 ){
$eof=1;
} else {
$line_count++;
foreach $i ( 0.. $#cmds ){
push ( @{$queues[$i]}, $line);
}
}
}
if ( ! $is_pipe || $eof == 1) { # not pipe - don't read the whole disk file
$continue=0;
}
}
$qmax[$i]=$#{$queues[$i]} if ! $qmax[$i] || $qmax[$i] < $#{$queues[$i]};
# try to empty internal queues
foreach $i ( 0.. $#cmds ){
while ( $#{$queues[$i]} >= 0 ){
if ( $blocked[$i] < 0 ){ # never
printf "blocked index %d for %d\n", $i, $blocked[$i] if $opt_debug && $blocked[$i] > 1;
usleep(100 * $blocked[$i]); # additive increase in sleep_time
$n_slept[$i] ++;
$t_slept[$i] += $blocked[$i];
}
my $line = shift(@{$queues[$i]});
my $rv = syswrite( $fh[$i], $line, length $line);
if (!defined($rv) && $!{EAGAIN}) { # would block
unshift @{$queues[$i]}, $line;
$would_block[$i]++;
$blocked[$i]++;
break;
} elsif ($rv != length $line) { # incomplete write
unshift @{$queues[$i]}, substr( $line, $rv );
$partial_write[$i]++;
$blocked[$i]++;
break;
} else { # successfully wrote
$normal_write[$i]++;
$blocked[$i]=0;
}
}
}
$remaining_output=0;
foreach $i ( 0.. $#cmds ){
$remaining_output += $#{$queues[$i]} + 1;
}
$!=0; # reset errors before can_read
} # of input cycle
} # of input files
summary();
exit (0);
#================================================================================
sub summary{
if ( $opt_v){
printf "summary: lines in : %d\n", $line_count;
printf "%15s %15s %15s %15s %15s %15s\n", qw /would_block partial_write normal_write n_slept t_slept qmax/;
foreach $i(0..$#cmds){
printf "%15d %15d %15d %15d %15d %15d %s\n", $would_block[$i], $partial_write[$i], $normal_write[$i],
$n_slept[$i], $t_slept[$id], $qmax[$i], $cmds[$i];
}
}
}
sub final_summary{
summary();
exit(1);
}
sub set_pipe_size{
my $fh = shift;
my $pipe_size = shift;
if ($debug and -p $fh ) {
printf "fh %d is pipe\n", $fh;
printf "pipe size was %d\n", fcntl($fh, Fcntl::F_GETPIPE_SZ, 0);
}
my $new = fcntl( $fh, Fcntl::F_SETPIPE_SZ, int($pipe_size))
|| warn "### could not set pipe size $pipe_size for $cmd: $!\n";
warn sprintf "### set-pipe-size for $cmd failed: new size $new, got: %d",
fcntl($fh, Fcntl::F_GETPIPE_SZ, 0)
if $new < $pipe_size;
return $fh;
}
......@@ -10,9 +10,24 @@ fi
if pgrep -u $USER -f "bin/crude " >/dev/null; then
pkill -u $USER -f "bin/crude " || echo crude kill failed
fi
if pgrep -u $USER -f "bin/crude-spread-ana " >/dev/null; then
pkill -u $USER -f "bin/crude-spread-ana "|| echo crude-spread-ana kill failed
if pgrep -u $USER -f "bin/crude-ana-gz" >/dev/null; then
pkill -u $USER -f "bin/crude-ana-gz"|| echo crude-ana-gz kill failed
fi
if pgrep -u $USER -f "bin/qstream-gap-ana" >/dev/null; then
pkill -u $USER -f "bin/qstream-gap-ana" # to be sure if it does not die by the above
fi
if pgrep -u $USER -f 'tail.*crude-.*gz' >/dev/null; then
pkill -u $USER -f 'tail.*crude-.*gz'
fi
### kill yesterdays solutions
if pgrep -u $USER -f "bin/crude-spread-queue " >/dev/null; then
pkill -u $USER -f "bin/crude-spread-queue "|| echo crude-spread-queue kill failed
fi
if pgrep -u $USER -f "bin/crude-spread-noblock " >/dev/null; then
pkill -u $USER -f "bin/crude-spread-noblock "|| echo crude-spread-noblock kill failed
fi
### done yesterdays
if pgrep -u $USER -f "bin/trace.sh " >/dev/null; then
pkill -u $USER -f "bin/trace.sh " || echo trace kill failed
fi
......
......@@ -258,7 +258,8 @@ sub handle_signals(){
make_summary();
# while ( $#ARGV >= 0) { printf "Got signal %s\n", shift @ARGV }
# die "Caught a signal $!";
exit(0);
close JSON if $opt_json;
exit(1);
}
......
......@@ -7,10 +7,13 @@ my $list='mp-list.txt';
my $default_port=10001;
my $index; # the index for event records
my $rude_port=3100; # start local port number udp
my $max_hops=30;
use Getopt::Long;
@opts=( 'target=s' => \$target, 'list=s' => \$list, 'index=s' => \$index, 'rude_start=i' => \$rude_port);
my $usage="usage: $0 [ -target <target host> ] [-list <mp-list.txt>] -index <index> config-file -rude_start <3001>\n";
@opts=( 'target=s' => \$target, 'list=s' => \$list, 'index=s' => \$index, 'rude_start=i' => \$rude_port,
'max_hops=i' => \$max_hops);
my $usage="usage: $0 [ -target <target host> ] [-list <mp-list.txt>] -index <index> config-file
-rude_start <3001> --max_hops <n> (30)\n";
GetOptions(@opts) || die $usage;
die $usage if !$index;
......@@ -109,6 +112,7 @@ foreach $name ( keys %port){
printf $start "index=%s\n", $index ;
my $crude_port= $crude_port{$name} || $port{$name} || $default_port;
printf $start "crude_port=%s\n", $crude_port;
printf $start "max_hops=%s\n", $max_hops;
# printf $start "%s", $startcfg;
close $start;
}
......
#!/usr/bin/perl
# usage: name-of-open-fifo size-in-bytes
# will hang on writer to connect to pipe
# modeled after : http://unix.stackexchange.com/a/353761/119298
use strict;
use Fcntl;
# debian 8++ does not have Fcntl::F_GETPIPE_SZ
use constant F_GETPIPE_SZ => 1024+7;
use constant F_SETPIPE_SZ => 1024+8;
my $fifo = shift @ARGV or die "usage: fifo size";
my $size = shift @ARGV or die "usage: fifo size";
my $verbose = shift @ARGV or 0;
open(FD, "<", $fifo) or die "### set-pipe-size cannot open $fifo : $!";
printf "old size %d\n", fcntl(\*FD, Fcntl::F_GETPIPE_SZ, 0) if $verbose > 0 ;
printf "old size %d\n", fcntl(\*FD, F_GETPIPE_SZ, 0) if $verbose > 0 ;
my $new=5;
my $new = fcntl(\*FD, Fcntl::F_SETPIPE_SZ, int($size));
my $new = fcntl(\*FD, F_SETPIPE_SZ, int($size));
printf "new size %d\n",fcntl(\*FD, Fcntl::F_GETPIPE_SZ, 0) if $verbose > 0 ;
printf "new size %d\n",fcntl(\*FD, F_GETPIPE_SZ, 0) if $verbose > 0 ;
die "### set-pipe-size for $fifo failed: new size $new" if $new<$size;
......@@ -7,9 +7,15 @@ source $(dirname "${BASH_SOURCE[0]}")/../etc/start.cfg
phour=-1
hour=$(date +%H)