Skip to content

GitLab

Commit 5c672be5 authored by Olav Kvittem's avatar Olav Kvittem
Browse files

Mpeg TS and Crude support

parent 7a37b902
Hide whitespace changes
Inline Side-by-side
README
View file @ 5c672be5
......@@ -4,3 +4,5 @@ see http://software.uninett.no/qstream
revision 3
support for Crude packet log format
revision 8
mpeg ts analysys for jitter and discontinuity
qstream
View file @ 5c672be5
......@@ -16,44 +16,59 @@ use Net::RTP;
use Net::RTP::Packet;
use Time::HiRes qw/ sleep time tv_interval gettimeofday/;
use Data::Dumper;
#use bigint qw/ hex oct/;
use Math::BigInt;
use Math::BigFloat;
use Carp;
use constant MTU => 1580;
#use mpegpackets;
$DEFAULT_PORT = 1955;
my $timeout=1 ; # frac seconds to wait for receiving packets
require "newgetopt.pl";
$usage="$0 [option]... [file...|ip/|:port]...
-list list flows in files
-dump dump data part to file
-format [full|pretty] print full numbers foror short pretty numbers
-net open network stream rather than file(s)
-rtp look forinto rtp headers (default just udp)
-mpeg decode mpeg-2 transport stream and look for sequence and mediatime
-pcap listen to all multicast groups with pcap at the same time
-period seconds - period in seconds between stats output
-nperiod number of periodes
-last seconds - exit after seconds
-packets packets - exit after packets
-src source - limit stats to given source
-flow_key flow key - limit stats to this flow(src_ip:src_port->dest_ip:dest_port)
-flow_no flow no - limit stats to this flow no from -list
-flow_min numb - discard flows with less packets that numb
-nohead - no headings for batch use
-id name - report name instead of flow_key in flows
-sum name - make a summary of all flows to name rather than per flow
-verbose - more details packet distances and sizes
-debug - print debugging info
-help/h this message
-list list flows in files
-dump dump data part to file
-format [full|pretty] print full numbers foror short pretty numbers
-net open network stream rather than file(s)
-crude the log is from crude
-bins bins for gaps in ms : b1,b2,,,
-log log gaps longer than 1 (use with crude)
-rtp look forinto rtp headers (default just udp)
-mpeg decode mpeg-2 transport stream and look for sequence and mediatime
-pcap listen to all multicast groups with pcap at the same time
-period seconds - period in seconds between stats output
-nperiod number of periodes
-last seconds - exit after seconds
-packets packets - exit after packets
-src source - limit stats to given source
-flow_key flow key - limit stats to this flow(src_ip:src_port->dest_ip:dest_port)
-flow_no flow no - limit stats to this flow no from -list
-flow_min numb - discard flows with less packets that numb
-nohead - no headings for batch use
-id name - report name instead of flow_key in flows
-sum name - make a summary of all flows to name rather than per flow
-ttl - print ttl values for stream
-verbose - more details packet distances and sizes
-debug - print debugging info
-help/h this message
\n";
@opts=('list', 'fullformat', 'net', 'rtp', 'mpeg', 'pcap', 'crude', 'nperiod=s', 'period=s', 'last=s', 'packets=s', 'src=s', 'flow_key=s', 'flow_no=s', 'flow_min=s', 'nohead', 'dump=s', 'format=s', 'id=s', 'sum=s', 'verbose', 'v', 'help', 'h', 'debug');
@opts=('list', 'fullformat', 'net', 'rtp', 'mpeg', 'pcap', 'crude', 'bins=s', 'log', 'nperiod=s', 'period=s', 'last=s', 'packets=s', 'src=s', 'flow_key=s', 'flow_no=s', 'flow_min=s', 'nohead', 'dump=s', 'format=s', 'id=s', 'sum=s', 'ttl', 'log', 'verbose', 'v', 'help', 'h', 'debug');
&NGetOpt(@opts) || die $usage;
die $usage if $opt_h or $opt_help;
$continous= ! $opt_packets;
$opt_v=$opt_v or $opt_verbose;
my @streams=@ARGV;
if ($opt_bins){
@bins=split(/,/, $opt_bins);
}
my $format = $opt_format || "pretty";
......@@ -65,6 +80,39 @@ my $nint=0;
my $n_packets=0;
my $endstream=0;
#MPEG-TS measurement globals (most are reset in init)
my $CHECK_NUM_TSP = 2;
#globals for mpeg ts
my @cc = ();
my $discontinuities = 0;
my %last_ccs = ();
my $packets_between_pcr = 0;
#my $PTS_MAX = 95443.7176888888888888
my $PTS_MAX = 95443;
my $PCR_MAX = 95443;
# pcr_stats {filename/handle}
my %pcr_stats = ();
#PTS-stats {filename/handle}
my %pts_stats = ();
my %mpeg_stats = ();
#PTS stuff
#my $real_time = time();
my $late_video_packets = 0;
#CC distribution
my $num_cc = 0;
my %cc_dist = ();
my %pid_next_cc = ();
my %dist1 = ();
my %dist2 = ();
$opt_period = $opt_last if not $opt_period;
#MPEG-TS END
$SIG{USR2} = sub {confess "Caught by SIGUSR2"; };
......@@ -76,9 +124,11 @@ $SIG{ALRM} = sub {
# die "No packets received" if $n_packets < 1;
# $uninterrupted=0;
&display_stats();
# die "End after alarm\n";
# return(0);
$endstream=1;
print "endstream=1" if $opt_debug;
};
};
if ($opt_dump){
open DUMP, ">$opt_dump";
......@@ -109,6 +159,8 @@ if ($opt_pcap){ # listen in parallell
}
&list_flows if $opt_list;
&display_stats() if $opt_sum;
&display_bins if $opt_bins;
exit(0);
......@@ -121,7 +173,9 @@ sub prepare_ip {
$uri =~ s/^(udp|rtp)://;
$uri =~ s#//##;
die "Unsupported streaming $1" if $uri =~/^(\w:)/;
my $multicast = $uri =~ s/^@//;
if ($multicast = ($uri =~ /@/)){
($source, $uri)=split(/@/, $uri);
}
my ($host, $port)=split(/[\/:]/, $uri);
die $usage . "undefined address inn $host" unless (defined $host);
......@@ -252,6 +306,7 @@ sub eat_stream {
if ($opt_dump){
print DUMP $packet;
}
$endstream = tv_interval($t0, $tc) >= $opt_last if $opt_last;
} else { # assume that we got an interrupt
# die "Failed to receive packet: $!";
}
......@@ -272,20 +327,26 @@ sub eat_stream {
}
if ($opt_crude){
my $flow_key;
open CRUDE, "<", $pcap_file || die "Could not open $f : $err";
open CRUDE, "<$pcap_file" || die "Could not open $f : $err";
while(<CRUDE>){ # crude.sourceforge.net
my $tx0;
if (($fid, $seq, $src, $dst, $tx, $rx, $size)=
/ID=(\d+)\s+SEQ=(\d+)\s+SRC=([\d.:]+)\s+DST=([\d.:]+)\s+Tx=([\d.,]+)\s+Rx=([\d.,]+)\s+SIZE=(\d+)/){
$flow_key="$src->$dst";
$flow_pkts{$flow_key}++;
$tc=[int($rx), ($tx-int($rx))*10^6] ; # tval
if ($flow_pkts{$flow_key} <= 1) { # new flow
push(@flows, $flow_key); #
$flow_no++;
$file{$flow_key}=$f;
$source{$flow_key}=$src;
}
$rx0{$flow_key}=$rx;
$tx0{$flow_key}=$tx;
$tjoined{$flow_key}=$tc;
}
next if $opt_list; # just count packets per flow
next if $opt_flow_key && $flow_key !~ $wanted_flow;
next if $opt_flow_no && $opt_flow_no !~ /\b$flow_no\b/;
......@@ -295,8 +356,13 @@ sub eat_stream {
push (@myflows, $flow_key);
}
$us=($rc-$tc)*10^6; # us
&pkt_stats($flow_key, , $us, $size);
$seq_num{$flow_key}=$seq;
$ptx{$flow_key}=$tx{$flow_key}; # previous
$tx{$flow_key}=$tx;
$rx{$flow_key}=$rx;
$mtime{$flow_key}=($tx-$tx0{$flow_key})*10**6;
$us=($rx-$rx0{$flow_key})*10**6; # us
&pkt_stats($flow_key, 0, $us, $size);
}
}
......@@ -309,6 +375,8 @@ sub eat_stream {
eat_pcap($f, $pcap, $pkt);
$n_packets++;
}
printf "%s\n", $ttl_log if $opt_ttl;
} else { # open failed
if ($format ne "full"){ # always put out data when format full
die "Could not open $f : $err";
......@@ -325,20 +393,17 @@ sub eat_pcap_streams{ # open groups and listen on pcap
my @streams=@_;
my $filter="( ";
my $mc = IO::Socket::Multicast->new(LocalPort=>$DEFAULT_PORT, ReuseAddr=>1) ||
die "Could not connect to port $port : $!";
die "Could not connect to port $port : $!";
foreach $id (@streams){ # listen to all groups
my ($host, $port) = split(/[\/:]/, $id);
die $usage . "undefined address inn $id" unless (defined $host);
$port = $DEFAULT_PORT unless (defined $port);
my $address=$host;
if ($host !~ /^\d+\.\d+\.\d+\.\d+$/){ # hostname
my $padr=gethostbyname($host);
$address=Socket::inet_ntoa($padr);
}
$mc->mcast_add($address) || die "could not join $address:$port : $!";
$filter .= " or " if $filter ne "( ";
$filter .= "host $address";
my ($address, $port, $multicast) = prepare_ip($id);
$filter .= " or " if $filter =~ /\(\s+\(/;
if ($multicast){
$mc->mcast_add($address) || die "could not join $address:$port : $!";
$filter .= "(dst $address and port $port)";
} else {
$filter .= "(src $address and port $port)";
}
}
$tjoined{$f}=[gettimeofday];
$filter .= " )";
......@@ -357,13 +422,15 @@ sub eat_pcap_streams{ # open groups and listen on pcap
$r=Net::Pcap::setfilter($pcap, $filter_t);
die "Could not set pcap filter : $r : $filter : $! " if $r != 0 ;
my $select=IO::Select->new( Net::Pcap::fileno($pcap));
while ($select->can_read($timeout) && (my $pkt=Net::Pcap::next($pcap, \%hdr)) ) { # get all packets
last if ! $uninterrupted || $endstream;
eat_pcap("pcap", $pcap, $pkt);
$n_packets++;
last if $opt_packets && $n_packets > $opt_packets;
}
printf "%s\n", $ttl_log if $opt_ttl;
}
......@@ -398,6 +465,21 @@ sub eat_pcap { # process pcap packets
$file{$flow_key}=$f;
$source{$flow_key} =sprintf "%s:%d", $ip->{src_ip}, $udp->{src_port};
}
if ($opt_ttl){
if ($pttl{$flow_key} && ($opt_log || ($pttl{$flow_key} != $ip->{ttl}))){ # change in ttl
printf "%s\n", $ttl_log;
$nttl{$flow_key}=0;
}
$pttl{$flow_key}= $ip->{ttl};
$nttl{$flow_key}++;
($sequence_number{$flow_key}, $tx_time_seconds{$flow_key}, $tx_time_useconds{$flow_key}, $flow_id{$flow_key})= unpack('NNNN', $udp->{data});
$ttl_log=sprintf "ID=%u SEQ=%u SRC=%s DST=%s Tx=%u.%06u Rx=%u.%06u TTL=%u",
$flow_id{$flow_key}, $sequence_number{$flow_key},
$ip->{src_ip}, $ip->{dest_ip}, $tx_time_seconds{$flow_key},
$tx_time_useconds{$flow_key}, $hdr{tv_sec}, $hdr{tv_usec}, $pttl{$flow_key};
$ttl_log.= sprintf(" No=%08d\n", $nttl{$flow_key},) if !$opt_log;
}
next if $opt_list; # just count packets per flow
next if $opt_flow_key && $flow_key !~ $wanted_flow;
next if $opt_flow_no && $opt_flow_no !~ /\b$flow_no\b/;
......@@ -423,31 +505,48 @@ sub eat_pcap { # process pcap packets
# $first_port = $udp->{dest_port};
# }
&pkt_stats($flow_key, $udp->{data}, $us, $dlen);
&pkt_stats($flow_key, $udp->{data}, $us, $dlen);
}
sub init_stats {
my $f=shift;
my $f=shift;
$pinterval=0;
$npkt{$f}=$sumbyte{$f}=$ssbyte{$f}=0;
$pinterval=0;
$npkt{$f}=$sumbyte{$f}=$ssbyte{$f}=0;
$first_sec=0;
delete $pmtime{$f}; delete $pus{$f} ; delete $first_us{$f}; delete $last_us{$f};
delete $prev_1s{$f}; delete $prev_100ms{$f};
delete $per1s{$f}; delete $per100ms{$f};
delete $sum1s{$f}; delete $sum100ms{$f};
$n100ms{$f}=0; $sum100ms{$f}=0; $ss100bps{$f}; $akk100ms{$f}=0;
$sumbyte{$f}=$sumps{$f}=0; # sum byte per second
$maxbps{$f}=0; $max100ms{$f}=0;
$dup{$f}=$lost{$f}=$late{$f}=0;
$maxbyte{$f}=0; $minbyte{$f}=2**31;
$sumgap{$f}=$ssgap{$f}=0;
$ngap{$f}=0; $maxgap{$f}=0, $mingap{$f}=2**63-5;
$njitter{$f}=$sumjitter{$f}=$ssjitter{$f}=0;
$maxjitter{$f}=-2**63-5; $minjitter{$f}=2**63-5; delete $est_jitter{$f}; delete $pest_jitter{$f};
$first_port = 0; $n_seq=-1;
delete $tstart{$f}; delete $tend{$f};
delete $pmtime{$f}; delete $pus{$f} ; delete $first_us{$f}; delete $last_us{$f};
delete $prev_1s{$f}; delete $prev_100ms{$f};
delete $per1s{$f}; delete $per100ms{$f};
delete $sum1s{$f}; delete $sum100ms{$f};
$n100ms{$f}=0; $sum100ms{$f}=0; $ss100bps{$f}; $akk100ms{$f}=0;
$sumbyte{$f}=$sumps{$f}=0; # sum byte per second
$maxbps{$f}=0; $max100ms{$f}=0;
$dup{$f}=$lost{$f}=$late{$f}=0;
$maxbyte{$f}=0; $minbyte{$f}=2**31;
$sumgap{$f}=$ssgap{$f}=0;
$ngap{$f}=0; $maxgap{$f}=0, $mingap{$f}=2**63-5;
$njitter{$f}=$sumjitter{$f}=$ssjitter{$f}=0;
$maxjitter{$f}=-2**63-5; $minjitter{$f}=2**63-5; delete $est_jitter{$f}; delete $pest_jitter{$f};
$first_port = 0; $n_seq=-1;
delete $tstart{$f}; delete $tend{$f};
#MPEG-TS initalizing and resetting (new period!)
$discontinuities = 0;
%last_ccs = ();
@cc = ();
# r_stats {filename/handle}
@pcr_stats{$f} = ();
#P-stats {filename/handle}
@pts_stats{$f} = ();
@mpeg_stats{$f} = ();
#P stuff
$late_video_packets = 0;
#MPEG-TS END
%dist1 = ();
%dist2 = ();
$num_cc = 0;
}
sub init_file {
......@@ -465,185 +564,229 @@ sub init_file {
delete($pus{$f});
}
sub pkt_stats {
my $f = shift; # file or stream name
my $packet = shift;
my $us = shift;
my $dlen = shift;
my $rtp = new Net::RTP::Packet();
if (! exists($tstart{$f})){
$tstart{$f} = $tc;
$first_us{$f}=$prev1s_us{$f}=$prevt100ms_us{$f}=$us;
if (! exists($setuptime{$f}) ){
if ( $pinterval){
$setuptime{$f}=0;
} else {
$setuptime{$f}=tv_interval ( $tjoined{$f}, $tstart{$f}) * 10**3;
}
printf "setup $f : %.3f\n", $setuptime{$f} if $opt_debug;
}
}
$tend{$f} = $tc;
$last_us{$f}=$us;
# next if $dlen < 20; # typically small background noice packets added
$npkt{$f}++;
$sumbyte{$f}+=$dlen; $sum1s{$f}+=$dlen; $sum100ms{$f}+=$dlen;
$ssbyte{$f}+=$dlen**2;
$maxbyte{$f}= $dlen if $dlen > $maxbyte{$f};
$minbyte{$f}= $dlen if $dlen < $minbyte{$f};
$sumps{$f}+=$dlen; # sum per second
if ($opt_dump){
print DUMP $packet;
}
if ($opt_rtp){
if ( $rtp->decode($packet) ) { #assume RTP
# printf "RTP: %10s %5d %5d\n", $rtp->ssrc(), $rtp->timestamp(), $rtp->size();
# $dlen = $rtp->size();
} else {
print "??? $f # $npkt{$f} $dlen B\n";
if ($opt_debug){
my $n=0;
foreach $w( unpack("C*", $packet)) {
printf "%0X",$w;
if ($n > 0){
print " " if $n % 4 == 0;
print "\n" if $n % 32 == 0;
}
$n++;
}
printf "\n";
}
next;
}
# Calculate next number in sequence
$n_seq = $rtp->seq_num()+1;
if ($n_seq > 65535) {
$n_seq=0;
}
$mtime=$rtp->timestamp() * &time_unit($rtp->payload_type); # mediatime in us
}
if (defined($pus{$f})) {
$gap = $us - $pus{$f}; # packet time gap
$ngap{$f}++;
$sumgap{$f} += $gap;
$ssgap{$f} += $gap**2;
$maxgap{$f}= $gap if $gap > $maxgap{$f};
$mingap{$f}= $gap if $gap < $mingap{$f};
if ($opt_rtp){
my $seq_diff= $rtp->seq_num - $pseq_num{$f};
# printf "%0d-%0d, ", $seq_diff, $n_seq - $rtp->seq_num;
sub seq_stat {
my ($f, $seq_num, $mtime)=@_;
my $seq_diff= $seq_num - $pseq_num{$f};
unless ($opt_crude && $seq_diff == 1){ #
$gap = $us - $pus{$f}; # packet time gap
$ngap{$f}++;
$sumgap{$f} += $gap;
$ssgap{$f} += $gap**2;
$maxgap{$f}= $gap if $gap > $maxgap{$f};
$mingap{$f}= $gap if $gap < $mingap{$f};
foreach $i (0 .. $#bins){
if( $gap < ($bins[$i]*1000)){
$nbin[$i]++;
last;
}
}
printf "Tx=%.6f gap=%d gap=%.1fms\n", $ptx{$f}, $seq_diff-1, $gap/1000, if $opt_log;
}
if ($seq_diff != 1) {
if ($seq_diff != 1) {
# printf "seq %d,", $rtp->seq_num;
if ($seq_diff == 0) { # Duplicated
$dup{$f} ++;
} elsif ($seq_diff < 0) { # Out Of Order
$late{$f}++;
if ($seq_diff == 0) { # Duplicated
$dup{$f} ++;
} elsif ($seq_diff < 0) { # Out Of Order
$late{$f}++;
# $lost{$f}--;
} else { # Lost
$lost{$f}+= $seq_diff - 1;
$nloss{$f}++;
$sumloss{$f} += $seq_diff;
$ssumloss{$f} += $seq_diff**2;
$maxloss{$f} = $seq_diff if $seq_diff > $maxloss{$f};
}
} elsif ( $mtime <= $pmtime{$f}){
$ntimeerr{$f}++;
} else { # the jitter can be measured
} else { # Lost
$lost{$f}+= $seq_diff - 1;
$nloss{$f}++;
$sumloss{$f} += $seq_diff;
$ssumloss{$f} += $seq_diff**2;
$maxloss{$f} = $seq_diff if $seq_diff > $maxloss{$f};
}
} elsif ( $mtime <= $pmtime{$f}){
$ntimeerr{$f}++;
} else { # the jitter can be measured
# compute jitter relative to previous packet
$mgap= ($mtime - $pmtime{$f} ); # media time gap in samples 8khz=us
$jitter=abs($gap - $mgap);
$mgap= ($mtime - $pmtime{$f} ); # media time gap in samples 8khz=us
$jitter=abs($gap - $mgap);
#ok compute jitter relative to start
#ok $mgap= ($mtime - $first_mtime); # media time gap in us
#ok $jitter=abs($us - $mtime - $first_jitter);
$njitter{$f}++;
$sumjitter{$f} += $jitter;
$ssjitter{$f} += $jitter**2;
$maxjitter{$f}= $jitter if $jitter > $maxjitter{$f};
$minjitter{$f}= $jitter if $jitter < $minjitter{$f};
# rfc 3550
if (defined($est_jitter{$f})){
$est_jitter{$f} = $est_jitter{$f} + ( $jitter - $est_jitter{$f} )/16;
} else { # inital value
$est_jitter{$f} = $jitter;
$njitter{$f}++;
$sumjitter{$f} += $jitter;
$ssjitter{$f} += $jitter**2;
$maxjitter{$f}= $jitter if $jitter > $maxjitter{$f};
$minjitter{$f}= $jitter if $jitter < $minjitter{$f};
# rfc 3550
if (defined($est_jitter{$f})){
$est_jitter{$f} = $est_jitter{$f} + ( $jitter - $est_jitter{$f} )/16;
} else { # inital value
$est_jitter{$f} = $jitter;
}
}
}
sub pkt_stats {
my $f = shift; # file or stream name
my $packet = shift;
my $us = shift;
my $dlen = shift;
my $rtp = new Net::RTP::Packet();
if (! exists($tstart{$f})){
$tstart{$f} = $tc;
$first_us{$f}=$prev1s_us{$f}=$prevt100ms_us{$f}=$us;
if (! exists($setuptime{$f}) ){
if ( $pinterval){
$setuptime{$f}=0;
} else {
$setuptime{$f}=tv_interval ( $tjoined{$f}, $tstart{$f}) * 10**3;
}
printf "setup $f : %.3f\n", $setuptime{$f} if $opt_debug;
}
}
$tend{$f} = $tc;
$last_us{$f}=$us;
# next if $dlen < 20; # typically small background noice packets added
$npkt{$f}++;
$sumbyte{$f}+=$dlen; $sum1s{$f}+=$dlen; $sum100ms{$f}+=$dlen;
$ssbyte{$f}+=$dlen**2;
$maxbyte{$f}= $dlen if $dlen > $maxbyte{$f};
$minbyte{$f}= $dlen if $dlen < $minbyte{$f};
$sumps{$f}+=$dlen; # sum per second
if ($opt_dump){
print DUMP $packet;
}
if ($opt_mpeg){
mpeg_stats($f, $packet, $us);
}
if ($opt_rtp){
if ( $rtp->decode($packet) ) { #assume RTP
# printf "RTP: %10s %5d %5d\n", $rtp->ssrc(), $rtp->timestamp(), $rtp->size();
# $dlen = $rtp->size();
} else {
print "??? $f # $npkt{$f} $dlen B\n";
if ($opt_debug){
my $n=0;
foreach $w( unpack("C*", $packet)) {
printf "%0X",$w;
if ($n > 0){
print " " if $n % 4 == 0;
print "\n" if $n % 32 == 0;
}
$n++;
}
printf "\n";
}
next;
}
# Calculate next number in sequence
$n_seq = $rtp->seq_num()+1;
if ($n_seq > 65535) {
$n_seq=0;
}
$mtime{$f}=$rtp->timestamp() * &time_unit($rtp->payload_type); # mediatime in us
$seq_num{$f}=$rtp->seq_num;
}
if (defined($pus{$f})) {
$gap = $us - $pus{$f}; # packet time gap
$ngap{$f}++;
$sumgap{$f} += $gap;
$ssgap{$f} += $gap**2;
$maxgap{$f}= $gap if $gap > $maxgap{$f};
$mingap{$f}= $gap if $gap < $mingap{$f};
if ($opt_rtp){
&seq_stat($f, $seq_num{$f}, $mtime{$f});
}
if ($adapt == 1 ) { # adaption field coming
my ($length, $aflags, $rest2)=unpack "C1C1C*", $rest;
my ($dif, $raif, $espif, $pcrf, $opcrf, $splicef, $tpdf, $afextf)=split(//, $aflags);
if ($pcrf == 1){
;
}
printf "%5d %6d %5d %5d %5d %5d %5d\n",$rtp->seq_num,$us,$rtp->timestamp(), $gap, $mgap, $jitter, $first_jitter if $opt_debug;
}
} elsif ($opt_mpeg){ # incomplete code under exploration
# meg transport stream layout http://en.wikipedia.org/wiki/Transport_stream
my ($flag, $b2, $b3, $rest)= unpack "H2n1C1C*", $packet;
} elsif ($opt_crude){
&seq_stat($f, $seq_num{$f}, $mtime{$f});