The art of extracting metrics

2022-05-10T00:00:00+00:00
Awk</h2>

filtering between log timestamps</h3>
second precision</p>
awk \</span></span>
  -v from="$(date +'%s' -d '2021-10-29 02:34' --utc)" \</span></span>
  -v to="$(date +'%s' -d '2021-10-29 02:36' --utc)" \</span></span>
  '</span></span>
  split($0, parts, /[\- :]/) {</span></span>
    linefmt=""</span></span>
    for(i=1;i<=6;i++) linefmt = sprintf("%s %s",linefmt, parts[i]);</span></span>
    ts=mktime(sprintf("%s UTC", linefmt))</span></span>
    if (ts > from && ts < to) print</span></span>
  }</span></span>
  '<<<'2021-10-29 04:50:00 172.31.43.164'</span></span>
</span>
awk \</span></span>
  -v from="$(date +'%s' -d '2021-11-27 04:15:00 UTC' --utc)" \</span></span>
  -v to="$(date +'%s' -d 'now UTC' --utc)" \</span></span>
  '</span></span>
  patsplit($1, p, /[0-9]+/) {</span></span>
    ts=mktime(sprintf("%d %d %d %d %d %d", p[1], p[2], p[3], p[4], p[5], p[6]), 1)</span></span>
    if (ts > from && ts < to) print</span></span>
  }</span></span>
  ' <(tac /var/log/frr/frr.log)|head</span></span></code></pre>perl alternative</h2>
from="$(date -Is -d '2021-11-27T04:15:00 UTC' --utc)" \</span></span>
to="$(date -Is -d 'now UTC' --utc)" \</span></span>
perl \</span></span>
  -MDate::Parse \</span></span>
  -lane \</span></span>
  '</span></span>
  $ts=str2time($F[0]);</span></span>
  $f=str2time($ENV{from});</span></span>
  $t=str2time($ENV{to});</span></span>
  print if ($ts >= $f && $ts <= $t )</span></span>
  ' <(zcat -f /var/log/syslog*) | tee "/syslog-$(date -Im)"</span></span></code></pre>from ms precision unix timestamp</h2>
awk \</span></span>
  -v format="%F %T" \</span></span>
  -v from="$(date +'%s' -d '2021-10-29 02:34' --utc)" \</span></span>
  -v to="$(date +'%s' -d '2021-10-29 02:36' --utc)" \</span></span>
  '{</span></span>
  ts=substr($0, 0, index($0, ".")-1)</span></span>
  if (ts > from && ts < to) print strftime(format, ts), substr($0,index($0,$1))</span></span>
  }</span></span>
  ' <<<'2021-10-29 04:50:00 172.31.43.164'</span></span></code></pre>Sed</h2>
Filtering interesting lines for nxos. Note, NXOS has an ancient version of sed, it only supports the -n</code> flag.</p>
monitoring procfs</h2>
Continuously monitor a procfs file, output is tabular.</p>
cat /proc/pressure/cpu</span></span>
some avg10=1.34 avg60=1.63 avg300=2.33 total=2483371233</span></span>
full avg10=0.18 avg60=0.03 avg300=0.06 total=827064553</span></span></code></pre>

sleep trap: https://stackoverflow.com/questions/65644307/ctrl-c-doesnt-always-terminate-a-shell-script</a></li>
getline summary: https://www.gnu.org/software/gawk/manual/gawk.html#Getline-Summary</a></li>
printf formatting: https://bl831.als.lbl.gov/~gmeigs/scripting_help/printf_awk_notes.txt</a>, https://www.gnu.org/software/gawk/manual/html_node/Printf.html</a></li>
</ul>

awk -F'[ =]' -vlinefmt="%-8s%-8s%-10s%-10s%-8s\n" -vcpu_pressure="/proc/pressure/cpu" 'BEGIN {</span></span>
    printf linefmt, "avg10", "avg60", "avg300", "increase", "total"</span></span>
    do {</span></span>
        if (system("trap true INT; sleep 1")) {</span></span>
            exit</span></span>
        }</span></span>
        if ($1=="full") {</span></span>
            printf linefmt, $3, $5, $7, int($9-prev_tot), $9</span></span>
            prev_tot = $9</span></span>
        }</span></span>
    } while ((getline < cpu_pressure) || (close(cpu_pressure) == 0))</span></span>
}'</span></span>
</span>
avg10   avg60   avg300    increase  total</span></span>
0.36    0.17    0.07      829472174 829472174</span></span>
0.36    0.17    0.07      0         829472174</span></span>
0.24    0.16    0.07      24957     829497131</span></span></code></pre>
collect metrics for all PSI.</p>
awk -F'[ =\t]' -vlinefmt="%-14s" -vfields='"cpuavg10", "cpuavg60", "cpuavg300", "cputotal", "cpuincrease", "ioavg10", "ioavg60", "ioavg300", "iototal", "ioincrease", "memavg10", "memavg60", "memavg300", "memtotal","memincrease"' -vcpu_pressure="bash -c 'paste /proc/pressure/{cpu,io,memory}'" \</span></span>
'BEGIN { patsplit(fields, fmts, /[^", ]+/)</span></span>
    do {</span></span>
        if (system("trap true INT; sleep 1")) {</span></span>
            exit</span></span>
        }</span></span>
        if ($1=="some") {</span></span>
            for(x in fmts) {</span></span>
                printf linefmt, fmts[x]</span></span>
            }</span></span>
            printf "\n"</span></span>
            for(i=3;i<=NF;i+=2) {</span></span>
                if(i%9 == 0 && i%3 == 0) {</span></span>
                    printf linefmt""linefmt, $(i), int($(i)-increase[i]) </span></span>
                    increase[i]=$i</span></span>
                    i+=1</span></span>
                } else {</span></span>
                    printf linefmt, $i</span></span>
                }</span></span>
                </span></span>
            }</span></span>
            printf "\n\n"</span></span>
        }</span></span>
    } while ((cpu_pressure|getline) || (close(cpu_pressure) == 0))</span></span>
}'</span></span></code></pre>histograms</h2>
The .1 decimal for size helps tie breakers when min == max.</p>
while :; do</span></span>
  ss -n -t -o $(printf " state %s" {fin-wait-1,fin-wait-2,time-wait,close-wait,last-ack,closing}) |\</span></span>
    awk -v bmin=0 -vsize=10000.1 'BEGIN{tm["sec"] = 1000;tm["min"] = "60000"} match($6,/,([0-9.]+)(sec|ms|min)/,m){t=(m[2]=="ms"?int(m[1]):int(tm[m[2]]*m[1]) );b=int(t/size);a[b]++;bmax=b>bmax?b:bmax; bmin=b<bmin?b:bmin} END{for(i=bmin;i<=bmax;++i) printf "%d %d %d\n", (i*size)/tm["sec"],((i+1)*size)/tm["sec"],a[i]}'</span></span>
  sleep 1</span></span>
done</span></span></code></pre>
Using time functions.</p>
LC_ALL=C awk -v bmin=0 -vsize=10.1 -v date="2022 03 06" 'split($4,p,/[: ]/){t=mktime(date" "p[2]" "p[3]" "p[4]); b=int(t/size);a[b]++;bmax=b>bmax?b:bmax; bmin=b<bmin?b:bmin} END{for(i=bmin;i<=bmax;++i) if(a[i]) printf "%s %s %d\n", strftime("%T",(i*size)),strftime("%T",((i+1)*size)),a[i]}' gopkg.in-access.log-2022-03-06T13\:00.log</span></span></code></pre>
Useful squid one</p>
for i in $(awk '!s[$4]++{print $4}' squid-access.log); do  LC_ALL=C awk -v bmin=0 -vsize=300.1 -vhist="$i" '$0 !~ hist{next} {t=int($1);b=int(t/size);a[b]++;bmax=b>bmax?b:bmax; bmin=b<bmin?b:bmin} END{printf "%s 300s histogram\n", hist;for(i=bmin;i<=bmax;++i) if(a[i]) printf "%s %s %d\n", strftime("%T",(i*size)),strftime("%T",((i+1)*size)),a[i] > "histograms.sout"}' squid-access.log; done</span></span>
</span>
LC_ALL=C awk -v bmin=0 -vsize=10.1 -v date="2022 03 06" \</span></span>
'split($4, p, /[: ]/) {</span></span>
  t = mktime(date " " p[2] " " p[3] " " p[4])</span></span>
  b = int(t / size)</span></span>
  a[b]++</span></span>
  bmax = b > bmax ? b : bmax</span></span>
  bmin = b < bmin ? b : bmin</span></span>
}</span></span>
</span>
END {</span></span>
  for (i = bmin; i <= bmax; ++i) {</span></span>
    if (a[i]) {</span></span>
      printf "%s %s %d\n", strftime("%T", (i * size)), strftime("%T", ((i + 1) * size)), a[i]</span></span>
    }</span></span>
  }</span></span>
}'</span></span></code></pre>
iptables filtering</p>
sudo iptables -t nat  -nL |awk -vrgxp="$(rgxg cidr 10.131.31.0/24)" '/^(Chain|target)/{f=1} f==1||$0 ~ rgxp{s[NR]=$0;f++}; length($0)==0{if(f>2){for(i in s) print s[i]};split("",s);f=0}'</span></span></code></pre>
Realm of Chaos

The art of extracting metrics

Awk</h2>

Sed</h2>
Filtering interesting lines for nxos. Note, NXOS has an ancient version of sed, it only supports the `-n</code> flag.</p>`

networking

Application Performance Analysis

Realm of Chaos

The art of extracting metrics

Awk</h2>

Sed</h2> Filtering interesting lines for nxos. Note, NXOS has an ancient version of sed, it only supports the -n</code> flag.</p>

networking

Application Performance Analysis

Sed</h2>
Filtering interesting lines for nxos. Note, NXOS has an ancient version of sed, it only supports the `-n</code> flag.</p>`
