#!/bin/bash
exit 1

##
## regarding benchmarking

cat <<'EOF'
use https://ocv.me/dev/?filedrop-1.2.py for python threads
use https://ocv.me/dev/?filedrop-1.3-beta.py for multiprocessing
use https://ocv.me/dev/?up2k-beta.go to go

changes to make for benchmarking in filedrop-1.2:
  disable writing to disk
    nullwrite = True
  enable reading the buffersize from the url (optional)
    uncomment #self.bufsz

changes to make for benchmarking in up2k-beta.go:
  enable reading the buffersize from the url (optional)
    replace POSTcs with POST
EOF


##
## generating data which my multipart parser will choke on

moz: boundary=---------------------------2176004356067937191948767842
mod: boundary=---------------------------------------------------7842

# post.moz
echo "[payload removed: 556 characters]" | gzip -d > post.moz

# generate 0.979 GiB (1052253180) of tricky data
b="\r\n\r\n---------------------------------------------------"
printf "$b" > /dev/shm/x1; for n in {1..13}; do cat /dev/shm/x{1,1,1} > /dev/shm/x2; mv /dev/shm/x2 /dev/shm/x1; done; ls -al /dev/shm/x1; for n in {1..12}; do cat /dev/shm/x1; done | wc -c

# and post it
{ cat post.moz | head -16 | sed -r "s/^(Content-Length: )235/\1$((1052253180+235-6))/"; for n in {1..12}; do stdbuf -o32K cat /dev/shm/x1; done; tail -c 63 post.moz; } | stdbuf -o32K pv | ncat 127.0.0.1 1234

# sha1:
#     asdfaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 526.431
#     asdf--------------------------------------------------- 248.080
# \r\n\r\n--------------------------------------------------- 154.247
# \r\n\r\naaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 525.603

# sha256:
#     asdfaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 282.459
#     asdf--------------------------------------------------- 174.740
# \r\n\r\n--------------------------------------------------- 121.598

# sha512:
#     asdfaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 397.327
#     asdf--------------------------------------------------- 219.977
# \r\n\r\n--------------------------------------------------- 146.606



##
## testing with various chunksizes
## usage:  chungus | tee log

chungus() { for s in 1 2 4 8 16 32 64 128 256 512; do echo $s; for n in {1..5}; do curl -sF "f=@Various.zip" http://127.0.0.1:1234/$s 2>&1; done; done; }


##
## define a faster alternative to curl
## NOTE: this was a mistake and left here as a reminder that curl is pretty good

ncurl() { local host=$1; local port=$2; local file=$3; bd='---------------------------2176004356067937191948767842'; len=$(stat -c%s -- "$file"); len=$((229+len)); { printf 'POST / HTTP/1.1\r\nHost: %s:%s\r\nContent-Type: multipart/form-data; boundary=%s\r\nContent-Length: %s\r\n\r\n--%s\r\nContent-Disposition: form-data; name="f"; filename="txt.foobar"\r\nContent-Type: application/octet-stream\r\n\r\n' $host $port $bd $len $bd; stdbuf -o8M cat -- "$file"; printf '\r\n--%s--\r\n' $bd; } | stdbuf -i8M socat -b$((8*1024*1024)) - tcp4-connect:$host:$port; }


##
## testing multiple parallel uploads
## usage:  para | tee log

para() { for s in 1 2 3 4 5 6 7 8 12 16 24 32 48 64; do echo $s; for r in {1..5}; do for ((n=0;n<s;n++)); do curl -sF "f=@Various.zip" http://127.0.0.1:1234/32 2>&1 & done; wait; echo; done; done; }

# to test both py2 and py3,
# in one terminal, run
python2 some.py; python3 some.py
# and in another
{ para; kill $(ps ax | awk '/python[23][ ]filedrop-1/ {print $1}'); sleep 3; para; } | tee log


##
## display average speed
## usage: avg logfile

avg() { awk 'function pr(ncsz) {if (nsmp>0) {printf "%3s %s\n", csz, sum/nsmp} csz=$1;sum=0;nsmp=0} {sub(/\r$/,"")} /^[0-9]+$/ {pr($1);next} / MiB/ {sub(/ MiB.*/,"");sub(/.* /,"");sum+=$1;nsmp++} END {pr(0)}' "$1"; }


##
## single-file transfers

py2 sha1
  1 145.599
  2 216.425
  4 318.762
  8 406.126
 16 467.63
 32 515.881 <<
 64 467.387
128 408.473
256 414.206
512 426.884

py3 sha1
  1 123.203
  2 190.095
  4 271.314
  8 357.161
 16 431.232
 32 495.497 <<
 64 452.302
128 402.355
256 411.294
512 424.052

py2 sha256
  1 115.792
  2 160.673
  4 209.988
  8 247.213
 16 271.109
 32 286.047 <<
 64 267.542
128 247.843
256 249.666
512 252.906

py3 sha256
  1 103.465
  2 146.527
  4 188.808
  8 226.639
 16 259.11
 32 279.305 <<
 64 263.558
128 245.375
256 248.694
512 254.248

py2 sha512
  1 134.593
  2 200.357
  4 271.939
  8 335.88
 16 382.582
 32 409.404 <<
 64 376.239
128 339.286
256 347.649
512 364.236

py3 sha512
  1 125.271
  2 183.11
  4 247.61
  8 308.373
 16 360.964
 32 396.999 <<
 64 370.343
128 337.082
256 346.031
512 363.292

go sha1 no-parser
  1 494.919
  2 530.073
  4 545.513
  8 584.639
 16 607.402
 32 622.322
 64 625.957
128 656.719 <<
256 656.381
512 652.624

go sha256 no-parser
  1 260.885
  2 264.308
  4 267.062
  8 275.842
 16 279.374
 32 281.292
 64 284.399
128 284.817
256 286.744
512 287.293 <<

493.284  go sha1 native-parser
251.369  go sha256 native-parser


##
## best configs for single-file transfers:

515.881 py2  32k sha1   nfiles-parser
495.497 py3  32k sha1   nfiles-parser
286.047 py2  32k sha256 nfiles-parser
279.305 py3  32k sha256 nfiles-parser
656.719  go 128k sha1   no-parser
284.817  go 128k sha256 no-parser
493.284  go   ?k sha1   native-parser
251.369  go   ?k sha256 native-parser


##
## parallel transfers (average speed of each file)

py2 sha1 32k
  1 511.174
  2 467.323
  3 408.93
  4 287.761

py3 sha1 32k
  1 491.266
  2 438.167
  3 350.399
  4 229.666

py2 sha256 32k
  1 283.549
  2 271.477
  3 255.95
  4 216.031
  5 168.676
  6 139.05
  7 118.696
  8 103.167
 12 68.146
 16 50.4546
 24 34.0745
 32 25.2755

py3 sha256 32k
  1 278.185
  2 263.225
  3 245.006
  4 199.829
  5 157.253
  6 129.728
  7 109.887
  8 95.7741
 12 61.589
 16 45.9331
 24 30.8858
 32 22.0811

py2 sha256 32k multiprocessing
  1 275.976
  2 273.101
  4 223.982
  8 114.976
 16 55.279
 24 35.407
 32 26.185

py3 sha256 32k multiprocessing
  1 267.873
  2 262.830
  4 219.882
  8 107.956
 16 53.633
 24 35.393
 

for sz in 512 0; do for s in 1 2 3 4 5 6 7 8; do echo $s; nr=$((12-s)); for ((r=0;r<$nr;r++)); do for ((n=0;n<s;n++)); do curl -sF "f=@Various.zip" http://127.0.0.1:1234/$sz & done; wait; echo; done; done; done

go sha256 (just-sha) no-parser, 512k
  1 287.453
  2 279.72
  3 271.312
  4 227.719
  5 183.109
  6 154.601
  7 132.562
  8 115.223

go sha256 (just-sha) no-parser, io.Copy
  1 280.821
  2 274.967
  3 265.849
  4 239.029
  5 192.383
  6 159.675
  7 135.927
  8 118.798

go sha256 (just-sha) builtin-parser
  1 250.025
  2 242.723
  3 236.816
  4 210.926
  5 169.589
  6 141.005
  7 122.495
  8 107.677

go sha256 (sha+/dev/null) builtin-parser
  1 241.479
  2 234.959
  3 228.138
  4 210.458
  5 168.198
  6 141.156
  7 121.144
  8 104.968
 12 70.6911
 16 53.1318
 24 36.0473
 32 26.5867

"""
