anonfunc/upload-baker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

inital impl

Browse Source
This commit is contained in:
Giovanni Harting
2022-07-04 23:13:24 +02:00
parent fb23a4dbb0
commit 8f9afad583
5 changed files with 399 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
.gitignore vendored Normal file
View File

@@ -0,0 +1,135 @@
# Created by https://www.toptal.com/developers/gitignore/api/goland+all,go,linux
# Edit at https://www.toptal.com/developers/gitignore?templates=goland+all,go,linux
### Go ###
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
### Go Patch ###
/vendor/
/Godeps/
### GoLand+all ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# SonarLint plugin
.idea/sonarlint/
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### GoLand+all Patch ###
# Ignore everything but code style settings and run configurations
# that are supposed to be shared within teams.
.idea/*
!.idea/codeStyles
!.idea/runConfigurations
### Linux ###
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
# .nfs files are created when an open file is removed but is still being accessed
.nfs*
# End of https://www.toptal.com/developers/gitignore/api/goland+all,go,linux

13
config.yaml Normal file
View File

@@ -0,0 +1,13 @@
bandwidth:
min: 2
max: 45
step: 2
interval: 5
host: "itsh.dev"
ppp: 5
conformation_ppp: 30
log_level: DEBUG
upload_interface: "enp1s0f1"
cake_options: "docsis diffserv4 ack-filter nat"
throttle_ping_threshold: 50
try_restoring_after: 30

18
go.mod Normal file
View File

@@ -0,0 +1,18 @@
module uploadBaker
go 1.18
require (
github.com/go-ping/ping v1.1.0
github.com/sirupsen/logrus v1.8.1
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3
gopkg.in/yaml.v2 v2.4.0
)
require (
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/google/uuid v1.2.0 // indirect
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 // indirect
)

31
go.sum Normal file
View File

@@ -0,0 +1,31 @@
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3 h1:shC1HB1UogxN5Ech3Yqaaxj1X/P656PPCB4RbojIJqc=
github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3/go.mod h1:XCsSkdKK4gwBMNrOCZWww0pX6AOt+2gYc5Z6jBRrNVg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 h1:pDMpM2zh2MT0kHy037cKlSby2nEhD50SYqwQk76Nm40=
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

202
main.go Normal file
View File

@@ -0,0 +1,202 @@
package main
import (
"flag"
"fmt"
"github.com/go-ping/ping"
log "github.com/sirupsen/logrus"
"github.com/wercker/journalhook"
"gopkg.in/yaml.v2"
"os"
"os/exec"
"os/signal"
"regexp"
"strconv"
"strings"
"syscall"
"time"
)
type Conf struct {
Bandwidth struct {
Max int `yaml:"max"`
Min int `yaml:"min"`
Step int `yaml:"step"`
} `yaml:"bandwidth"`
Interval float64 `yaml:"interval"`
Host string `yaml:"host"`
PPP int `yaml:"ppp"`
ConformationPPP int `yaml:"conformation_ppp"`
LogLevel string `yaml:"log_level"`
UploadInterface string `yaml:"upload_interface"`
CakeOptions string `yaml:"cake_options"`
ThrottlePingThreshold int64 `yaml:"throttle_ping_threshold"`
TryRestoringAfter int `yaml:"try_restoring_after"`
}
var (
conf *Conf
journalLog = flag.Bool("journal", false, "Log to systemd journal instead of stdout")
reBandwidth = regexp.MustCompile(`(?m)bandwidth\s(\d+)Mbit`)
)
type UploadManager struct {
LastOver time.Time
}
func (m UploadManager) Upload() (int, error) {
tcCmd := exec.Command("tc", "qdisc", "show", "dev", conf.UploadInterface)
out, err := tcCmd.CombinedOutput()
if err != nil {
return 0, err
}
if reBandwidth.MatchString(string(out)) {
up, err := strconv.Atoi(reBandwidth.FindAllStringSubmatch(string(out), -1)[0][1])
if err != nil {
return 0, err
}
return up, nil
} else {
return 0, fmt.Errorf("could not parse bandwidth from tc")
}
}
func (m UploadManager) SetUpload(upload int) error {
m.clearTC()
args := []string{"qdisc", "add", "dev", conf.UploadInterface, "root", "cake", "bandwidth", fmt.Sprintf("%dMbit", upload)}
args = append(args, strings.Split(conf.CakeOptions, " ")...)
tcCmd := exec.Command("tc", args...)
log.Debugf("[TC] exec %s", tcCmd.String())
out, err := tcCmd.CombinedOutput()
if err != nil {
log.Debugf("[TC] executing %s failed with: %v (%s)", tcCmd.String(), err, out)
return err
}
return nil
}
func (m UploadManager) clearTC() {
tcCmd := exec.Command("tc", "qdisc", "del", "dev", conf.UploadInterface, "root")
log.Debugf("[TC] exec %s", tcCmd.String())
out, err := tcCmd.CombinedOutput()
if err != nil {
log.Debugf("[TC] executing %s failed with: %v (%s)", tcCmd.String(), err, out)
}
}
func doPing(host string, count int, interval time.Duration) (*ping.Statistics, error) {
pinger, err := ping.NewPinger(host)
if err != nil {
return nil, err
}
pinger.SetPrivileged(true)
pinger.Count = count
pinger.Interval = interval
err = pinger.Run()
if err != nil {
return nil, err
}
return pinger.Statistics(), nil
}
func (m UploadManager) isBWInRange(bw int) bool {
return bw <= conf.Bandwidth.Max && bw >= conf.Bandwidth.Min
}
func (m UploadManager) pingWorker() error {
for {
stats, err := doPing(conf.Host, conf.PPP, time.Second)
if err != nil {
log.Warningf("ping to %s failed: %v", conf.Host, err)
time.Sleep(time.Duration(conf.Interval) * time.Minute)
continue
}
log.Debugf("[PING] %s±%s", stats.AvgRtt, stats.StdDevRtt)
if stats.AvgRtt.Milliseconds() >= conf.ThrottlePingThreshold {
m.LastOver = time.Now()
up, err := m.Upload()
if err != nil {
return err
}
if m.isBWInRange(up - conf.Bandwidth.Step) {
log.Infof("Ping over threshold (%s) detected, confirming...", stats.AvgRtt)
stats, err = doPing(conf.Host, conf.ConformationPPP, time.Second)
if err != nil {
log.Warningf("ping to %s failed: %v", conf.Host, err)
time.Sleep(time.Duration(conf.Interval) * time.Minute)
continue
}
if stats.AvgRtt.Milliseconds() >= conf.ThrottlePingThreshold {
log.Infof("Ping confirmed, adjusting upload TC %d->%d", up, up-conf.Bandwidth.Step)
err = m.SetUpload(up - conf.Bandwidth.Step)
if err != nil {
return err
}
m.LastOver = time.Now()
}
} else {
log.Infof("Ping over threshold (%s) detected, but no bandwidth available", stats.AvgRtt)
m.LastOver = time.Now()
}
} else if time.Since(m.LastOver) >= time.Duration(conf.TryRestoringAfter)*time.Minute {
up, err := m.Upload()
if err != nil {
return err
}
if m.isBWInRange(up + conf.Bandwidth.Step) {
log.Infof("trying to increase upload TC %d->%d", up, up+conf.Bandwidth.Step)
err = m.SetUpload(up + conf.Bandwidth.Step)
if err != nil {
return err
}
m.LastOver = time.Now()
}
}
time.Sleep(time.Duration(conf.Interval) * time.Minute)
}
}
func main() {
killSignals := make(chan os.Signal, 1)
signal.Notify(killSignals, syscall.SIGINT, syscall.SIGTERM)
reloadSignals := make(chan os.Signal, 1)
signal.Notify(reloadSignals, syscall.SIGUSR1)
flag.Parse()
confStr, err := os.ReadFile("config.yaml")
if err != nil {
log.Fatalf("Error reading config file: %v", err)
}
err = yaml.Unmarshal(confStr, &conf)
if err != nil {
log.Fatalf("Error parsing config file: %v", err)
}
lvl, err := log.ParseLevel(conf.LogLevel)
if err != nil {
log.Fatalf("Error parsing log level from config: %v", err)
}
log.SetLevel(lvl)
if *journalLog {
journalhook.Enable()
}
um := new(UploadManager)
go func() {
err := um.pingWorker()
if err != nil {
log.Fatalf("Error in ping-loop: %v", err)
}
}()
killLoop:
for {
select {
case <-killSignals:
break killLoop
}
}
}