From 075ba1614b5f583a203a199b4278d5ba48bc8fcf Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Tue, 31 Jan 2023 17:17:22 +0000 Subject: [PATCH] Make more robust to SNMP failures --- .gitignore | 2 + flake.lock | 46 +++++++++++++++-- flake.nix | 14 +++++- main.go | 142 ++++++++++++++++++++++++++++++++++------------------- 4 files changed, 147 insertions(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index 5b6c096..41f0887 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ config.toml +apc2mqtt +.direnv diff --git a/flake.lock b/flake.lock index 1f1ec81..0608ff1 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,26 @@ { "nodes": { + "devshell": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1649691969, + "narHash": "sha256-nY1aUWIyh3TcGVo3sn+3vyCh+tOiEZL4JtMX3aOZSeY=", + "owner": "numtide", + "repo": "devshell", + "rev": "e22633b05fec2fe196888c593d4d9b3f4f648a25", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1637014545, @@ -17,11 +38,27 @@ }, "nixpkgs": { "locked": { - "lastModified": 1637156900, - "narHash": "sha256-nusyaSsL1RLyUEWufUUywDrGKMXw+4ugSSZ3ss8TSuw=", + "lastModified": 1643381941, + "narHash": "sha256-pHTwvnN4tTsEKkWlXQ8JMY423epos8wUOhthpwJjtpc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "12fc0f19fefa9dff68bc3e0938b815ab8d89df90", + "rev": "5efc8ca954272c4376ac929f4c5ffefcc20551d5", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1675153841, + "narHash": "sha256-EWvU3DLq+4dbJiukfhS7r6sWZyJikgXn6kNl7eHljW8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ea692c2ad1afd6384e171eabef4f0887d2b882d3", "type": "github" }, "original": { @@ -31,8 +68,9 @@ }, "root": { "inputs": { + "devshell": "devshell", "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs_2" } } }, diff --git a/flake.nix b/flake.nix index 1ae0b88..7c261e8 100644 --- a/flake.nix +++ b/flake.nix @@ -2,10 +2,18 @@ description = "Samw's APC PDU MQTT bridge"; inputs.flake-utils.url = "github:numtide/flake-utils"; + inputs.devshell = { + url = "github:numtide/devshell"; + inputs.flake-utils.follows = "flake-utils"; + }; - outputs = { self, nixpkgs, flake-utils }: + outputs = { self, nixpkgs, flake-utils, devshell }: flake-utils.lib.eachDefaultSystem (system: - let pkgs = import nixpkgs { inherit system; }; + let + pkgs = import nixpkgs { + inherit system; + overlays = [ devshell.overlay ]; + }; in rec { packages.apc2mqtt = pkgs.buildGoModule { name = "apc2mqtt"; @@ -16,5 +24,7 @@ ''; }; defaultPackage = packages.apc2mqtt; + devShell = + pkgs.devshell.mkShell { packages = with pkgs; [ go gopls mosquitto ]; }; }); } diff --git a/main.go b/main.go index a7663f7..4056d08 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,8 @@ type config struct { MQTT struct { Host string Port uint16 + User string + Pass string } Targets []targetConfig } @@ -76,6 +78,38 @@ type PDUCommand struct { State bool } +func getPDUState(snmp *gosnmp.GoSNMP) (*PDUState, error) { + res, err := snmp.Get([]string{sPDUMasterConfigPDUName, sPDUIdentSerialNumber, sPDUIdentModelNumber}) + if err != nil { + return nil, fmt.Errorf("get PDU info: %w", err) + } + state := PDUState{ + Name: string(res.Variables[0].Value.([]byte)), + Serial: string(res.Variables[1].Value.([]byte)), + Model: string(res.Variables[2].Value.([]byte)), + } + + outletNames, err := snmp.WalkAll(sPDUOutletName) + if err != nil { + return nil, fmt.Errorf("walk PDU outlet names: %w", err) + } + + for _, val := range outletNames { + state.Outlets = append(state.Outlets, Outlet{ + Name: string(val.Value.([]byte)), + }) + } + + outletStates, err := snmp.WalkAll(sPDUOutletCtl) + if err != nil { + return nil, fmt.Errorf("walk PDU outlet states: %w", err) + } + for i, val := range outletStates { + state.Outlets[i].State = val.Value.(int) == 1 + } + return &state, nil +} + func runSNMP(host string, port uint16, stateCh chan PDUState, commandCh chan PDUCommand) { snmp := &gosnmp.GoSNMP{ Target: host, @@ -86,51 +120,45 @@ func runSNMP(host string, port uint16, stateCh chan PDUState, commandCh chan PDU Version: gosnmp.Version1, Retries: 3, } - err := snmp.Connect() - check(err) - defer snmp.Conn.Close() - poll := time.NewTicker(1 * time.Second) + // Connect loop +connect_loop: for { - select { - case <-poll.C: - res, err := snmp.Get([]string{sPDUMasterConfigPDUName, sPDUIdentSerialNumber, sPDUIdentModelNumber}) - check(err) - state := PDUState{ - Name: string(res.Variables[0].Value.([]byte)), - Serial: string(res.Variables[1].Value.([]byte)), - Model: string(res.Variables[2].Value.([]byte)), - } - - outletNames, err := snmp.WalkAll(sPDUOutletName) - check(err) - - for _, val := range outletNames { - state.Outlets = append(state.Outlets, Outlet{ - Name: string(val.Value.([]byte)), - }) - } - - outletStates, err := snmp.WalkAll(sPDUOutletCtl) - check(err) - for i, val := range outletStates { - state.Outlets[i].State = val.Value.(int) == 1 - } - stateCh <- state - case cmd := <-commandCh: - var value int - if cmd.State { - value = 1 - } else { - value = 2 - } - res, err := snmp.Set([]gosnmp.SnmpPDU{{ - Value: value, - Name: fmt.Sprintf("%s.%d", sPDUOutletCtl, cmd.Outlet), - Type: gosnmp.Integer, - }}) - check(err) - if res.Error != gosnmp.NoError { - log.Errorf("error in snmp set: %s", res.Error) + log.Info("Opening SNMP connection") + if err := snmp.Connect(); err != nil { + log.Warnf("Error connecting SNMP target %s: `%s`. Sleeping...", host, err.Error()) + time.Sleep(time.Second * 5) + continue // Try again + } + defer snmp.Conn.Close() + poll := time.NewTicker(1 * time.Second) + for { + select { + case <-poll.C: + state, err := getPDUState(snmp) + if err != nil { + log.Errorf("getting pdu state: %s", err.Error()) + continue connect_loop // Reconnect + } + log.Debugf("Got state: %s", state) + stateCh <- *state + case cmd := <-commandCh: + var value int + if cmd.State { + value = 1 + } else { + value = 2 + } + res, err := snmp.Set([]gosnmp.SnmpPDU{{ + Value: value, + Name: fmt.Sprintf("%s.%d", sPDUOutletCtl, cmd.Outlet), + Type: gosnmp.Integer, + }}) + if err != nil { + log.Errorf("setting pdu state: %s", err.Error()) + } + if res.Error != gosnmp.NoError { + log.Errorf("error in snmp set: %s", res.Error) + } } } } @@ -202,21 +230,33 @@ func spawnTarget(target targetConfig, mqttClient mqtt.Client) { } func main() { var configpath = flag.String("conf", "config.toml", "Path to toml config file") + var verbose = flag.Bool("v", false, "Enable debug logging") flag.Parse() + if *verbose { + log.SetLevel(log.DebugLevel) + } log.Infof("Starting version %s", version) conf, err := parseConfig(*configpath) check(err) mqttOpts := mqtt.NewClientOptions().AddBroker( fmt.Sprintf("tcp://%s:%d", conf.MQTT.Host, conf.MQTT.Port), - ).SetClientID("apc2mqtt") + ).SetClientID("apc2mqtt").SetConnectTimeout(time.Second * 5) + if conf.MQTT.User != "" { + mqttOpts.SetUsername(conf.MQTT.User) + } + if conf.MQTT.Pass != "" { + mqttOpts.SetPassword(conf.MQTT.Pass) + } mqttClient := mqtt.NewClient(mqttOpts) - if token := mqttClient.Connect(); token.Wait() && token.Error() != nil { - panic(token.Error()) - } + for { + if token := mqttClient.Connect(); token.Wait() && token.Error() != nil { + log.Fatalf("connect mqtt: %s", token.Error()) + } - for _, target := range conf.Targets { - go spawnTarget(target, mqttClient) + for _, target := range conf.Targets { + go spawnTarget(target, mqttClient) + } + select {} } - select {} }