nixos/modules/monitoring/monitoring.nix

{ config, pkgs, ... }:
let
  configure_prom = builtins.toFile "prometheus.yml" ''
    scrape_configs:
    - job_name: 'bebop'
      stream_parse: true
      static_configs:
      - targets:
        - 127.0.0.1:9100
  '';
in
{
  # Open grafana port, but only for local network
  networking.firewall.extraCommands = ''
    iptables -A nixos-fw -p tcp --source 192.168.0.0/24 --dport 2342:2342 -j nixos-fw-accept
    iptables -A nixos-fw -p udp --source 192.168.0.0/24 --dport 2342:2342 -j nixos-fw-accept
  '';

  networking.firewall.allowedTCPPorts = [ 9100 ];
  services = {
    victoriametrics = {
      enable = true;
      retentionPeriod = 48; # months. Should be enough, right?
    };

    prometheus = {
      # Enable prometheus node_exporter
      exporters = {
        node = {
          enable = true;
          enabledCollectors = [
            "systemd"
          ];
          port = 9100;
        };
      };
    };

    loki = {
      enable = true;
      configuration = {
        server.http_listen_port = 3030;
        auth_enabled = false;

        ingester = {
          lifecycler = {
            address = "127.0.0.1";
            ring = {
              kvstore.store = "inmemory";
              replication_factor = 1;
            };
          };
          chunk_idle_period = "1h";
          max_chunk_age = "1h";
          chunk_target_size = 999999;
          chunk_retain_period = "30s";
          max_transfer_retries = 0;
        };
        schema_config = {
          configs = [{
            from = "2022-06-06";
            store = "boltdb-shipper";
            object_store = "filesystem";
            schema = "v11";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }];
        };

        storage_config = {
          boltdb_shipper = {
            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
            cache_location = "/var/lib/loki/boltdb-shipper-cache";
            cache_ttl = "24h";
            shared_store = "filesystem";
          };

          filesystem = {
            directory = "/var/lib/loki/chunks";
          };
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
        };

        chunk_store_config = {
          max_look_back_period = "0s";
        };

        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };

        compactor = {
          working_directory = "/var/lib/loki";
          shared_store = "filesystem";
          compactor_ring = {
            kvstore = {
              store = "inmemory";
            };
          };
        };
      };
    };

    promtail = {
      enable = true;
      configuration = {
        server = {
          http_listen_port = 3031;
          grpc_listen_port = 0;
        };
        positions = {
          filename = "/tmp/positions.yaml";
        };
        clients = [{
          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
        }];
        scrape_configs = [{
          job_name = "journal";
          journal = {
            max_age = "12h";
            labels = {
              job = "systemd-journal";
              host = "bebop";
            };
          };
          relabel_configs = [{
            source_labels = [ "__journal__systemd_unit" ];
            target_label = "unit";
          }];
        }];
      };
    };

    # Show data with fancy graphs
    grafana = {
      enable = true;
      provision = {
        enable = true;
        datasources.settings.datasources = [
          {
            name = "VictoriaMetrics (Prometheus API)";
            type = "prometheus";
            access = "proxy";
            url = "http://127.0.0.1:8428";
          }
          {
            name = "Loki";
            type = "loki";
            access = "proxy";
            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
          }
        ];
      };
      settings = {
        feature_toggles = {
          publicDashboards = false;
        };
        analytics.reporting_enabled = false;
        server = {
          domain = "bebop.bhankas.org";
          http_port = 2342;
          http_addr = "0.0.0.0";
        };
        security.csrf_trusted_origins = "https://bebop.bhankas.org,127.0.0.1";
        live.allowed_origins = "127.0.0.1,https://bebop.bhankas.org,bebop.bhankas.org";
      };
    };

    nginx = {
      virtualHosts = {
        "bebop.bhankas.org" = {
          addSSL = true;
          enableACME = true;
          locations."/" = {
            proxyPass = "http://127.0.0.1:2342";
            proxyWebsockets = true;
            extraConfig =
              "proxy_set_header Host $host;"
            ;
          };
        };
      };
    };
  };

  security.acme = {
    acceptTerms = true;
    certs = {
      "bebop.bhankas.org" = {
        email = "admin@bhankas.org";
        dnsResolver = "1.1.1.1:53";
      };
    };
  };

  systemd.services.export-to-prometheus = {
    path = with pkgs; [ victoriametrics ];
    enable = true;
    after = [ "network-online.target" ];
    requires = [ "network-online.target" ];
    wantedBy = [ "multi-user.target" ];
    script = "vmagent -promscrape.config=${configure_prom} -remoteWrite.url=http://127.0.0.1:8428/api/v1/write";
  };
}