Модуль локального сбора логов (Vector)

apiVersion: observability.kaasops.io/v1alpha1
kind: ClusterVectorPipeline
metadata:
  name: k8s-logs
  namespace: monitoring
spec:
  # Источник данных - логи Kubernetes
  sources:
    kubernetes_logs:
      type: kubernetes_logs

  # Цепочка трансформаций
  transforms:
    # Преобразование временной метки
    remap_timestamp:
      type: remap
      inputs: [kubernetes_logs]
      source: |
        parsed_time = parse_timestamp!(.timestamp, format: "%+")
        if is_timestamp(parsed_time) {
          ."@timestamp" = parsed_time
          del(.timestamp)
        }        

    # Замена точек на подчеркивания в именах полях
    dedot_fields:
      type: remap
      inputs: [remap_timestamp]
      source: |
                . = map_keys(., recursive: true) -> |key| { replace(key, ".", "_") }

    # Фильтрация системных пространств имен
    filter_system_namespaces:
      type: filter
      inputs: [dedot_fields]
      condition: |
                exists(.kubernetes.namespace_labels."shturval_tech/system-namespace")

    # Фильтрация пользовательских пространств имен
    filter_user_namespaces:
      type: filter
      inputs: [dedot_fields]
      condition: |
                !exists(.kubernetes.namespace_labels."shturval_tech/system-namespace")

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: <имя ресурса>
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval: 
      k8s_audit: 
        enable: true
        sinks: 
          siem: 
            type: socket
            inputs: 
            - dedot_specific_fields
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json
      journald: 
        sinks: 
          siem: 
            type: socket
            inputs: 
            - journald
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json
      k8s_logs:  
        transforms: 
          filter_backend: 
            condition: |
              .kubernetes.pod_namespace == "shturval-backend" &&
              ((.kubernetes.container_name == "opa" && starts_with(string!(.kubernetes.pod_name), "shturval-backend-")) || (.kubernetes.container_name == "authn" &starts_with    (string!(.kubernetes.pod_name), "authn-")))              
            inputs: 
            - filter_system_namespaces
            type: filter
        sinks: 
          siem: 
            type: socket
            inputs: 
            - filter_backend
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json

apiVersion: v1
kind: Secret
metadata:
  name: external-es-ca
  namespace: logging
type: Opaque
stringData:
  ca.crt: |
    -----BEGIN CERTIFICATE-----
    <CA-сертификат внешнего Elasticsearch/OpenSearch>
    -----END CERTIFICATE-----    

apiVersion: v1
kind: Secret
metadata:
  name: external-es-creds
  namespace: logging
type: Opaque
stringData:
  username: <username>
  password: <password>

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: <имя ресурса>
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    vector:
      agent:
        envFrom: # Все ключи Secret — переменные окружения агента
          - secretRef:
              name: <secret-creds>
        volumes: # Тома агента; список полностью заменяет values чарта
          - name: var-log
            hostPath:
              path: /var/log/
              type: ""
          - name: journal
            hostPath:
              path: /var/log/journal/
              type: ""
          - name: var-lib
            hostPath:
              path: /var/lib/
              type: ""
          - name: <volume-ca>
            secret:
              secretName: <secret-ca>
              items:
                - key: ca.crt
                  path: ca.crt
        volumeMounts: # Монтирование томов в контейнер агента
          - name: var-log
            mountPath: /var/log/
          - name: journal
            mountPath: /run/log/journal/
          - name: var-lib
            mountPath: /var/lib/
          - name: <volume-ca>
            mountPath: /etc/ssl/external-es
            readOnly: true
    aggregator:
      env: # Переменные окружения агрегатора; список полностью заменяет values чарта
        - name: PLATFORM_USERNAME
          valueFrom:
            secretKeyRef:
              name: platform-creds
              key: username
        - name: PLATFORM_PASSWORD
          valueFrom:
            secretKeyRef:
              name: platform-creds
              key: password
        - name: username
          valueFrom:
            secretKeyRef:
              name: <secret-creds>
              key: username
        - name: password
          valueFrom:
            secretKeyRef:
              name: <secret-creds>
              key: password
      volumes: # Тома агрегатора; список полностью заменяет values чарта
        - name: data
          emptyDir: {}
        - name: <volume-ca>
          secret:
            secretName: <secret-ca>
            items:
              - key: ca.crt
                path: ca.crt
      volumeMounts: # Монтирование томов в контейнер агрегатора
        - name: <volume-ca>
          mountPath: /etc/ssl/external-es
          readOnly: true
    shturval:
      <ваше значение>: # Тип логов для перенаправления
        enable: true # Включает отправку логов elasticsearch
        transforms: {} # Преобразование логов
        sinks: # Конечные точки для отправки данных
          elastic_bank: # Имя получателя данных
            type: <ваше значение параметра>
            auth:  # Настройки аутентификации для подключения к elasticsearch
              user: "${username}"
              password: "${password}"
              strategy: <ваше значение параметра>
            endpoints: # URL-адрес для подключения
              - <ваше значение параметра>
            tls: # Настройки TLS-соединения
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: <ваше значение параметра>
            buffer: # Настройка буфера для хранения логов перед отправкой 
              max_size: <ваше значение параметра> 
              type: <ваше значение параметра>

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-external-es
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    vector:
      agent:
        envFrom:
          - secretRef:
              name: external-es-creds
        volumes:
          - name: var-log
            hostPath:
              path: /var/log/
              type: ""
          - name: journal
            hostPath:
              path: /var/log/journal/
              type: ""
          - name: var-lib
            hostPath:
              path: /var/lib/
              type: ""
          - name: external-es-ca
            secret:
              secretName: external-es-ca
              items:
                - key: ca.crt
                  path: ca.crt
        volumeMounts:
          - name: var-log
            mountPath: /var/log/
          - name: journal
            mountPath: /run/log/journal/
          - name: var-lib
            mountPath: /var/lib/
          - name: external-es-ca
            mountPath: /etc/ssl/external-es
            readOnly: true
    aggregator:
      env:
        - name: PLATFORM_USERNAME
          valueFrom:
            secretKeyRef:
              name: platform-creds
              key: username
        - name: PLATFORM_PASSWORD
          valueFrom:
            secretKeyRef:
              name: platform-creds
              key: password
        - name: username
          valueFrom:
            secretKeyRef:
              name: external-es-creds
              key: username
        - name: password
          valueFrom:
            secretKeyRef:
              name: external-es-creds
              key: password
      volumes:
        - name: data
          emptyDir: {}
        - name: external-es-ca
          secret:
            secretName: external-es-ca
            items:
              - key: ca.crt
                path: ca.crt
      volumeMounts:
        - name: external-es-ca
          mountPath: /etc/ssl/external-es
          readOnly: true
    shturval:
      auditd:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - auditd-fmt
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_auditd-%Y.%m.%d"
            auth:
              user: "${username}"
              password: "${password}"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200/insert/elasticsearch/"
            tls:
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
      journald:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - journald
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_journald-%Y.%m.%d"
            auth:
              user: "${username}"
              password: "${password}"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200/insert/elasticsearch/"
            tls:
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
      k8s_audit:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - dedot_specific_fields
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_kaudit-%Y.%m.%d"
            auth:
              user: "${username}"
              password: "${password}"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200/insert/elasticsearch/"
            tls:
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
      k8s_events:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - delete_managed_fields
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_events-%Y.%m.%d"
            auth:
              user: "${username}"
              password: "${password}"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200/insert/elasticsearch/"
            tls:
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
      k8s_logs:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - post_transforms_system_namespaces
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_logs-%Y.%m.%d"
            auth:
              user: "${username}"
              password: "${password}"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200/insert/elasticsearch/"
            tls:
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
          user-namespaces:
            type: elasticsearch
            inputs:
              - post_transforms_user_namespaces
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_user-logs-%Y.%m.%d"
            auth:
              user: "${username}"
              password: "${password}"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200/insert/elasticsearch/"
            tls:
              ca_file: /etc/ssl/external-es/ca.crt
              verify_certificate: true
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-logs-to-loki
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      platform_logs_enable: true
      k8s_logs:
        enable: true
        sinks:
          loki_external:
            type: loki
            inputs:
              - post_transforms_system_namespaces
              - post_transforms_user_namespaces
            endpoint: <ваше значение параметра>
            path: /loki/api/v1/push
            compression: snappy
            tenant_id: "<ваше значение параметра>"
            encoding:
              codec: json
            tls:
              verify_certificate: false
            labels:
              __tenant_id__: "<ваше значение параметра>"
              resourceRealm: "<ваше значение параметра>"
              log_type: "kubernetes"
      k8s_audit:
        enable: true
        sinks:
          loki_external:
            type: loki
            inputs:
              - parse_kube_audit
            endpoint: <ваше значение параметра>
            path: /loki/api/v1/push
            compression: snappy
            tenant_id: "<ваше значение параметра>"
            encoding:
              codec: json
            tls:
              verify_certificate: false
            labels:
              __tenant_id__: "<ваше значение параметра>"
              resourceRealm: "<ваше значение параметра>"
              log_type: "k8s_audit"
      k8s_events:
        enable: true
        sinks:
          loki_external:
            type: loki
            inputs:
              - delete_managed_fields
            endpoint: <ваше значение параметра>
            path: /loki/api/v1/push
            compression: snappy
            tenant_id: "<ваше значение параметра>"
            encoding:
              codec: json
            tls:
              verify_certificate: false
            labels:
              __tenant_id__: "<ваше значение параметра>"
              resourceRealm: "<ваше значение параметра>"
              log_type: "k8s_events"
      journald:
        enable: true
        sinks:
          loki_external:
            type: loki
            inputs:
              - journald
            endpoint: <ваше значение параметра>
            path: /loki/api/v1/push
            compression: snappy
            tenant_id: "<ваше значение параметра>"
            encoding:
              codec: json
            tls:
              verify_certificate: false
            labels:
              __tenant_id__: "<ваше значение параметра>"
              resourceRealm: "<ваше значение параметра>"
              log_type: "journald"
      auditd:
        enable: true
        sinks:
          loki_external:
            type: loki
            inputs:
              - auditd
            endpoint: <ваше значение параметра>
            path: /loki/api/v1/push
            compression: snappy
            tenant_id: "<ваше значение параметра>"
            encoding:
              codec: json
            tls:
              verify_certificate: false
            labels:
              __tenant_id__: "<ваше значение параметра>"
              resourceRealm: "<ваше значение параметра>"
              log_type: "auditd"

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-syslog
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      k8s_audit:
        sinks:
          syslog:
            type: socket 
            inputs: 
             - <ваше значение параметра>
            address: <ваше значение параметра>
            mode: <ваше значение параметра>

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-disable
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      platform_logs_enable: false

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-disable-journald
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      journald:
        enable: false