Модуль локального сбора логов (Vector)

apiVersion: observability.kaasops.io/v1alpha1
kind: ClusterVectorPipeline
metadata:
  name: k8s-logs
  namespace: monitoring
spec:
  # Источник данных - логи Kubernetes
  sources:
    kubernetes_logs:
      type: kubernetes_logs

  # Цепочка трансформаций
  transforms:
    # Преобразование временной метки
    remap_timestamp:
      type: remap
      inputs: [kubernetes_logs]
      source: |
        parsed_time = parse_timestamp!(.timestamp, format: "%+")
        if is_timestamp(parsed_time) {
          ."@timestamp" = parsed_time
          del(.timestamp)
        }        

    # Замена точек на подчеркивания в именах полях
    dedot_fields:
      type: remap
      inputs: [remap_timestamp]
      source: |
                . = map_keys(., recursive: true) -> |key| { replace(key, ".", "_") }

    # Фильтрация системных пространств имен
    filter_system_namespaces:
      type: filter
      inputs: [dedot_fields]
      condition: |
                exists(.kubernetes.namespace_labels."shturval_tech/system-namespace")

    # Фильтрация пользовательских пространств имен
    filter_user_namespaces:
      type: filter
      inputs: [dedot_fields]
      condition: |
                !exists(.kubernetes.namespace_labels."shturval_tech/system-namespace")

    shturval: 
      k8s_audit: 
        enable: true
        sinks: 
          siem: 
            type: socket
            inputs: 
            - critical_level
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json
      journald: 
        sinks: 
          siem: 
            type: socket
            inputs: 
            - critical_level
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json
      k8s_logs:  
        transforms: 
          filter_backend: 
            condition: |
              .kubernetes.pod_namespace == "shturval-backend" &&
              ((.kubernetes.container_name == "opa" && starts_with(string!(.kubernetes.pod_name), "shturval-backend-")) || (.kubernetes.container_name == "authn" &starts_with    (string!(.kubernetes.pod_name), "authn-")))              
            inputs: 
            - post_transforms_system_namespaces
            type: filter
        sinks: 
          siem: 
            type: socket
            inputs: 
            - filter_backend
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json

    shturval:
      <ваше значение>: # Тип логов для перенаправления
        enable: true # Включает отправку логов elasticsearch
        transforms: {} # Преобразование логов
        sinks: # Конечные точки для отправки данных
          elastic_logs: # Имя получателя данных
            type: <ваше значение параметра>
            auth:  # Настройки аутентификации для подключения к elasticsearch
              user: <ваше значение параметра>
              password: <ваше значение параметра>
              strategy: <ваше значение параметра>
            endpoints: # URL-адрес для подключения
              - <ваше значение параметра>
            tls: # Настройки TLS-соединения
              verify_certificate: true
            api_version: <ваше значение параметра>
            buffer: # Настройка буфера для хранения логов перед отправкой 
              max_size: <ваше значение параметра> 
              type: <ваше значение параметра>

    shturval:
      auditd:
        sinks:
          elastic_logs:
            type: elasticsearch
            inputs:
              - auditd
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_auditd-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-elastic.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

    shturval:
      k8s_audit:
        sinks:
          elastic_logs:
            type: elasticsearch
            inputs: 
              - critical_level
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_kaudit-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-elastic.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

    shturval:
      k8s_events:
        sinks:
          elastic_logs:
            type: elasticsearch
            inputs:
              - critical_level
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_events-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-elastic.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

    shturval:
      k8s_logs:
        transforms:
          system_namespaces_message_encode:
            type: remap
            inputs:
              - post_transforms_system_namespaces
            source: |
              .message =
                encode_json(.message)                            
          user_namespaces_message_encode:
            type: remap
            inputs:
              - post_transforms_user_namespaces
            source: |
              .message =
                encode_json(.message)                            
        sinks:
          elastic_logs:
            type: elasticsearch
            inputs:
              - system_namespaces_message_encode
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_logs-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-elastic.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
          elastic_user_logs:
            type: elasticsearch
            inputs:
              - user_namespaces_message_encode
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_user-logs-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-elastic.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

    shturval:
      journald:
        sinks:
          elastic_logs:
            type: elasticsearch
            inputs:
              - critical_level
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_journald-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-elastic.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

shturval:
  k8s_audit:
    sinks:
      syslog:
        type: socket 
        inputs: 
         - <ваше значение параметра>
        address: <ваше значение параметра>
        mode: <ваше значение параметра>

shturval:
  platform_logs_enable: false

shturval:
  journald:
    enable: false