Модуль локального сбора логов (Vector)

apiVersion: observability.kaasops.io/v1alpha1
kind: ClusterVectorPipeline
metadata:
  name: k8s-logs
  namespace: monitoring
spec:
  # Источник данных - логи Kubernetes
  sources:
    kubernetes_logs:
      type: kubernetes_logs

  # Цепочка трансформаций
  transforms:
    # Преобразование временной метки
    remap_timestamp:
      type: remap
      inputs: [kubernetes_logs]
      source: |
        parsed_time = parse_timestamp!(.timestamp, format: "%+")
        if is_timestamp(parsed_time) {
          ."@timestamp" = parsed_time
          del(.timestamp)
        }        

    # Замена точек на подчеркивания в именах полях
    dedot_fields:
      type: remap
      inputs: [remap_timestamp]
      source: |
                . = map_keys(., recursive: true) -> |key| { replace(key, ".", "_") }

    # Фильтрация системных пространств имен
    filter_system_namespaces:
      type: filter
      inputs: [dedot_fields]
      condition: |
                exists(.kubernetes.namespace_labels."shturval_tech/system-namespace")

    # Фильтрация пользовательских пространств имен
    filter_user_namespaces:
      type: filter
      inputs: [dedot_fields]
      condition: |
                !exists(.kubernetes.namespace_labels."shturval_tech/system-namespace")

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: <имя ресурса>
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval: 
      k8s_audit: 
        enable: true
        sinks: 
          siem: 
            type: socket
            inputs: 
            - dedot_specific_fields
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json
      journald: 
        sinks: 
          siem: 
            type: socket
            inputs: 
            - journald
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json
      k8s_logs:  
        transforms: 
          filter_backend: 
            condition: |
              .kubernetes.pod_namespace == "shturval-backend" &&
              ((.kubernetes.container_name == "opa" && starts_with(string!(.kubernetes.pod_name), "shturval-backend-")) || (.kubernetes.container_name == "authn" &starts_with    (string!(.kubernetes.pod_name), "authn-")))              
            inputs: 
            - filter_system_namespaces
            type: filter
        sinks: 
          siem: 
            type: socket
            inputs: 
            - filter_backend
            address: "<ваше значение параметра>"
            mode: <ваше значение параметра>
            encoding: 
              codec: json

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: <имя ресурса>
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      <ваше значение>: # Тип логов для перенаправления
        enable: true # Включает отправку логов elasticsearch
        transforms: {} # Преобразование логов
        sinks: # Конечные точки для отправки данных
          elastic_bank: # Имя получателя данных
            type: <ваше значение параметра>
            auth:  # Настройки аутентификации для подключения к elasticsearch
              user: <ваше значение параметра>
              password: <ваше значение параметра>
              strategy: <ваше значение параметра>
            endpoints: # URL-адрес для подключения
              - <ваше значение параметра>
            tls: # Настройки TLS-соединения
              verify_certificate: true
            api_version: <ваше значение параметра>
            buffer: # Настройка буфера для хранения логов перед отправкой 
              max_size: <ваше значение параметра> 
              type: <ваше значение параметра>

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-auditd
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      auditd:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - auditd-fmt
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_auditd-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-kaudit
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      k8s_audit:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs: 
              - dedot_specific_fields
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_kaudit-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-kevents
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      k8s_events:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - delete_managed_fields
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_events-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-klogs
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      k8s_logs:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - post_transforms_system_namespaces
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_logs-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk
          user-namespaces:
            inputs:
              - post_transforms_system_namespaces
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_user-logs-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-journald
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      journald:
        sinks:
          elastic_bank:
            type: elasticsearch
            inputs:
              - journald
            bulk:
              index: "{{ .Values.CLUSTER_NAME }}_journald-%Y.%m.%d"
            auth:
              user: my_username
              password: "my_password"
              strategy: basic
            endpoints:
              - "https://external-opensearch.address:9200"
            tls:
              verify_certificate: false
            api_version: "v8"
            buffer:
              max_size: 268435488
              type: disk

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-syslog
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      k8s_audit:
        sinks:
          syslog:
            type: socket 
            inputs: 
             - <ваше значение параметра>
            address: <ваше значение параметра>
            mode: <ваше значение параметра>

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-disable
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      platform_logs_enable: false

apiVersion: ops.shturval.tech/v1beta2
kind: ShturvalServicePatch
metadata:
  name: shturval-log-collector-disable-journald
spec:
  shturvalServiceConfigName: shturval-log-collector
  customvalues:
    shturval:
      journald:
        enable: false