meteorplan的博客

# elk总结

#获取现有es索引格式，通过settings和mapping返回结果组装
#查看settings
http://172.17.110.248:9200/goods_search/_settings
GET /goods_search/_settings
#查看mapping
http://172.17.110.248:9200/goods_search/_mapping
GET /goods_search/_mapping

#查询索引
GET /_cat/indices?v

# elk安装(单节点)

#路径
/usr/local/docker/elk


#创建logstash配置
logstash/
├── config/
│   └── logstash.yml
└── pipeline/
    └── logstash.conf


mkdir config
vim config/logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]



mkdir pipeline
vim pipeline/logstash.conf

input {
  beats {
    port => 5000
  }
}

filter {
  grok {
    match => { "message" => "%{COMMONAPACHELOG}" }
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    index => "logstash-%{+YYYY.MM.dd}"
  }
  stdout { codec => rubydebug }
}



#创建
vim docker-compose.yml

version: '3.8'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: elasticsearch
    environment:
      - "cluster.name=elasticsearch"
      - "discovery.type=single-node"
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - /usr/local/docker/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
      - /usr/local/docker/elk/elasticsearch/config:/usr/share/elasticsearch/config
#      - /usr/local/docker/elk/elasticsearch/data:/usr/share/elasticsearch/data
      - ./elasticsearch/data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - elk

  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.1
    container_name: kibana
#    volumes:
#      - /usr/local/docker/elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
    depends_on:
      - elasticsearch
    ports:
      - 5601:5601
    networks:
      - elk

  logstash:
    image: docker.elastic.co/logstash/logstash:7.15.1
    container_name: logstash
    volumes:
#      - /usr/local/docker/elk/logstash:/usr/share/logstash
#      - ./logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
      - /usr/local/docker/elk/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
    depends_on:
      - elasticsearch
    ports:
      - 4560:4560
      - 5044:5044
    networks:
      - elk

networks:
  elk:
    driver: bridge


#启动
docker-compose up -d


#无法启动时拷贝容器内目录
docker cp elasticsearch:/usr/share/elasticsearch/config .


#elasticsearch访问
http://172.17.110.248:9200/
#kibana访问
http://172.17.110.248:5601/
#logstash访问
http://172.17.110.248:9600/

# elk安装(集群)

#路径
/usr/local/docker/elk-cluster


#创建logstash配置
├── docker-compose.yml
├── logstash/
│   ├── config/
│   │   └── logstash.yml
│   └── pipeline/
│       └── logstash.conf


mkdir logstash

mkdir config
vim config/logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch1:9200" ]



mkdir pipeline
vim pipeline/logstash.conf

input {
  beats {
    port => 5000
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch1:9200"]
    index => "logstash-%{+YYYY.MM.dd}"
  }
  stdout { codec => rubydebug }
}



#创建
vim docker-compose.yml

version: '3.7'

services:
  elasticsearch1:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
    container_name: elasticsearch1
    environment:
      - node.name=elasticsearch1
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=elasticsearch2,elasticsearch3
      - cluster.initial_master_nodes=elasticsearch1,elasticsearch2,elasticsearch3
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata1:/usr/share/elasticsearch/data
    ports:
      - 9200:9200  # 端口映射
    networks:
      - esnet

  elasticsearch2:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
    container_name: elasticsearch2
    environment:
      - node.name=elasticsearch2
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=elasticsearch1,elasticsearch3
      - cluster.initial_master_nodes=elasticsearch1,elasticsearch2,elasticsearch3
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata2:/usr/share/elasticsearch/data
    ports:
      - 9201:9200  # 端口映射
    networks:
      - esnet

  elasticsearch3:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
    container_name: elasticsearch3
    environment:
      - node.name=elasticsearch3
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=elasticsearch1,elasticsearch2
      - cluster.initial_master_nodes=elasticsearch1,elasticsearch2,elasticsearch3
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata3:/usr/share/elasticsearch/data
    ports:
      - 9202:9200  # 端口映射
    networks:
      - esnet

  kibana:
    image: docker.elastic.co/kibana/kibana:7.17.9
    container_name: kibana
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch1:9200
    ports:
      - 5601:5601
    networks:
      - esnet

  logstash:
    image: docker.elastic.co/logstash/logstash:7.17.9
    container_name: logstash
    ports:
      - "5000:5000"
      - "9600:9600"
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
    networks:
      - esnet

volumes:
  esdata1:
    driver: local
  esdata2:
    driver: local
  esdata3:
    driver: local

networks:
  esnet:
    driver: bridge


#启动
docker-compose up -d


#elasticsearch访问
http://172.17.110.248:9200/
#kibana访问
http://172.17.110.248:5601/
#logstash访问
http://172.17.110.248:9600/




#错误提示：max virtual memory areas vm.max_map_count

#查看当前设置
sysctl vm.max_map_count
#临时设置
sudo sysctl -w vm.max_map_count=262144
#永久设置 /etc/sysctl.conf或/etc/sysctl.d/
vm.max_map_count=262144

# 微服务集成elasticsearch

#pom
    <dependency>
        <groupId>net.logstash.logback</groupId>
        <artifactId>logstash-logback-encoder</artifactId>
        <version>5.3</version>
    </dependency>
#resources
logback-spring.xml

#yml

#es配置
elasticsearch:
  client:
    ipAddrs: 172.17.110.248:9200,172.17.110.248:9201,172.17.110.248:9202
    connectTimeOut: 5000
    socketTimeOut: 6000
    connectionRequestTime: 5000
    maxConnectNum: 60
    maxConnectPerRoute: 30
#    username: es
#    password: es@123
  # 商品搜索索引
  goodsSearchIndex: product_descx
  userSearchIndex: user_search


# logback日志配置
log:
  level: DEBUG
  path: /opt/logs/lx-consumer-feign
  name: lx-consumer-feign
  package: com.meteorplan
  recently:
    maxSize: 2 mb
    days: 1
    capacity: 1 gb

#创建索引（Kibana控制台 Management -> Dev Tools ）

PUT /goods_search  
{  
  "settings": {  
    "number_of_shards": 1,  
    "number_of_replicas": 1  
  },  
  "mappings": {  
    "properties": {  
      "id": {  
        "type": "integer"  
      },  
      "name": {  
        "type": "text"  
      },  
      "price": {  
        "type": "text" 
      },  
      "url": {  
        "type": "keyword"
      },  
      "type": {  
        "type": "integer"  
      },  
      "createDate": {  
        "type": "date",  
        "format": "yyyy-MM-dd HH:mm:ss"
      }  
    }  
  }  
}

#查询索引
GET /_cat/indices?v

#删除索引
DELETE /goods_search

#添加数据
POST /goods_search/_doc  
{  
  "id": 1,  
  "name": "商品名称",  
  "price": "99.99",  
  "url": "http://example.com/product1",  
  "type": 1,  
  "createDate": "2024-07-02 16:26:33"  
}

#查询数据
GET /goods_search/_search  
{  
  "query": {  
    "match_all": {}  
  }  
}



#  添加自定义词库


#关闭索引
POST /goods_search/_close

#添加词库Synonym.txt
PUT /goods_search/_settings
{
  "index": {
    "analysis": {
      "filter": {
        "my_synonym_filter": {
          "type": "synonym",
          "synonyms_path": "config/Synonym.txt"
        }
      }
    }
  }
}

#开启索引
POST /goods_search/_open


#查看是否生效
GET /goods_search/_settings


# GET /cs_product_descx/_settings

{
  "cs_product_descx" : {
    "settings" : {
      "index" : {
        "number_of_shards" : "1",
        "provided_name" : "cs_product_descx",
        "similarity" : {
          "my_bm25" : {
            "type" : "BM25",
            "b" : "0",
            "k1" : "0"
          }
        },
        "creation_date" : "1678095211349",
        "analysis" : {
          "filter" : {
            "my_synonym_filter" : {
              "type" : "synonym",
              "synonyms_path" : "Synonym.txt"
            }
          },
          "analyzer" : {
            "my_hanlp_index" : {
              "filter" : [
                "lowercase",
                "my_synonym_filter"
              ],
              "char_filter" : [
                "html_strip"
              ],
              "type" : "custom",
              "tokenizer" : "hanlp_index"
            },
            "my_hanlp" : {
              "filter" : [
                "lowercase",
                "my_synonym_filter"
              ],
              "char_filter" : [
                "html_strip"
              ],
              "type" : "custom",
              "tokenizer" : "my_hanlp"
            }
          },
          "tokenizer" : {
            "my_hanlp" : {
              "type" : "hanlp"
            }
          }
        },
        "number_of_replicas" : "1",
        "uuid" : "DbpfQkpYTjmZl61fTLAfig",
        "version" : {
          "created" : "7080199"
        }
      }
    }
  }
}


# GET /cs_product_descx/_mapping
{
  "cs_product_descx" : {
    "mappings" : {
      "properties" : {
        "activityId" : {
          "type" : "keyword"
        },
        "createdTime" : {
          "type" : "long"
        },
        "deleteStatus" : {
          "type" : "keyword"
        },
        "depositAmount" : {
          "type" : "float"
        },
        "description" : {
          "type" : "text",
          "analyzer" : "ik_max_word"
        },
        "exclusiveGroup" : {
          "type" : "keyword"
        },
        "exclusiveType" : {
          "type" : "integer"
        },
        "faceValue" : {
          "type" : "float"
        },
        "giftMsg" : {
          "type" : "text",
          "analyzer" : "my_hanlp"
        },
        "giftStock" : {
          "type" : "keyword"
        },
        "goodsBrandId" : {
          "type" : "long"
        },
        "goodsType" : {
          "type" : "keyword"
        },
        "id" : {
          "type" : "keyword"
        },
        "incomeRate" : {
          "type" : "keyword"
        },
        "keywords" : {
          "type" : "text",
          "analyzer" : "ik_max_word"
        },
        "merchantSort" : {
          "type" : "integer"
        },
        "merchantUuid" : {
          "type" : "keyword"
        },
        "name" : {
          "type" : "text",
          "analyzer" : "ik_max_word"
        },
        "newStatus" : {
          "type" : "keyword"
        },
        "newUserPrice" : {
          "type" : "float"
        },
        "originalPrice" : {
          "type" : "float"
        },
        "pcsId" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "pic" : {
          "type" : "keyword"
        },
        "prefectureType" : {
          "type" : "integer"
        },
        "price" : {
          "type" : "float"
        },
        "productCategoryId" : {
          "type" : "keyword"
        },
        "productCategoryName" : {
          "type" : "keyword"
        },
        "productCategorySort" : {
          "type" : "integer"
        },
        "productMerchantCategorySort" : {
          "type" : "integer"
        },
        "productType" : {
          "type" : "integer"
        },
        "promotionPrice" : {
          "type" : "float"
        },
        "recommandStatus" : {
          "type" : "keyword"
        },
        "rewardsMsg" : {
          "type" : "text",
          "analyzer" : "my_hanlp"
        },
        "sale" : {
          "type" : "integer"
        },
        "showOriginal" : {
          "type" : "integer"
        },
        "showRegionId" : {
          "type" : "keyword"
        },
        "sort" : {
          "type" : "integer"
        },
        "updatedTime" : {
          "type" : "long"
        }
      }
    }
  }
}

# kibana

PUT /logstash-2024.07.03
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}

image-20240703153918085

image-20240703152533762

源数据 _index

倒排索引 基于搜索内容排列出对应id

← git →