searchusermenu
  • 发布文章
  • 消息中心
点赞
收藏
评论
分享
原创

Logstash迁移Elasticsearch7.x至Opensearch2.x实战

2024-07-18 09:48:12
10
0
  1. Elasticsearch7.4.2配置
'network.host': '0.0.0.0'
'discovery.type': 'single-node'

创建mapping

curl -XPUT "localhost:9200/my_index" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "properties": {
      "text": {
        "type": "text"
      },
      "name":{
        "type": "text"
      }
    }
  },
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}'

插入数据

curl -XPOST "localhost:9200/my_index/_doc/1" -H 'Content-Type: application/json' -d'
{
"text": "student1",
"name": "xiaoming"
}'

curl -XPOST "localhost:9200/my_index/_doc/2" -H 'Content-Type: application/json' -d'
{
"text": "student2",
"name": "xiaohong"
}'

curl -XPOST "localhost:9200/my_index/_doc/3" -H 'Content-Type: application/json' -d'
{
"text": "student3",
"name": "xiaoli"
}'

curl -XPOST "localhost:9200/my_index/_doc/4" -H 'Content-Type: application/json' -d'
{
"text": "student4",
"name": "xiaozhang"
}'

2.Opensearch2.9.0配置

plugins.security.disabled: true
discovery.type: single-node
network.host: 0.0.0.0

创建mapping

curl -XPUT "localhost:9200/my_index" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "properties": {
      "text": {
        "type": "text"
      },
      "name":{
        "type": "text"
      }
    }
  },
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}'
  1. Logstash7.10.2配置
input {
      elasticsearch {
        hosts => "{IP}:9200"
        index => "my*"
        docinfo => true
   }
}

filter {
  mutate {
    remove_field => ["@timestamp", "@version"]
  }
}

output {
    opensearch {
      hosts => ["{IP}:9200"]
      index => "%{[@metadata][_index]}"
      document_id => "%{[@metadata][_id]}"
  }
}

把IP替换成实际的IP

可以看到数据被正常迁移了

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 4,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "my_index",
        "_id": "1",
        "_score": 1,
        "_source": {
          "name": "xiaoming",
          "text": "student1"
        }
      },
      {
        "_index": "my_index",
        "_id": "2",
        "_score": 1,
        "_source": {
          "name": "xiaohong",
          "text": "student2"
        }
      },
      {
        "_index": "my_index",
        "_id": "3",
        "_score": 1,
        "_source": {
          "name": "xiaoli",
          "text": "student3"
        }
      },
      {
        "_index": "my_index",
        "_id": "4",
        "_score": 1,
        "_source": {
          "name": "xiaozhang",
          "text": "student4"
        }
      }
    ]
  }
}
0条评论
0 / 1000
chenxx
3文章数
1粉丝数
chenxx
3 文章 | 1 粉丝
chenxx
3文章数
1粉丝数
chenxx
3 文章 | 1 粉丝
原创

Logstash迁移Elasticsearch7.x至Opensearch2.x实战

2024-07-18 09:48:12
10
0
  1. Elasticsearch7.4.2配置
'network.host': '0.0.0.0'
'discovery.type': 'single-node'

创建mapping

curl -XPUT "localhost:9200/my_index" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "properties": {
      "text": {
        "type": "text"
      },
      "name":{
        "type": "text"
      }
    }
  },
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}'

插入数据

curl -XPOST "localhost:9200/my_index/_doc/1" -H 'Content-Type: application/json' -d'
{
"text": "student1",
"name": "xiaoming"
}'

curl -XPOST "localhost:9200/my_index/_doc/2" -H 'Content-Type: application/json' -d'
{
"text": "student2",
"name": "xiaohong"
}'

curl -XPOST "localhost:9200/my_index/_doc/3" -H 'Content-Type: application/json' -d'
{
"text": "student3",
"name": "xiaoli"
}'

curl -XPOST "localhost:9200/my_index/_doc/4" -H 'Content-Type: application/json' -d'
{
"text": "student4",
"name": "xiaozhang"
}'

2.Opensearch2.9.0配置

plugins.security.disabled: true
discovery.type: single-node
network.host: 0.0.0.0

创建mapping

curl -XPUT "localhost:9200/my_index" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "properties": {
      "text": {
        "type": "text"
      },
      "name":{
        "type": "text"
      }
    }
  },
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}'
  1. Logstash7.10.2配置
input {
      elasticsearch {
        hosts => "{IP}:9200"
        index => "my*"
        docinfo => true
   }
}

filter {
  mutate {
    remove_field => ["@timestamp", "@version"]
  }
}

output {
    opensearch {
      hosts => ["{IP}:9200"]
      index => "%{[@metadata][_index]}"
      document_id => "%{[@metadata][_id]}"
  }
}

把IP替换成实际的IP

可以看到数据被正常迁移了

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 4,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": "my_index",
        "_id": "1",
        "_score": 1,
        "_source": {
          "name": "xiaoming",
          "text": "student1"
        }
      },
      {
        "_index": "my_index",
        "_id": "2",
        "_score": 1,
        "_source": {
          "name": "xiaohong",
          "text": "student2"
        }
      },
      {
        "_index": "my_index",
        "_id": "3",
        "_score": 1,
        "_source": {
          "name": "xiaoli",
          "text": "student3"
        }
      },
      {
        "_index": "my_index",
        "_id": "4",
        "_score": 1,
        "_source": {
          "name": "xiaozhang",
          "text": "student4"
        }
      }
    ]
  }
}
文章来自个人专栏
大数据与AI的修炼之路
3 文章 | 1 订阅
0条评论
0 / 1000
请输入你的评论
0
0