一. 部署Elasticsearch
1. docker查询docker容器中的es
docker search elasticsearch
2. 安装(PS:查看自己的springBoot的版本号 对应的es版本安装)
docker pull elasticsearch:7.6.2
3. 查看已安装的docker镜像
docker images
4. 创建挂在目录
mkdir -p /data/elk/es/{config,data,logs}
5. 授权:docker中elasticsearch的用户UID是1000.
chown -R 1000:1000 /data/elk/es
6. 创建挂载配置文件
cd /data/elk/es/config
touch elasticsearch.yml
sudo vi elasticsearch.yml
#[elasticsearch.yml]
cluster.name: "geb-es"
network.host: 0.0.0.0
http.port: 9200
7. 运行elasticsearch
通过镜像,启动一个容器,并将9200和9300端口映射到本机(elasticsearch的默认端口是9200,我们把宿主环境9200端口映射到Docker容器中的4200端口)
docker run -it -d -p 4200:9200 -p 4300:9300 --name es -e ES_JAVA_OPTS="-Xms1g -Xmx1g" -e "discovery.type=single-node" --restart=always -v /date/data/elk/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /date/data/elk/es/data:/usr/share/elasticsearch/data -v /date/data/elk/es/logs:/usr/share/elasticsearch/logs elasticsearch:7.6.2
8. 验证是否安装成功
curl http://localhost:4200
如上 es安装完成。
9. PS: ES的索引类似于milvus中的集合Collection不可重复
ES可以分不同的索引进行查询 然后不同索引中存储json格式的document文档来存储
二. SpringBoot整合ES
1. Java Maven项目中pom.xml引入ES的SDK
还是看springBoot对应的elasticsearch的版本,自行查看~
<!-- ES 7.6.2 SDK --><dependency><groupId>org.elasticsearch.client</groupId><artifactId>elasticsearch-rest-high-level-client</artifactId><version>7.6.2</version></dependency>
2. application.yml文件,设置es相关配置
elasticsearch:ip: 10.100.111.11port: 4200
3. 新建ES配置类(实体类略~)
package com.geb.common.config;import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;@Configuration
public class ElasticSearchConfig {@Value("${elasticsearch.ip}")private String ip;@Value("${elasticsearch.port}")private Integer port;@Beanpublic RestHighLevelClient restHighLevelClient() {return new RestHighLevelClient(RestClient.builder(new HttpHost(ip, port, "http")));}}
4. Service业务接口处理类
package com.geb.service;import com.baomidou.mybatisplus.extension.service.IService;
import com.geb.domain.IndexText;import java.io.IOException;
import java.util.List;/*** ES公共操作接口 service层* @author jx*/
public interface EsService extends IService<IndexText> {/*** 判断索引库是否存在* @param index* @return*/boolean checkIndexExists(String index) throws IOException;/*** 删除索引* @param index* @return* @throws IOException*/boolean deleteIndex(String index) throws IOException;/*** 批量删除文本文档* @param index* @param idList* @throws IOException*/void deleteDocument(String index, List<Long> idList) throws IOException;/*** 创建Es索引并存入documents数据入文档* @param indexTextList* @param indexName* @throws IOException*/void saveData(List<IndexText> indexTextList, String indexName) throws IOException;/*** 根据keyword查询es相匹配的数据** @param keyword 查询分词器* @param pageNo 当前页* @param pageSize 每页条数* @param indexName 索引名称* @return List<IndexText>*/List<IndexText> search(String keyword, Integer pageNo, Integer pageSize, String indexName) throws Exception ;}
5. Service业务具体实现类
package com.geb.service.impl;import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.geb.common.utils.StringUtils;
import com.geb.domain.IndexText;
import com.geb.mapper.WdIndexTextMapper;
import com.geb.service.EsService;
import com.google.common.collect.Lists;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;/*** ES服务实现类* @author aaa*/
@Slf4j
@Service
@AllArgsConstructor
public class EsServiceImpl extends ServiceImpl<WdIndexTextMapper, IndexText> implements EsService {@Autowiredprivate RestHighLevelClient client;private static final String TEXT = "text";/**************************************************************************** 索引操作 - 类似于milvus中的集合cllection **********************************************************************/// 创建索引public boolean createIndex(String index) throws IOException {CreateIndexRequest request = new CreateIndexRequest(index);CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);System.out.println("索引创建状态: " + createIndexResponse.isAcknowledged());return createIndexResponse.isAcknowledged();}// 检查索引是否存在@Overridepublic boolean checkIndexExists(String index) throws IOException {GetIndexRequest request = new GetIndexRequest();request.indices(index);boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);System.out.println("索引存在: " + exists);return exists;}// 删除索引@Overridepublic boolean deleteIndex(String index) throws IOException {DeleteIndexRequest request = new DeleteIndexRequest(index);AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);System.out.println("索引删除状态: " + deleteIndexResponse.isAcknowledged());return deleteIndexResponse.isAcknowledged();}/**************************************************************************** documents操作 - 类似于milvus中的向量数据 **********************************************************************//*** 删除文档* @param index* @param idList* @throws IOException*/@Overridepublic void deleteDocument(String index, List<Long> idList) throws IOException {// 批量删除数据BulkRequest request = new BulkRequest();for (Long id : idList) {request.add(new DeleteRequest().index(index).id(String.valueOf(id)));}BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);}/*** 创建Es索引并存入documents数据入文档* @param indexTextList* @param indexName* @throws IOException*/@Overridepublic void saveData(List<IndexText> indexTextList, String indexName) throws IOException {// 判断该索引是否储存在boolean indexExit = checkIndexExists(indexName);// 不存在则直接创建索引if(!indexExit){createIndex(indexName);}// 存在-则批量存储在该索引中的数据文本document - 从数据库查询所有数据BulkRequest bulkRequest = new BulkRequest(indexName);for (IndexText indexText : indexTextList) {IndexRequest request = new IndexRequest();request.id(indexText.getId().toString());String jsonString = JSON.toJSONString(indexText);request.source(jsonString, XContentType.JSON);bulkRequest.add(request);}BulkResponse responses = client.bulk(bulkRequest, RequestOptions.DEFAULT);System.out.println(responses.status());}/*** 根据keyword查询es相匹配的数据** @param keyword 查询分词器* @param pageNo 当前页* @param pageSize 每页条数* @param indexName 索引名称* @return List<IndexText>*/@Overridepublic List<IndexText> search(String keyword, Integer pageNo, Integer pageSize, String indexName) throws Exception {List<Map<String, Object>> mapList = Lists.newArrayList(); // 获取到的List<Map<String, Object>>对象List<IndexText> resultList = Lists.newArrayList();SearchSourceBuilder builder = new SearchSourceBuilder();builder.from((pageNo - 1) * pageSize);builder.size(pageSize);if (StringUtils.isBlank(keyword)) { // 根据indexName全量查询该索引中的数据builder.query(QueryBuilders.matchAllQuery());} else { // 根据keyword分词查询该索引中匹配的数据builder.query(QueryBuilders.matchQuery(TEXT, keyword));}builder.timeout(new TimeValue(60L, TimeUnit.SECONDS));try {SearchResponse response = client.search(new SearchRequest(indexName).source(builder), RequestOptions.DEFAULT);SearchHits responseHits = response.getHits();SearchHit[] hits = responseHits.getHits();if (hits.length > 0) {Arrays.stream(hits).forEach(e -> {
// float source = e.getScore();
// String sourceAsString = e.getSourceAsString();mapList.add(e.getSourceAsMap());});}// 查询到的es数据Map -> ListJSONArray jsonArray = new JSONArray();jsonArray.addAll(mapList);resultList = jsonArray.toJavaList(IndexText.class);} catch (Exception e) {throw new Exception(e);}return resultList;}}
6. 测试结果查询(可自行编写单元测试/接口测试)
package com.geb.controller;import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.geb.common.core.controller.BaseController;
import com.geb.common.domain.R;
import com.geb.domain.IndexText;
import com.geb.domain.dto.VectorTextDataDto;
import com.geb.mapper.WdIndexTextMapper;
import com.geb.service.EsService;
import com.geb.service.IWdKnowledgeBaseService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;import java.io.IOException;
import java.util.List;@RestController
@RequestMapping("/vector")
@Slf4j
public class VectorController extends BaseController {@Autowiredprivate WdIndexTextMapper wdIndexTextMapper;@Autowiredprivate EsService esService;@PostMapping("/saveEsData")@ApiOperation(value = "将用户信息保存进es中")public R<String> saveEsData() throws IOException {List<IndexText> indexTextList = wdIndexTextMapper.selectList(new LambdaQueryWrapper<IndexText>().eq(IndexText::getFileId, 56));esService.saveData(indexTextList, "test-es");return R.ok();}@GetMapping("/searchData/{query}")@ApiOperation(value = "es中查询数据")public R<List<IndexText>> searchData(@PathVariable String query) throws Exception {esService.checkIndexExists("test-es");return R.ok(esService.search(null, 1, 20, "test-es"));}@PostMapping("/createIndex")@ApiOperation(value = "es创建索引")public R<String> createIndex() throws IOException {esService.checkIndexExists("test-es");esService.deleteIndex("test-es");
// List<Long> idList = new ArrayList<>();
// idList.add(355L);
// idList.add(362L);
// idList.add(361L);
// idList.add(360L);
// idList.add(359L);
// idList.add(358L);
// esService.deleteDocument("test-es", idList);return R.ok();}
}