华为fusionInsight为用户提供海量数据的管理及分析功能,快速从结构化和非结构化的海量数据中挖掘您所需要的价值数据。开源组件结构复杂,安装、配置、管理过程费时费力,使用华为FusionInsight Manager将为您提供企业级的集群的统一管理平台,在工作中遇到使用华为集群的es由于过于安全,操作反而不便,为此记录下使用工具
1.使用账号密码登陆web界面下载认证凭据
2.1使用如下pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.example</groupId><artifactId>huawei_es_tools</artifactId><version>0.0.1-SNAPSHOT</version><name>huawei_es_tools</name><description>huawei_es_tools</description><properties><java.version>1.8</java.version></properties><dependencies><dependency><groupId>com.fasterxml.jackson.core</groupId><artifactId>jackson-core</artifactId><version>2.12.5</version></dependency><dependency><groupId>com.huawei</groupId><artifactId>elasticsearch-rest-client</artifactId><version>6.7.1</version></dependency><dependency><groupId>com.huawei</groupId><artifactId>elasticsearch</artifactId><version>6.7.1</version></dependency><dependency><groupId>com.huawei</groupId><artifactId>elasticsearch-rest-high-level-client</artifactId><version>6.7.1</version></dependency></dependencies><repositories><repository><id>huaweicloudsdk</id><url>https://mirrors.huaweicloud.com/repository/maven/huaweicloudsdk/</url><releases><enabled>true</enabled></releases><snapshots><enabled>true</enabled></snapshots></repository><repository><id>central</id><name>Mavn Centreal</name><url>https://repo1.maven.org/maven2/</url></repository></repositories><build><sourceDirectory>src/main/java</sourceDirectory><plugins><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-dependency-plugin</artifactId></plugin><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-jar-plugin</artifactId><version>3.2.0</version><configuration><excludes><exclude>log4j.properties</exclude><exclude>log4j2.xml</exclude></excludes></configuration></plugin><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><configuration><source>1.8</source><target>1.8</target></configuration></plugin></plugins></build>
</project>
3.在项目目录下建立conf文件夹并存放1步骤中的凭据文件
4.测试工具类
public class EsTest {/*** 配置文件路径位置*/private static final int CONFIG_PATH_ARGUMENT_INDEX = 0;/*** 获取HwRestClient** @param args 配置参数* @return HwRestClient*/public static HwRestClient getHwRestClient(String[] args) {HwRestClient hwRestClient;if (args == null|| args.length < 1|| args[CONFIG_PATH_ARGUMENT_INDEX] == null|| args[CONFIG_PATH_ARGUMENT_INDEX].isEmpty()) {hwRestClient = new HwRestClient();} else {String configPath = args[CONFIG_PATH_ARGUMENT_INDEX];File configFile = new File(configPath);if (configFile.exists()) {if (configFile.isDirectory()) {hwRestClient = new HwRestClient(configPath);} else {try {hwRestClient =new HwRestClient(configFile.getCanonicalPath().substring(0,configFile.getCanonicalPath().lastIndexOf(File.separator) + 1));} catch (IOException e) {hwRestClient = new HwRestClient();}}} else {hwRestClient = new HwRestClient();}}return hwRestClient;}/*** 查询指定索引下数据** @param highLevelClient* @param index*/public static void search(RestHighLevelClient highLevelClient, String index) {try {//A search source builder allowing to 创建一个搜索源SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();//SearchRequest按一个或多个索引查询,需要一个SearchSourceBuilder,搜索源提供了搜索选项SearchRequest searchRequest = new SearchRequest();//text类型不能用于索引或排序,必须转成keyword类型//String AggregationName = "application_aggregations";//脚本//String painlessScript = "((doc['S_IP.keyword'].value))";//TermsAggregationBuilder aggregation = AggregationBuilders// .terms(AggregationName)// .script(new Script(ScriptType.INLINE, "painless", painlessScript, new HashMap<>()))// //应返回桶的数量--全量返回// .size(Integer.MAX_VALUE)// //最少1条// .minDocCount(1)// .shardMinDocCount(0)// //返回文档计数错误// .showTermDocCountError(false);//添加bool过滤器,进行条件查询BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();//must --时间必须满足//存在关键字sourceOrDestinationTerm(S_IP)//boolQueryBuilder.must(QueryBuilders.existsQuery("S_IP"));//定义sourceBuilder,范围为0-9999,按时间排序,正序,再传入之前的查询条件,from 0 size 0 不查原始数据//sourceBuilder.sort("TIME.keyword", SortOrder.ASC).from(0).size(0).query(boolQueryBuilder).aggregation(aggregation);//定义查询的索引,定义搜索源,即sourceBuilder对象searchRequest.indices(index);searchRequest.source(sourceBuilder);//开始搜索,拿到结果SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);System.out.println("Search response is==" + searchResponse.toString());} catch (IOException e) {System.out.println("Search is failed, exception occurred." + e);}}public static void addData(RestHighLevelClient highLevelClient, String index, String id, String dataStr) {try {IndexRequest indexRequest = new IndexRequest(index).id(id);indexRequest.source(dataStr, XContentType.JSON);indexRequest.type("_doc");IndexResponse indexResponse = highLevelClient.index(indexRequest, RequestOptions.DEFAULT);System.out.println("addData response is " + indexResponse.toString());} catch (IOException e) {System.out.println("addData is failed,exception occurred." + e);}}public static void main(String[] args) {RestHighLevelClient highLevelClient = null;HwRestClient hwRestClient = getHwRestClient(args);try {highLevelClient = new RestHighLevelClient(hwRestClient.getRestClientBuilder());addData(highLevelClient, "sql_log_2023-11-07", "1", "{\"title\":\"其余信息\",\"key\":\"other\"}");search(highLevelClient, "sql_log_log*");} finally {try {if (highLevelClient != null) {highLevelClient.close();}} catch (IOException e) {System.out.println("Failed to close RestHighLevelClient." + e);}}}
}