From 14b5853e4c6fec2e4e86388e93ecc1c48c8bffc8 Mon Sep 17 00:00:00 2001 From: dunwu Date: Wed, 21 Feb 2024 22:32:55 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=20elasticsearch?= =?UTF-8?q?=20=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch/elasticsearch6/pom.xml | 50 ++- .../dunwu/javadb/elasticsearch/Demo.java | 11 +- .../elasticsearch/ElasticsearchFactory.java | 166 +++++++++ ...chUtil.java => ElasticsearchTemplate.java} | 330 ++++++++++-------- .../config/ElasticsearchConfig.java | 30 ++ .../config/EnableElasticsearch.java | 26 ++ .../elasticsearch/entity/BaseEsEntity.java | 27 ++ .../javadb/elasticsearch/entity/EsEntity.java | 16 - .../javadb/elasticsearch/entity/Page.java | 4 +- .../javadb/elasticsearch/entity/User.java | 10 +- .../elasticsearch/mapper/BaseEsMapper.java | 108 +++--- .../javadb/elasticsearch/mapper/EsMapper.java | 45 ++- .../elasticsearch/mapper/UserEsMapper.java | 8 +- .../javadb/elasticsearch/util/JsonUtil.java | 2 +- .../elasticsearch/BaseApplicationTests.java | 36 ++ .../ElasticsearchTemplateTest.java | 193 ++++++++++ .../javadb/elasticsearch/TestApplication.java | 20 ++ .../mapper/UserEsMapperTest.java | 191 ++++++++++ 18 files changed, 1012 insertions(+), 261 deletions(-) create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java rename codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/{util/ElasticsearchUtil.java => ElasticsearchTemplate.java} (53%) create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/EnableElasticsearch.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java delete mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/EsEntity.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/TestApplication.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java diff --git a/codes/javadb/elasticsearch/elasticsearch6/pom.xml b/codes/javadb/elasticsearch/elasticsearch6/pom.xml index ffd5de25..b9d8859b 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/pom.xml +++ b/codes/javadb/elasticsearch/elasticsearch6/pom.xml @@ -1,8 +1,14 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + o + org.springframework.boot + spring-boot-starter-parent + 2.7.7 + + io.github.dunwu javadb-elasticsearch6 1.0.0 @@ -17,37 +23,47 @@ + + org.springframework.boot + spring-boot-starter-aop + + + org.springframework.boot + spring-boot-starter-web + org.elasticsearch.client elasticsearch-rest-high-level-client - 6.4.3 org.projectlombok lombok - 1.18.22 cn.hutool hutool-all - 5.7.20 - - - com.fasterxml.jackson.core - jackson-databind - 2.15.2 + 5.8.8 - ch.qos.logback - logback-classic - 1.2.10 - - - org.apache.logging.log4j - log4j-to-slf4j - 2.17.1 + org.springframework.boot + spring-boot-starter-test + test + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.4.3 + + + org.elasticsearch + elasticsearch + 6.4.3 + + + diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java index 6e7523fb..3f97a580 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java @@ -2,8 +2,6 @@ import io.github.dunwu.javadb.elasticsearch.entity.User; import io.github.dunwu.javadb.elasticsearch.mapper.UserEsMapper; -import io.github.dunwu.javadb.elasticsearch.util.ElasticsearchUtil; -import org.elasticsearch.client.RestHighLevelClient; import java.io.IOException; import java.util.Arrays; @@ -11,12 +9,13 @@ public class Demo { - private static final String HOSTS = "127.0.0.1:9200"; - private static final RestHighLevelClient restHighLevelClient = ElasticsearchUtil.newRestHighLevelClient(HOSTS); + private static final String env = "test"; + private static final ElasticsearchTemplate elasticsearchTemplate + = ElasticsearchFactory.newElasticsearchTemplate(env); public static void main(String[] args) throws IOException, InterruptedException { - UserEsMapper mapper = new UserEsMapper(restHighLevelClient); + UserEsMapper mapper = new UserEsMapper(elasticsearchTemplate); System.out.println("索引是否存在：" + mapper.isIndexExists()); @@ -24,7 +23,7 @@ public static void main(String[] args) throws IOException, InterruptedException User tom = User.builder().id(2L).username("tom").age(20).build(); List users = Arrays.asList(jack, tom); - System.out.println("批量插入：" + mapper.batchInsert(users)); + System.out.println("批量插入：" + mapper.batchSave(users)); System.out.println("根据ID查询：" + mapper.getById("1").toString()); System.out.println("根据ID查询：" + mapper.pojoById("2").toString()); System.out.println("根据ID批量查询：" + mapper.pojoListByIds(Arrays.asList("1", "2")).toString()); diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java new file mode 100644 index 00000000..8582fdcf --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java @@ -0,0 +1,166 @@ +package io.github.dunwu.javadb.elasticsearch; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.StrUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Elasticsearch 客户端实例工厂 + * + * @author Zhang Peng + * @date 2024-02-07 + */ +@Slf4j +public class ElasticsearchFactory { + + public static int CONNECT_TIMEOUT_MILLIS = 1000; + + public static int SOCKET_TIMEOUT_MILLIS = 30000; + + public static int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500; + + public static int MAX_CONN_TOTAL = 30; + + public static int MAX_CONN_PER_ROUTE = 10; + + public static RestClient newRestClient() { + // 从配置中心读取环境变量 + String env = "test"; + return newRestClient(env); + } + + public static RestClient newRestClient(String env) { + String hosts = getDefaultEsAddress(env); + return newRestClient(toHttpHostList(hosts)); + } + + public static RestClient newRestClient(HttpHost[] httpHosts) { + RestClientBuilder builder = getRestClientBuilder(httpHosts); + if (builder == null) { + return null; + } + try { + return builder.build(); + } catch (Exception e) { + log.error("【ES】connect failed.", e); + return null; + } + } + + public static RestHighLevelClient newRestHighLevelClient() { + // 从配置中心读取环境变量 + String env = "test"; + return newRestHighLevelClient(env); + } + + public static RestHighLevelClient newRestHighLevelClient(String env) { + String hosts = getDefaultEsAddress(env); + return newRestHighLevelClient(toHttpHostList(hosts)); + } + + public static RestHighLevelClient newRestHighLevelClient(HttpHost[] httpHosts) { + RestClientBuilder builder = getRestClientBuilder(httpHosts); + if (builder == null) { + return null; + } + try { + return new RestHighLevelClient(builder); + } catch (Exception e) { + log.error("【ES】connect failed.", e); + return null; + } + } + + public static ElasticsearchTemplate newElasticsearchTemplate() { + // 从配置中心读取环境变量 + String env = "test"; + return newElasticsearchTemplate(env); + } + + public static ElasticsearchTemplate newElasticsearchTemplate(String env) { + String hosts = getDefaultEsAddress(env); + return newElasticsearchTemplate(toHttpHostList(hosts)); + } + + public static ElasticsearchTemplate newElasticsearchTemplate(HttpHost[] httpHosts) { + RestHighLevelClient client = newRestHighLevelClient(httpHosts); + if (client == null) { + return null; + } + return new ElasticsearchTemplate(client); + } + + public static ElasticsearchTemplate newElasticsearchTemplate(RestHighLevelClient client) { + if (client == null) { + return null; + } + return new ElasticsearchTemplate(client); + } + + public static RestClientBuilder getRestClientBuilder(HttpHost[] httpHosts) { + if (ArrayUtil.isEmpty(httpHosts)) { + log.error("【ES】connect failed. hosts are empty."); + return null; + } + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + restClientBuilder.setRequestConfigCallback(builder -> { + builder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS); + builder.setSocketTimeout(SOCKET_TIMEOUT_MILLIS); + builder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS); + return builder; + }); + restClientBuilder.setHttpClientConfigCallback(builder -> { + builder.setMaxConnTotal(MAX_CONN_TOTAL); + builder.setMaxConnPerRoute(MAX_CONN_PER_ROUTE); + return builder; + }); + return restClientBuilder; + } + + private static HttpHost[] toHttpHostList(String hosts) { + if (StrUtil.isBlank(hosts)) { + return null; + } + List strList = StrUtil.split(hosts, ","); + List list = strList.stream().map(str -> { + List params = StrUtil.split(str, ":"); + return new HttpHost(params.get(0), Integer.parseInt(params.get(1)), "http"); + }).collect(Collectors.toList()); + if (CollectionUtil.isEmpty(list)) { + return new HttpHost[0]; + } + return list.toArray(new HttpHost[0]); + } + + public static String getDefaultEsAddress() { + // 从配置中心读取环境变量 + String env = "test"; + return getDefaultEsAddress(env); + } + + private static String getDefaultEsAddress(String env) { + String defaultAddress; + switch (env) { + case "prd": + defaultAddress = "127.0.0.1:9200,127.0.0.2:9200,127.0.0.3:9200"; + break; + case "pre": + defaultAddress = "127.0.0.1:9200"; + break; + case "test": + default: + defaultAddress = "127.0.0.1:9200"; + break; + } + return defaultAddress; + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/ElasticsearchUtil.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java similarity index 53% rename from codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/ElasticsearchUtil.java rename to codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java index e3352082..cbe281a7 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/ElasticsearchUtil.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java @@ -1,13 +1,14 @@ -package io.github.dunwu.javadb.elasticsearch.util; +package io.github.dunwu.javadb.elasticsearch; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.io.IoUtil; +import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.ArrayUtil; import cn.hutool.core.util.StrUtil; -import cn.hutool.json.JSONUtil; -import io.github.dunwu.javadb.elasticsearch.entity.EsEntity; +import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; import io.github.dunwu.javadb.elasticsearch.entity.Page; +import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpHost; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BackoffPolicy; @@ -15,6 +16,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetRequest; @@ -23,28 +25,26 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Requests; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.stream.Collectors; @@ -57,37 +57,19 @@ * @date 2023-06-27 */ @Slf4j -public class ElasticsearchUtil { - - public static int CONNECT_TIMEOUT_MILLIS = 1000; - public static int SOCKET_TIMEOUT_MILLIS = 30000; - public static int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500; - public static int MAX_CONN_TOTAL = 30; - public static int MAX_CONN_PER_ROUTE = 10; - - public static RestClient newRestClient(String hosts) { - HttpHost[] httpHosts = toHttpHostList(hosts); - RestClientBuilder builder = builder(httpHosts); - try { - return builder.build(); - } catch (Exception e) { - log.error("【ES】connect failed.", e); - return null; - } +public class ElasticsearchTemplate implements Closeable { + + private final RestHighLevelClient client; + + public ElasticsearchTemplate(RestHighLevelClient client) { + this.client = client; } - public static RestHighLevelClient newRestHighLevelClient(String hosts) { - HttpHost[] httpHosts = toHttpHostList(hosts); - RestClientBuilder builder = builder(httpHosts); - try { - return new RestHighLevelClient(builder); - } catch (Exception e) { - log.error("【ES】connect failed.", e); - return null; - } + public RestHighLevelClient getClient() { + return client; } - public static BulkProcessor newAsyncBulkProcessor(RestHighLevelClient client) { + public BulkProcessor newAsyncBulkProcessor() { BulkProcessor.Listener listener = new BulkProcessor.Listener() { @Override public void beforeBulk(long executionId, BulkRequest request) { @@ -96,7 +78,7 @@ public void beforeBulk(long executionId, BulkRequest request) { @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { if (response.hasFailures()) { - log.error("Bulk [{}] executed with failures,response = {}", executionId, + log.error("【ES】Bulk [{}] executed with failures,response = {}", executionId, response.buildFailureMessage()); } } @@ -105,90 +87,87 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon public void afterBulk(long executionId, BulkRequest request, Throwable failure) { } }; + + int bulkTimeout = 30; + int bulkActions = 1000; + int bulkSize = 5; + int concurrentRequests = 2; + int flushInterval = 1000; + int retryInterval = 100; + int retryLimit = 3; BiConsumer> bulkConsumer = (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); + BackoffPolicy backoffPolicy = + BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(retryInterval), retryLimit); BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener) // 1000条数据请求执行一次bulk - .setBulkActions(1000) + .setBulkActions(bulkActions) // 5mb的数据刷新一次bulk - .setBulkSize(new ByteSizeValue(5L, ByteSizeUnit.MB)) + .setBulkSize(new ByteSizeValue(bulkSize, ByteSizeUnit.MB)) // 并发请求数量, 0不并发, 1并发允许执行 - .setConcurrentRequests(2) - // 固定1s必须刷新一次 - .setFlushInterval(TimeValue.timeValueMillis(1000L)) - // 重试3次，间隔100ms - .setBackoffPolicy( - BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(200L), - 3)).build(); + .setConcurrentRequests(concurrentRequests) + // 刷新间隔时间 + .setFlushInterval(TimeValue.timeValueMillis(flushInterval)) + // 重试次数、间隔时间 + .setBackoffPolicy(backoffPolicy).build(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { bulkProcessor.flush(); - bulkProcessor.awaitClose(30, TimeUnit.SECONDS); + bulkProcessor.awaitClose(bulkTimeout, TimeUnit.SECONDS); } catch (Exception e) { - log.error("Failed to close bulkProcessor", e); + log.error("【ES】Failed to close bulkProcessor", e); } - log.info("bulkProcessor closed!"); + log.info("【ES】bulkProcessor closed!"); })); return bulkProcessor; } - public static HttpHost[] toHttpHostList(String hosts) { - if (StrUtil.isBlank(hosts)) { + public T save(String index, String type, T entity) throws IOException { + + if (entity == null) { + log.warn("【ES】save 实体为空！"); return null; } - List strList = StrUtil.split(hosts, ","); - List list = strList.stream().map(str -> { - List params = StrUtil.split(str, ":"); - return new HttpHost(params.get(0), Integer.parseInt(params.get(1)), "http"); - }).collect(Collectors.toList()); - if (CollectionUtil.isEmpty(list)) { - return new HttpHost[0]; - } - return list.toArray(new HttpHost[0]); - } - - public static RestClientBuilder builder(HttpHost[] httpHosts) { - RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); - restClientBuilder.setRequestConfigCallback(builder -> { - builder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS); - builder.setSocketTimeout(SOCKET_TIMEOUT_MILLIS); - builder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS); - return builder; - }); - restClientBuilder.setHttpClientConfigCallback(builder -> { - builder.setMaxConnTotal(MAX_CONN_TOTAL); - builder.setMaxConnPerRoute(MAX_CONN_PER_ROUTE); - return builder; - }); - return restClientBuilder; - } - public static String insert(RestHighLevelClient client, String index, String type, T entity) - throws IOException { Map map = toMap(entity); + if (MapUtil.isEmpty(map)) { + log.warn("【ES】save 实体数据为空！"); + return null; + } + IndexRequest request = new IndexRequest(index, type).source(map); if (entity.getDocId() != null) { request.id(entity.getDocId()); } - IndexResponse response = client.index(request, RequestOptions.DEFAULT); - if (response != null && response.getResult() == DocWriteResponse.Result.CREATED) { - return response.getId(); + if (response == null) { + log.warn("【ES】save 响应结果为空！"); + return null; + } + + if (response.getResult() == DocWriteResponse.Result.CREATED + || response.getResult() == DocWriteResponse.Result.UPDATED) { + return entity; } else { + log.warn("【ES】save 响应结果无效！result: {}", response.getResult()); return null; } } - public static boolean batchInsert(RestHighLevelClient client, String index, String type, - Collection list) throws IOException { + public boolean batchSave(String index, String type, Collection list) + throws IOException { if (CollectionUtil.isEmpty(list)) { return true; } BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (T entity : list) { - Map map = ElasticsearchUtil.toMap(entity); + Map map = toMap(entity); + if (MapUtil.isEmpty(map)) { + continue; + } IndexRequest request = new IndexRequest(index, type).source(map); if (entity.getDocId() != null) { request.id(entity.getDocId()); @@ -200,16 +179,20 @@ public static boolean batchInsert(RestHighLevelClient clien return response != null && !response.hasFailures(); } - public static void asyncBatchInsert(RestHighLevelClient client, String index, String type, - Collection list, ActionListener listener) { + public void asyncBatchSave(String index, String type, Collection list, + ActionListener listener) { if (CollectionUtil.isEmpty(list)) { return; } BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (T entity : list) { - Map map = ElasticsearchUtil.toMap(entity); + Map map = toMap(entity); + if (MapUtil.isEmpty(map)) { + continue; + } IndexRequest request = new IndexRequest(index, type).source(map); if (entity.getDocId() != null) { request.id(entity.getDocId()); @@ -220,84 +203,107 @@ public static void asyncBatchInsert(RestHighLevelClient cli client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); } - public static boolean updateById(RestHighLevelClient client, String index, String type, - T entity) throws IOException { + public T updateById(String index, String type, T entity) throws IOException { - if (entity == null || entity.getDocId() == null) { - return false; + if (entity == null) { + log.warn("【ES】updateById 实体为空！"); + return null; + } + + if (entity.getDocId() == null) { + log.warn("【ES】updateById docId 为空！"); + return null; } Map map = toMap(entity); + if (MapUtil.isEmpty(map)) { + log.warn("【ES】updateById 实体数据为空！"); + return null; + } + UpdateRequest request = new UpdateRequest(index, type, entity.getDocId()).doc(map); UpdateResponse response = client.update(request, RequestOptions.DEFAULT); - return response != null && response.getResult() == DocWriteResponse.Result.UPDATED; + if (response == null) { + log.warn("【ES】updateById 响应结果为空！"); + return null; + } + + if (response.getResult() == DocWriteResponse.Result.UPDATED) { + return entity; + } else { + log.warn("【ES】updateById 响应结果无效！result: {}", response.getResult()); + return null; + } } - public static boolean batchUpdateById(RestHighLevelClient client, String index, String type, - Collection list) throws IOException { + public boolean batchUpdateById(String index, String type, Collection list) + throws IOException { if (CollectionUtil.isEmpty(list)) { return true; } - BulkRequest bulkRequest = new BulkRequest(); - for (T entity : list) { - if (entity == null || entity.getDocId() == null) { - continue; - } - Map map = ElasticsearchUtil.toMap(entity); - UpdateRequest request = new UpdateRequest(index, type, entity.getDocId()).doc(map); - bulkRequest.add(request); - } - + BulkRequest bulkRequest = toUpdateBulkRequest(index, type, list); BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); return response != null && !response.hasFailures(); } - public static void asyncBatchUpdateById(RestHighLevelClient client, String index, - String type, Collection list, ActionListener listener) { + public void asyncBatchUpdateById(String index, String type, Collection list, + ActionListener listener) { if (CollectionUtil.isEmpty(list)) { return; } + BulkRequest bulkRequest = toUpdateBulkRequest(index, type, list); + client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); + } + + private BulkRequest toUpdateBulkRequest(String index, String type, Collection list) { BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (T entity : list) { if (entity == null || entity.getDocId() == null) { continue; } - Map map = ElasticsearchUtil.toMap(entity); + Map map = toMap(entity); + if (MapUtil.isEmpty(map)) { + continue; + } UpdateRequest request = new UpdateRequest(index, type, entity.getDocId()).doc(map); bulkRequest.add(request); } - - client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); + return bulkRequest; } - public static boolean deleteById(RestHighLevelClient client, String index, String type, String id) - throws IOException { - return batchDeleteById(client, index, type, Collections.singleton(id)); + public boolean deleteById(String index, String type, String id) throws IOException { + return batchDeleteById(index, type, Collections.singleton(id)); } - public static boolean batchDeleteById(RestHighLevelClient client, String index, String type, Collection ids) - throws IOException { + public boolean batchDeleteById(String index, String type, Collection ids) throws IOException { if (CollectionUtil.isEmpty(ids)) { return true; } BulkRequest bulkRequest = new BulkRequest(); - ids.forEach(id -> { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + ids.stream().filter(Objects::nonNull).forEach(id -> { DeleteRequest deleteRequest = new DeleteRequest(index, type, id); bulkRequest.add(deleteRequest); }); BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); - return response != null && !response.hasFailures(); + if (response == null) { + log.warn("【ES】batchDeleteById 响应结果为空！"); + return false; + } + + return !response.hasFailures(); } - public static void asyncBatchDeleteById(RestHighLevelClient client, String index, String type, - Collection ids, ActionListener listener) { + public void asyncBatchDeleteById(String index, String type, Collection ids, + ActionListener listener) { if (CollectionUtil.isEmpty(ids)) { return; @@ -312,31 +318,32 @@ public static void asyncBatchDeleteById(RestHighLevelClient client, String index client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); } - public static SearchResponse getById(RestHighLevelClient client, String index, String type, String id) - throws IOException { - SearchRequest searchRequest = Requests.searchRequest(index).types(type); - QueryBuilder queryBuilder = QueryBuilders.idsQuery().addIds(id); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(queryBuilder); - searchRequest.source(sourceBuilder); - return client.search(searchRequest, RequestOptions.DEFAULT); + public GetResponse getById(String index, String type, String id) throws IOException { + return getById(index, type, id, null); } - public static T pojoById(RestHighLevelClient client, String index, String type, String id, Class clazz) - throws IOException { - SearchResponse response = getById(client, index, type, id); - if (response == null) { - return null; + public GetResponse getById(String index, String type, String id, Long version) throws IOException { + GetRequest getRequest = new GetRequest(index, type, id); + if (version != null) { + getRequest.version(version); } - List list = ElasticsearchUtil.toPojoList(response, clazz); - if (CollectionUtil.isEmpty(list)) { + return client.get(getRequest, RequestOptions.DEFAULT); + } + + public T pojoById(String index, String type, String id, Class clazz) throws IOException { + return pojoById(index, type, id, null, clazz); + } + + public T pojoById(String index, String type, String id, Long version, Class clazz) throws IOException { + GetResponse response = getById(index, type, id, version); + if (response == null) { return null; } - return list.get(0); + return toPojo(response, clazz); } - public static List pojoListByIds(RestHighLevelClient client, String index, String type, - Collection ids, Class clazz) throws IOException { + public List pojoListByIds(String index, String type, Collection ids, Class clazz) + throws IOException { if (CollectionUtil.isEmpty(ids)) { return null; @@ -359,7 +366,7 @@ public static List pojoListByIds(RestHighLevelClient client, String index if (itemResponse.isFailed()) { log.error("通过id获取文档失败", itemResponse.getFailure().getFailure()); } else { - T entity = ElasticsearchUtil.toPojo(itemResponse.getResponse(), clazz); + T entity = toPojo(itemResponse.getResponse(), clazz); if (entity != null) { list.add(entity); } @@ -368,36 +375,51 @@ public static List pojoListByIds(RestHighLevelClient client, String index return list; } - public static Page pojoPage(RestHighLevelClient client, String index, String type, - SearchSourceBuilder builder, Class clazz) throws IOException { - SearchResponse response = query(client, index, type, builder); + public Page pojoPage(String index, String type, SearchSourceBuilder builder, Class clazz) + throws IOException { + SearchResponse response = query(index, type, builder); if (response == null || response.status() != RestStatus.OK) { return null; } List content = toPojoList(response, clazz); SearchHits searchHits = response.getHits(); - return new Page<>(searchHits.getTotalHits(), builder.from(), builder.size(), content); + int offset = builder.from(); + int size = builder.size(); + int page = offset / size + (offset % size == 0 ? 0 : 1) + 1; + return new Page<>(page, size, searchHits.getTotalHits(), content); } - public static SearchResponse query(RestHighLevelClient client, String index, String type, - SearchSourceBuilder builder) throws IOException { + public long count(String index, String type, SearchSourceBuilder builder) throws IOException { + SearchResponse response = query(index, type, builder); + if (response == null || response.status() != RestStatus.OK) { + return -1L; + } + SearchHits searchHits = response.getHits(); + return searchHits.getTotalHits(); + } + + public SearchResponse query(String index, String type, SearchSourceBuilder builder) throws IOException { SearchRequest request = new SearchRequest(index).types(type); request.source(builder); return client.search(request, RequestOptions.DEFAULT); } - public static T toPojo(GetResponse response, Class clazz) { + public SearchResponse query(SearchRequest request) throws IOException { + return client.search(request, RequestOptions.DEFAULT); + } + + public T toPojo(GetResponse response, Class clazz) { if (null == response) { return null; } else if (StrUtil.isBlank(response.getSourceAsString())) { return null; } else { - return JSONUtil.toBean(response.getSourceAsString(), clazz); + return JsonUtil.toBean(response.getSourceAsString(), clazz); } } - public static List toPojoList(SearchResponse response, Class clazz) { + public List toPojoList(SearchResponse response, Class clazz) { if (response == null || response.status() != RestStatus.OK) { return new ArrayList<>(); @@ -408,12 +430,20 @@ public static List toPojoList(SearchResponse response, Class clazz) { } return Stream.of(response.getHits().getHits()) - .map(hit -> JSONUtil.toBean(hit.getSourceAsString(), clazz)) + .map(hit -> JsonUtil.toBean(hit.getSourceAsString(), clazz)) .collect(Collectors.toList()); } - public static Map toMap(T entity) { - return JsonUtil.toMap(JsonUtil.toJson(entity)); + public Map toMap(T entity) { + return JsonUtil.toMap(JsonUtil.toString(entity)); + } + + @Override + public synchronized void close() { + if (null == client) { + return; + } + IoUtil.close(client); } } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java new file mode 100644 index 00000000..bdd2f387 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java @@ -0,0 +1,30 @@ +package io.github.dunwu.javadb.elasticsearch.config; + +import io.github.dunwu.javadb.elasticsearch.ElasticsearchFactory; +import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * ES 配置 + * + * @author Zhang Peng + * @date 2024-02-07 + */ +@Configuration +@ComponentScan(value = "io.github.dunwu.javadb.elasticsearch.mapper") +public class ElasticsearchConfig { + + @Bean("restHighLevelClient") + public RestHighLevelClient restHighLevelClient() { + return ElasticsearchFactory.newRestHighLevelClient(); + } + + @Bean("elasticsearchTemplate") + public ElasticsearchTemplate elasticsearchTemplate() { + return ElasticsearchFactory.newElasticsearchTemplate(); + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/EnableElasticsearch.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/EnableElasticsearch.java new file mode 100644 index 00000000..c2c24479 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/EnableElasticsearch.java @@ -0,0 +1,26 @@ +package io.github.dunwu.javadb.elasticsearch.config; + +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * 启动 Elasticsearch 配置注解 + * + * @author Zhang Peng + * @date 2023-06-30 + */ +@Target({ ElementType.TYPE }) +@Retention(RetentionPolicy.RUNTIME) +@EnableAspectJAutoProxy( + proxyTargetClass = false +) +@Import({ ElasticsearchConfig.class }) +@Documented +public @interface EnableElasticsearch { +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java new file mode 100644 index 00000000..a7759763 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java @@ -0,0 +1,27 @@ +package io.github.dunwu.javadb.elasticsearch.entity; + +import lombok.Data; +import lombok.ToString; + +import java.io.Serializable; + +/** + * ES 实体接口 + * + * @author Zhang Peng + * @since 2023-06-28 + */ +@Data +@ToString +public abstract class BaseEsEntity implements Serializable { + + /** + * 获取版本 + */ + protected Long version; + + protected Float hitScore; + + public abstract String getDocId(); + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/EsEntity.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/EsEntity.java deleted file mode 100644 index df6d973f..00000000 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/EsEntity.java +++ /dev/null @@ -1,16 +0,0 @@ -package io.github.dunwu.javadb.elasticsearch.entity; - -/** - * ES 实体接口 - * - * @author Zhang Peng - * @since 2023-06-28 - */ -public interface EsEntity { - - /** - * 获取 ES 主键 - */ - String getDocId(); - -} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java index 76f178fa..8baef8b5 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java @@ -20,13 +20,13 @@ public class Page { private int size; private List content = new ArrayList<>(); - public Page(long total, int page, int size) { + public Page(int page, int size, long total) { this.total = total; this.page = page; this.size = size; } - public Page(long total, int page, int size, Collection list) { + public Page(int page, int size, long total, Collection list) { this.total = total; this.page = page; this.size = size; diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java index 58428a14..1594772c 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java @@ -1,7 +1,10 @@ package io.github.dunwu.javadb.elasticsearch.entity; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; /** * 用户实体 @@ -11,7 +14,10 @@ */ @Data @Builder -public class User implements EsEntity { +@EqualsAndHashCode(callSuper = true) +@AllArgsConstructor +@NoArgsConstructor +public class User extends BaseEsEntity { private Long id; private String username; @@ -21,7 +27,7 @@ public class User implements EsEntity { @Override public String getDocId() { - return null; + return String.valueOf(id); } } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java index 4033b4e8..cd7e53a9 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java @@ -1,14 +1,14 @@ package io.github.dunwu.javadb.elasticsearch.mapper; -import cn.hutool.core.lang.Assert; -import io.github.dunwu.javadb.elasticsearch.entity.EsEntity; +import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; +import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; import io.github.dunwu.javadb.elasticsearch.entity.Page; -import io.github.dunwu.javadb.elasticsearch.util.ElasticsearchUtil; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.IndicesClient; import org.elasticsearch.client.RequestOptions; @@ -26,26 +26,28 @@ * @date 2023-06-27 */ @Slf4j -public abstract class BaseEsMapper implements EsMapper { +public abstract class BaseEsMapper implements EsMapper { private BulkProcessor bulkProcessor; - protected final RestHighLevelClient restHighLevelClient; + protected final ElasticsearchTemplate elasticsearchTemplate; - public BaseEsMapper(RestHighLevelClient restHighLevelClient) { - this.restHighLevelClient = restHighLevelClient; + public BaseEsMapper(ElasticsearchTemplate elasticsearchTemplate) { + this.elasticsearchTemplate = elasticsearchTemplate; } @Override - public RestHighLevelClient getClient() throws IOException { - Assert.notNull(restHighLevelClient, () -> new IOException("【ES】not connected.")); - return restHighLevelClient; + public RestHighLevelClient getClient() { + if (elasticsearchTemplate == null) { + return null; + } + return elasticsearchTemplate.getClient(); } @Override - public synchronized BulkProcessor getBulkProcessor() throws IOException { + public synchronized BulkProcessor getBulkProcessor() { if (bulkProcessor == null) { - bulkProcessor = ElasticsearchUtil.newAsyncBulkProcessor(getClient()); + bulkProcessor = elasticsearchTemplate.newAsyncBulkProcessor(); } return bulkProcessor; } @@ -59,37 +61,57 @@ public boolean isIndexExists() throws IOException { } @Override - public SearchResponse getById(String id) throws IOException { - return ElasticsearchUtil.getById(getClient(), getIndex(), getType(), id); + public GetResponse getById(String id) throws IOException { + return getById(id, null); + } + + @Override + public GetResponse getById(String id, Long version) throws IOException { + return elasticsearchTemplate.getById(getIndex(), getType(), id, version); } @Override public T pojoById(String id) throws IOException { - return ElasticsearchUtil.pojoById(getClient(), getIndex(), getType(), id, getEntityClass()); + return pojoById(id, null); + } + + @Override + public T pojoById(String id, Long version) throws IOException { + return elasticsearchTemplate.pojoById(getIndex(), getType(), id, version, getEntityClass()); } @Override public List pojoListByIds(Collection ids) throws IOException { - return ElasticsearchUtil.pojoListByIds(getClient(), getIndex(), getType(), ids, getEntityClass()); + return elasticsearchTemplate.pojoListByIds(getIndex(), getType(), ids, getEntityClass()); } @Override public Page pojoPage(SearchSourceBuilder builder) throws IOException { - return ElasticsearchUtil.pojoPage(getClient(), getIndex(), getType(), builder, getEntityClass()); + return elasticsearchTemplate.pojoPage(getIndex(), getType(), builder, getEntityClass()); } @Override - public String insert(T entity) throws IOException { - return ElasticsearchUtil.insert(getClient(), getIndex(), getType(), entity); + public long count(SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.count(getIndex(), getType(), builder); } @Override - public boolean batchInsert(Collection list) throws IOException { - return ElasticsearchUtil.batchInsert(getClient(), getIndex(), getType(), list); + public SearchResponse query(SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.query(getIndex(), getType(), builder); } @Override - public void asyncBatchInsert(Collection list) throws IOException { + public T save(T entity) throws IOException { + return elasticsearchTemplate.save(getIndex(), getType(), entity); + } + + @Override + public boolean batchSave(Collection list) throws IOException { + return elasticsearchTemplate.batchSave(getIndex(), getType(), list); + } + + @Override + public void asyncBatchSave(Collection list) throws IOException { ActionListener listener = new ActionListener() { @Override public void onResponse(BulkResponse response) { @@ -105,57 +127,37 @@ public void onFailure(Exception e) { log.error("【ES】异步批量插入异常！", e); } }; - asyncBatchInsert(list, listener); + asyncBatchSave(list, listener); } @Override - public void asyncBatchInsert(Collection list, ActionListener listener) throws IOException { - ElasticsearchUtil.asyncBatchInsert(getClient(), getIndex(), getType(), list, listener); + public void asyncBatchSave(Collection list, ActionListener listener) { + elasticsearchTemplate.asyncBatchSave(getIndex(), getType(), list, listener); } @Override - public boolean updateById(T entity) throws IOException { - return ElasticsearchUtil.updateById(getClient(), getIndex(), getType(), entity); + public T updateById(T entity) throws IOException { + return elasticsearchTemplate.updateById(getIndex(), getType(), entity); } @Override public boolean batchUpdateById(Collection list) throws IOException { - return ElasticsearchUtil.batchUpdateById(getClient(), getIndex(), getType(), list); - } - - @Override - public void asyncBatchUpdateById(Collection list) throws IOException { - ActionListener listener = new ActionListener() { - @Override - public void onResponse(BulkResponse response) { - if (response != null && !response.hasFailures()) { - log.info("【ES】异步批量更新成功！"); - } else { - log.warn("【ES】异步批量更新失败！"); - } - } - - @Override - public void onFailure(Exception e) { - log.error("【ES】异步批量更新异常！", e); - } - }; - asyncBatchUpdateById(list, listener); + return elasticsearchTemplate.batchUpdateById(getIndex(), getType(), list); } @Override - public void asyncBatchUpdateById(Collection list, ActionListener listener) throws IOException { - ElasticsearchUtil.asyncBatchUpdateById(getClient(), getIndex(), getType(), list, listener); + public void asyncBatchUpdateById(Collection list, ActionListener listener) { + elasticsearchTemplate.asyncBatchUpdateById(getIndex(), getType(), list, listener); } @Override public boolean deleteById(String id) throws IOException { - return ElasticsearchUtil.deleteById(getClient(), getIndex(), getType(), id); + return elasticsearchTemplate.deleteById(getIndex(), getType(), id); } @Override public boolean batchDeleteById(Collection ids) throws IOException { - return ElasticsearchUtil.batchDeleteById(getClient(), getIndex(), getType(), ids); + return elasticsearchTemplate.batchDeleteById(getIndex(), getType(), ids); } @Override @@ -180,7 +182,7 @@ public void onFailure(Exception e) { @Override public void asyncBatchDeleteById(Collection ids, ActionListener listener) throws IOException { - ElasticsearchUtil.asyncBatchDeleteById(getClient(), getIndex(), getType(), ids, listener); + elasticsearchTemplate.asyncBatchDeleteById(getIndex(), getType(), ids, listener); } } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java index fa01d7c4..53eb9b2c 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java @@ -1,17 +1,21 @@ package io.github.dunwu.javadb.elasticsearch.mapper; -import io.github.dunwu.javadb.elasticsearch.entity.EsEntity; +import cn.hutool.core.collection.CollectionUtil; +import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; import io.github.dunwu.javadb.elasticsearch.entity.Page; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * ES Mapper @@ -19,7 +23,7 @@ * @author Zhang Peng * @date 2023-06-27 */ -public interface EsMapper { +public interface EsMapper { /** * 获取索引名 @@ -42,29 +46,48 @@ public interface EsMapper { boolean isIndexExists() throws IOException; - SearchResponse getById(String id) throws IOException; + GetResponse getById(String id) throws IOException; + + GetResponse getById(String id, Long version) throws IOException; T pojoById(String id) throws IOException; + T pojoById(String id, Long version) throws IOException; + List pojoListByIds(Collection ids) throws IOException; + default Map pojoMapByIds(Collection ids) throws IOException { + List list = pojoListByIds(ids); + if (CollectionUtil.isEmpty(list)) { + return new HashMap<>(0); + } + + Map map = new HashMap<>(list.size()); + for (T entity : list) { + map.put(entity.getDocId(), entity); + } + return map; + } + Page pojoPage(SearchSourceBuilder builder) throws IOException; - String insert(T entity) throws IOException; + long count(SearchSourceBuilder builder) throws IOException; - boolean batchInsert(Collection list) throws IOException; + SearchResponse query(SearchSourceBuilder builder) throws IOException; - void asyncBatchInsert(Collection list) throws IOException; + T save(T entity) throws IOException; - void asyncBatchInsert(Collection list, ActionListener listener) throws IOException; + boolean batchSave(Collection list) throws IOException; - boolean updateById(T entity) throws IOException; + void asyncBatchSave(Collection list) throws IOException; - boolean batchUpdateById(Collection list) throws IOException; + void asyncBatchSave(Collection list, ActionListener listener) throws IOException; + + T updateById(T entity) throws IOException; - void asyncBatchUpdateById(Collection list) throws IOException; + boolean batchUpdateById(Collection list) throws IOException; - void asyncBatchUpdateById(Collection list, ActionListener listener) throws IOException; + void asyncBatchUpdateById(Collection list, ActionListener listener); boolean deleteById(String id) throws IOException; diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java index c60af8aa..970b1b4d 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java @@ -1,7 +1,8 @@ package io.github.dunwu.javadb.elasticsearch.mapper; +import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; import io.github.dunwu.javadb.elasticsearch.entity.User; -import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.stereotype.Component; /** * User ES Mapper @@ -9,10 +10,11 @@ * @author Zhang Peng * @date 2023-06-27 */ +@Component public class UserEsMapper extends BaseEsMapper { - public UserEsMapper(RestHighLevelClient restHighLevelClient) { - super(restHighLevelClient); + public UserEsMapper(ElasticsearchTemplate elasticsearchTemplate) { + super(elasticsearchTemplate); } @Override diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/JsonUtil.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/JsonUtil.java index 0ed7ddb5..dabe0df5 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/JsonUtil.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/util/JsonUtil.java @@ -81,7 +81,7 @@ public static T toBean(String json, TypeReference typeReference) { return null; } - public static String toJson(T obj) { + public static String toString(T obj) { if (obj == null) { return null; } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java new file mode 100644 index 00000000..8d43d1b8 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java @@ -0,0 +1,36 @@ +package io.github.dunwu.javadb.elasticsearch; + +import io.github.dunwu.javadb.elasticsearch.config.EnableElasticsearch; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +@EnableElasticsearch +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public abstract class BaseApplicationTests { + + // ---------------------------------------------------------------------------- 测试常量数据 + + // ---------------------------------------------------------------------------- + protected MockMvc mockMvc; + + @Autowired + private WebApplicationContext context; + + @BeforeEach + public void setUp() { + mockMvc = MockMvcBuilders.webAppContextSetup(context).build(); //构造MockMvc + } + + @BeforeAll + public static void setEnvironmentInDev() { + } + +} \ No newline at end of file diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java new file mode 100644 index 00000000..bbddf70d --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java @@ -0,0 +1,193 @@ +package io.github.dunwu.javadb.elasticsearch; + +import io.github.dunwu.javadb.elasticsearch.entity.Page; +import io.github.dunwu.javadb.elasticsearch.entity.User; +import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; +import org.assertj.core.api.Assertions; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * ElasticsearchTemplate 测试 + * + * @author Zhang Peng + * @date 2023-11-13 + */ +public class ElasticsearchTemplateTest { + + public static final String INDEX = "user"; + public static final String TYPE = "_doc"; + public static final String TEST_ID_01 = "1"; + public static final String TEST_ID_02 = "2"; + + private static final ElasticsearchTemplate TEMPLATE; + + static { + TEMPLATE = ElasticsearchFactory.newElasticsearchTemplate(); + } + + @Test + @DisplayName("根据ID精确查询") + public void getById() throws IOException { + GetResponse response = TEMPLATE.getById(INDEX, TYPE, TEST_ID_01); + System.out.println("记录：" + JsonUtil.toString(response.getSourceAsMap())); + } + + @Test + @DisplayName("根据ID精确查询POJO") + public void pojoById() throws IOException { + User entity = TEMPLATE.pojoById(INDEX, TYPE, TEST_ID_01, User.class); + System.out.println("记录：" + JsonUtil.toString(entity)); + } + + @Test + @DisplayName("根据ID精确批量查询POJO") + public void pojoListByIds() throws IOException { + List ids = Arrays.asList(TEST_ID_01, TEST_ID_02); + List list = TEMPLATE.pojoListByIds(INDEX, TYPE, ids, User.class); + Assertions.assertThat(list).isNotEmpty(); + Assertions.assertThat(list.size()).isEqualTo(2); + for (User entity : list) { + System.out.println("记录：" + JsonUtil.toString(entity)); + } + } + + @Test + @DisplayName("分页查询") + public void pojoPage() throws IOException { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("theme", 3)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); + + Page page = TEMPLATE.pojoPage(INDEX, TYPE, searchSourceBuilder, User.class); + Assertions.assertThat(page).isNotNull(); + Assertions.assertThat(page.getContent()).isNotEmpty(); + for (User entity : page.getContent()) { + System.out.println("记录：" + JsonUtil.toString(entity)); + } + } + + @Test + @DisplayName("条件数量查询") + public void count() throws IOException { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("theme", 3)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); + long total = TEMPLATE.count(INDEX, TYPE, searchSourceBuilder); + Assertions.assertThat(total).isNotZero(); + System.out.println("符合条件的总记录数：" + total); + } + + @Test + @DisplayName("条件查询") + public void query() throws IOException { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("theme", 3)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); + SearchResponse response = TEMPLATE.query(INDEX, TYPE, searchSourceBuilder); + Assertions.assertThat(response).isNotNull(); + Assertions.assertThat(response.getHits()).isNotNull(); + for (SearchHit hit : response.getHits().getHits()) { + System.out.println("记录：" + hit.getSourceAsString()); + Map map = hit.getSourceAsMap(); + Assertions.assertThat(map).isNotNull(); + Assertions.assertThat(map.get("theme")).isEqualTo(3); + } + } + + @Nested + @DisplayName("写操作测试") + public class WriteTest { + + String json1 = + "{\"id\":1,\"username\":\"user1\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user1@xxx.com\"}"; + String json2 = + "{\"id\":2,\"username\":\"user2\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user2@xxx.com\"}"; + + @Test + @DisplayName("插入、更新") + public void saveAndUpdate() throws IOException, InterruptedException { + + User origin = JsonUtil.toBean(json1, User.class); + if (origin == null) { + System.err.println("反序列化失败！"); + return; + } + + TEMPLATE.save(INDEX, TYPE, origin); + TimeUnit.SECONDS.sleep(1); + User expectEntity = TEMPLATE.pojoById(INDEX, TYPE, origin.getDocId(), User.class); + Assertions.assertThat(expectEntity).isNotNull(); + + expectEntity.setAge(20); + TEMPLATE.updateById(INDEX, TYPE, expectEntity); + TimeUnit.SECONDS.sleep(18); + User expectEntity2 = + TEMPLATE.pojoById(INDEX, TYPE, origin.getDocId(), User.class); + Assertions.assertThat(expectEntity2).isNotNull(); + Assertions.assertThat(expectEntity2.getAge()).isEqualTo(20); + } + + @Test + @DisplayName("批量插入、更新") + public void batchSaveAndUpdate() throws IOException, InterruptedException { + + User origin1 = JsonUtil.toBean(json1, User.class); + if (origin1 == null) { + System.err.println("反序列化失败！"); + return; + } + + User origin2 = JsonUtil.toBean(json2, User.class); + if (origin2 == null) { + System.err.println("反序列化失败！"); + return; + } + + List list = Arrays.asList(origin1, origin2); + List ids = list.stream().map(User::getDocId).collect(Collectors.toList()); + + TEMPLATE.batchSave(INDEX, TYPE, list); + List newList = TEMPLATE.pojoListByIds(INDEX, TYPE, ids, User.class); + Assertions.assertThat(newList).isNotEmpty(); + + newList.forEach(entity -> { + entity.setAge(20); + }); + TEMPLATE.batchUpdateById(INDEX, TYPE, newList); + TimeUnit.SECONDS.sleep(1); + + List expectList = + TEMPLATE.pojoListByIds(INDEX, TYPE, ids, User.class); + Assertions.assertThat(expectList).isNotEmpty(); + for (User item : expectList) { + Assertions.assertThat(item.getAge()).isEqualTo(20); + } + } + + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/TestApplication.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/TestApplication.java new file mode 100644 index 00000000..e9ae33cd --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/TestApplication.java @@ -0,0 +1,20 @@ +package io.github.dunwu.javadb.elasticsearch; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; + +@SpringBootApplication +public class TestApplication extends SpringBootServletInitializer { + + public static void main(String[] args) { + SpringApplication.run(TestApplication.class, args); + } + + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) { + return builder.sources(TestApplication.class); + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java new file mode 100644 index 00000000..65c79274 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java @@ -0,0 +1,191 @@ +package io.github.dunwu.javadb.elasticsearch.mapper; + +import io.github.dunwu.javadb.elasticsearch.BaseApplicationTests; +import io.github.dunwu.javadb.elasticsearch.entity.Page; +import io.github.dunwu.javadb.elasticsearch.entity.User; +import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; +import org.assertj.core.api.Assertions; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * ElasticsearchTemplate 测试 + * + * @author Zhang Peng + * @date 2023-11-13 + */ +public class UserEsMapperTest extends BaseApplicationTests { + + @Autowired + private UserEsMapper mapper; + public static final String TEST_ID_01 = "1"; + public static final String TEST_ID_02 = "2"; + + @Test + @DisplayName("根据ID精确查询") + public void getById() throws IOException { + GetResponse response = mapper.getById(TEST_ID_01); + System.out.println("记录：" + JsonUtil.toString(response.getSourceAsMap())); + } + + @Test + @DisplayName("根据ID精确查询POJO") + public void pojoById() throws IOException { + User entity = mapper.pojoById(TEST_ID_01); + System.out.println("记录：" + JsonUtil.toString(entity)); + } + + @Test + @DisplayName("根据ID精确批量查询POJO") + public void pojoListByIds() throws IOException { + List ids = Arrays.asList(TEST_ID_01, TEST_ID_02); + List list = mapper.pojoListByIds(ids); + Assertions.assertThat(list).isNotEmpty(); + Assertions.assertThat(list.size()).isEqualTo(2); + for (User entity : list) { + System.out.println("记录：" + JsonUtil.toString(entity)); + } + } + + @Test + @DisplayName("分页查询") + public void pojoPage() throws IOException { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("id", 1)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); + + Page page = mapper.pojoPage(searchSourceBuilder); + Assertions.assertThat(page).isNotNull(); + Assertions.assertThat(page.getContent()).isNotEmpty(); + for (User entity : page.getContent()) { + System.out.println("记录：" + JsonUtil.toString(entity)); + } + } + + @Test + @DisplayName("条件数量查询") + public void count() throws IOException { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("id", 1)); + // boolQueryBuilder.must(QueryBuilders.rangeQuery("age") + // .from(18) + // .to(25) + // .includeLower(true) + // .includeUpper(true)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); + long total = mapper.count(searchSourceBuilder); + Assertions.assertThat(total).isNotZero(); + System.out.println("符合条件的总记录数：" + total); + } + + @Test + @DisplayName("条件查询") + public void query() throws IOException { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.termQuery("id", 1)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQueryBuilder); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); + SearchResponse response = mapper.query(searchSourceBuilder); + Assertions.assertThat(response).isNotNull(); + Assertions.assertThat(response.getHits()).isNotNull(); + for (SearchHit hit : response.getHits().getHits()) { + System.out.println("记录：" + hit.getSourceAsString()); + Map map = hit.getSourceAsMap(); + Assertions.assertThat(map).isNotNull(); + } + } + + @Nested + @DisplayName("写操作测试") + public class WriteTest { + + String json1 = + "{\"id\":1,\"username\":\"user1\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user1@xxx.com\"}"; + String json2 = + "{\"id\":2,\"username\":\"user2\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user2@xxx.com\"}"; + + @Test + @DisplayName("插入、更新") + public void saveAndUpdate() throws IOException, InterruptedException { + + User origin = JsonUtil.toBean(json1, User.class); + if (origin == null) { + System.err.println("反序列化失败！"); + return; + } + + mapper.save(origin); + TimeUnit.SECONDS.sleep(1); + User expectEntity = mapper.pojoById(origin.getDocId()); + Assertions.assertThat(expectEntity).isNotNull(); + + expectEntity.setAge(20); + mapper.updateById(expectEntity); + TimeUnit.SECONDS.sleep(1); + User expectEntity2 = mapper.pojoById(origin.getDocId()); + Assertions.assertThat(expectEntity2).isNotNull(); + Assertions.assertThat(expectEntity2.getAge()).isEqualTo(20); + } + + @Test + @DisplayName("批量插入、更新") + public void batchSaveAndUpdate() throws IOException, InterruptedException { + + User origin1 = JsonUtil.toBean(json1, User.class); + if (origin1 == null) { + System.err.println("反序列化失败！"); + return; + } + + User origin2 = JsonUtil.toBean(json2, User.class); + if (origin2 == null) { + System.err.println("反序列化失败！"); + return; + } + + List list = Arrays.asList(origin1, origin2); + List ids = list.stream().map(User::getDocId).collect(Collectors.toList()); + + mapper.batchSave(list); + List newList = mapper.pojoListByIds(ids); + Assertions.assertThat(newList).isNotEmpty(); + + newList.forEach(entity -> { + entity.setAge(20); + }); + mapper.batchUpdateById(newList); + TimeUnit.SECONDS.sleep(1); + + List expectList = mapper.pojoListByIds(ids); + Assertions.assertThat(expectList).isNotEmpty(); + for (User item : expectList) { + Assertions.assertThat(item.getAge()).isEqualTo(20); + } + } + + } + +} From 72e3c1142e360f4cf0f1ce635f0b8ba7e21a2b04 Mon Sep 17 00:00:00 2001 From: dunwu Date: Thu, 11 Apr 2024 07:49:44 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20elasticsearch=206.x=20=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch/elasticsearch6/pom.xml | 4 +- .../dunwu/javadb/elasticsearch/Demo.java | 35 -- .../elasticsearch/ElasticsearchFactory.java | 43 +- .../elasticsearch/ElasticsearchTemplate.java | 275 ++++++++- .../config/ElasticsearchConfig.java | 20 +- .../elasticsearch/constant/CodeMsg.java | 15 + .../elasticsearch/constant/ResultCode.java | 97 ++++ .../elasticsearch/entity/BaseEsEntity.java | 10 + .../javadb/elasticsearch/entity/Page.java | 36 -- .../javadb/elasticsearch/entity/User.java | 37 +- .../elasticsearch/entity/common/PageData.java | 37 ++ .../entity/common/ScrollData.java | 30 + .../exception/CodeMsgException.java | 128 +++++ .../exception/DefaultException.java | 72 +++ .../mapper/BaseDynamicEsMapper.java | 210 +++++++ .../elasticsearch/mapper/BaseEsMapper.java | 201 +++++-- .../javadb/elasticsearch/mapper/EsMapper.java | 53 +- .../elasticsearch/mapper/UserEsMapper.java | 18 +- .../elasticsearch/BaseApplicationTests.java | 5 +- .../BaseElasticsearchTemplateTest.java | 241 ++++++++ .../ElasticsearchTemplateTest.java | 193 ------- .../UserElasticsearchTemplateTest.java | 108 ++++ .../mapper/UserEsMapperTest.java | 520 ++++++++++++++---- 23 files changed, 1894 insertions(+), 494 deletions(-) delete mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/CodeMsg.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/ResultCode.java delete mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/PageData.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/ScrollData.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/CodeMsgException.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/DefaultException.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java delete mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java create mode 100644 codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java diff --git a/codes/javadb/elasticsearch/elasticsearch6/pom.xml b/codes/javadb/elasticsearch/elasticsearch6/pom.xml index b9d8859b..72683b1d 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/pom.xml +++ b/codes/javadb/elasticsearch/elasticsearch6/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - o + org.springframework.boot spring-boot-starter-parent 2.7.7 @@ -42,7 +42,7 @@ cn.hutool hutool-all - 5.8.8 + 5.8.25 diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java deleted file mode 100644 index 3f97a580..00000000 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/Demo.java +++ /dev/null @@ -1,35 +0,0 @@ -package io.github.dunwu.javadb.elasticsearch; - -import io.github.dunwu.javadb.elasticsearch.entity.User; -import io.github.dunwu.javadb.elasticsearch.mapper.UserEsMapper; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -public class Demo { - - private static final String env = "test"; - private static final ElasticsearchTemplate elasticsearchTemplate - = ElasticsearchFactory.newElasticsearchTemplate(env); - - public static void main(String[] args) throws IOException, InterruptedException { - - UserEsMapper mapper = new UserEsMapper(elasticsearchTemplate); - - System.out.println("索引是否存在：" + mapper.isIndexExists()); - - User jack = User.builder().id(1L).username("jack").age(18).build(); - User tom = User.builder().id(2L).username("tom").age(20).build(); - List users = Arrays.asList(jack, tom); - - System.out.println("批量插入：" + mapper.batchSave(users)); - System.out.println("根据ID查询：" + mapper.getById("1").toString()); - System.out.println("根据ID查询：" + mapper.pojoById("2").toString()); - System.out.println("根据ID批量查询：" + mapper.pojoListByIds(Arrays.asList("1", "2")).toString()); - - Thread.sleep(1000); - System.out.println("根据ID批量删除：" + mapper.batchDeleteById(Arrays.asList("1", "2"))); - } - -} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java index 8582fdcf..33109cb0 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchFactory.java @@ -9,6 +9,7 @@ import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -38,11 +39,13 @@ public static RestClient newRestClient() { } public static RestClient newRestClient(String env) { - String hosts = getDefaultEsAddress(env); - return newRestClient(toHttpHostList(hosts)); + String hostsConfig = getDefaultEsAddress(env); + List hosts = StrUtil.split(hostsConfig, ","); + return newRestClient(hosts); } - public static RestClient newRestClient(HttpHost[] httpHosts) { + public static RestClient newRestClient(Collection hosts) { + HttpHost[] httpHosts = toHttpHostList(hosts); RestClientBuilder builder = getRestClientBuilder(httpHosts); if (builder == null) { return null; @@ -62,11 +65,13 @@ public static RestHighLevelClient newRestHighLevelClient() { } public static RestHighLevelClient newRestHighLevelClient(String env) { - String hosts = getDefaultEsAddress(env); - return newRestHighLevelClient(toHttpHostList(hosts)); + String hostsConfig = getDefaultEsAddress(env); + List hosts = StrUtil.split(hostsConfig, ","); + return newRestHighLevelClient(hosts); } - public static RestHighLevelClient newRestHighLevelClient(HttpHost[] httpHosts) { + public static RestHighLevelClient newRestHighLevelClient(Collection hosts) { + HttpHost[] httpHosts = toHttpHostList(hosts); RestClientBuilder builder = getRestClientBuilder(httpHosts); if (builder == null) { return null; @@ -86,12 +91,13 @@ public static ElasticsearchTemplate newElasticsearchTemplate() { } public static ElasticsearchTemplate newElasticsearchTemplate(String env) { - String hosts = getDefaultEsAddress(env); - return newElasticsearchTemplate(toHttpHostList(hosts)); + String hostsConfig = getDefaultEsAddress(env); + List hosts = StrUtil.split(hostsConfig, ","); + return newElasticsearchTemplate(hosts); } - public static ElasticsearchTemplate newElasticsearchTemplate(HttpHost[] httpHosts) { - RestHighLevelClient client = newRestHighLevelClient(httpHosts); + public static ElasticsearchTemplate newElasticsearchTemplate(Collection hosts) { + RestHighLevelClient client = newRestHighLevelClient(hosts); if (client == null) { return null; } @@ -125,21 +131,22 @@ public static RestClientBuilder getRestClientBuilder(HttpHost[] httpHosts) { return restClientBuilder; } - private static HttpHost[] toHttpHostList(String hosts) { - if (StrUtil.isBlank(hosts)) { - return null; + private static HttpHost[] toHttpHostList(Collection hosts) { + if (CollectionUtil.isEmpty(hosts)) { + return new HttpHost[0]; } - List strList = StrUtil.split(hosts, ","); - List list = strList.stream().map(str -> { - List params = StrUtil.split(str, ":"); - return new HttpHost(params.get(0), Integer.parseInt(params.get(1)), "http"); - }).collect(Collectors.toList()); + List list = hosts.stream().map(ElasticsearchFactory::toHttpHost).collect(Collectors.toList()); if (CollectionUtil.isEmpty(list)) { return new HttpHost[0]; } return list.toArray(new HttpHost[0]); } + public static HttpHost toHttpHost(String host) { + List params = StrUtil.split(host, ":"); + return new HttpHost(params.get(0), Integer.parseInt(params.get(1)), "http"); + } + public static String getDefaultEsAddress() { // 从配置中心读取环境变量 String env = "test"; diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java index cbe281a7..730f9f17 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java @@ -6,11 +6,19 @@ import cn.hutool.core.util.ArrayUtil; import cn.hutool.core.util.StrUtil; import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; -import io.github.dunwu.javadb.elasticsearch.entity.Page; +import io.github.dunwu.javadb.elasticsearch.entity.common.PageData; +import io.github.dunwu.javadb.elasticsearch.entity.common.ScrollData; import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; @@ -23,19 +31,31 @@ import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import java.io.Closeable; import java.io.IOException; @@ -122,6 +142,99 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) return bulkProcessor; } + // ==================================================================== + // 索引管理操作 + // ==================================================================== + + public void createIndex(String index, String type, String alias, int shard, int replica) throws IOException { + + if (StrUtil.isBlank(index) || StrUtil.isBlank(type)) { + throw new ElasticsearchException("【ES】index、type 不能为空！"); + } + + CreateIndexRequest request = new CreateIndexRequest(index); + if (StrUtil.isNotBlank(alias)) { + request.alias(new Alias(alias)); + } + + Settings.Builder settings = + Settings.builder().put("index.number_of_shards", shard).put("index.number_of_replicas", replica); + request.settings(settings); + AcknowledgedResponse response = client.indices().create(request, RequestOptions.DEFAULT); + if (!response.isAcknowledged()) { + String msg = StrUtil.format("【ES】创建索引失败！index: {}, type: {}", index, type); + throw new ElasticsearchException(msg); + } + } + + public void deleteIndex(String index) throws IOException { + DeleteIndexRequest request = new DeleteIndexRequest(index); + AcknowledgedResponse response = client.indices().delete(request, RequestOptions.DEFAULT); + if (!response.isAcknowledged()) { + String msg = StrUtil.format("【ES】删除索引失败！index: {}", index); + throw new ElasticsearchException(msg); + } + } + + public void updateAlias(String index, String alias) throws IOException { + IndicesAliasesRequest request = new IndicesAliasesRequest(); + IndicesAliasesRequest.AliasActions aliasAction = + new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD).index(index) + .alias(alias); + request.addAliasAction(aliasAction); + AcknowledgedResponse response = client.indices().updateAliases(request, RequestOptions.DEFAULT); + if (!response.isAcknowledged()) { + String msg = StrUtil.format("【ES】更新索引别名失败！index: {}, alias: {}", index, alias); + throw new ElasticsearchException(msg); + } + } + + public boolean isIndexExists(String index) throws IOException { + GetIndexRequest request = new GetIndexRequest(); + return client.indices().exists(request.indices(index), RequestOptions.DEFAULT); + } + + public void setMapping(String index, String type, Map propertiesMap) throws IOException { + + if (MapUtil.isEmpty(propertiesMap)) { + throw new ElasticsearchException("【ES】设置 mapping 的 properties 不能为空！"); + } + + PutMappingRequest request = new PutMappingRequest(index).type(type); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(type); + builder.startObject("properties"); + + for (Map.Entry entry : propertiesMap.entrySet()) { + + String field = entry.getKey(); + String fieldType = entry.getValue(); + if (StrUtil.isBlank(field) || StrUtil.isBlank(fieldType)) { + continue; + } + + builder.startObject(field); + { + builder.field("type", fieldType); + } + builder.endObject(); + } + + builder.endObject(); + builder.endObject(); + builder.endObject(); + request.source(builder); + AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT); + if (!response.isAcknowledged()) { + throw new ElasticsearchException("【ES】设置 mapping 失败！"); + } + } + + // ==================================================================== + // CRUD 操作 + // ==================================================================== + public T save(String index, String type, T entity) throws IOException { if (entity == null) { @@ -154,7 +267,7 @@ public T save(String index, String type, T entity) thro } } - public boolean batchSave(String index, String type, Collection list) + public boolean saveBatch(String index, String type, Collection list) throws IOException { if (CollectionUtil.isEmpty(list)) { @@ -179,7 +292,7 @@ public boolean batchSave(String index, String type, Col return response != null && !response.hasFailures(); } - public void asyncBatchSave(String index, String type, Collection list, + public void asyncSaveBatch(String index, String type, Collection list, ActionListener listener) { if (CollectionUtil.isEmpty(list)) { @@ -236,7 +349,7 @@ public T updateById(String index, String type, T entity } } - public boolean batchUpdateById(String index, String type, Collection list) + public boolean updateBatchIds(String index, String type, Collection list) throws IOException { if (CollectionUtil.isEmpty(list)) { @@ -248,7 +361,7 @@ public boolean batchUpdateById(String index, String typ return response != null && !response.hasFailures(); } - public void asyncBatchUpdateById(String index, String type, Collection list, + public void asyncUpdateBatchIds(String index, String type, Collection list, ActionListener listener) { if (CollectionUtil.isEmpty(list)) { @@ -277,10 +390,10 @@ private BulkRequest toUpdateBulkRequest(String index, S } public boolean deleteById(String index, String type, String id) throws IOException { - return batchDeleteById(index, type, Collections.singleton(id)); + return deleteBatchIds(index, type, Collections.singleton(id)); } - public boolean batchDeleteById(String index, String type, Collection ids) throws IOException { + public boolean deleteBatchIds(String index, String type, Collection ids) throws IOException { if (CollectionUtil.isEmpty(ids)) { return true; @@ -302,7 +415,7 @@ public boolean batchDeleteById(String index, String type, Collection ids return !response.hasFailures(); } - public void asyncBatchDeleteById(String index, String type, Collection ids, + public void asyncDeleteBatchIds(String index, String type, Collection ids, ActionListener listener) { if (CollectionUtil.isEmpty(ids)) { @@ -375,21 +488,6 @@ public List pojoListByIds(String index, String type, Collection i return list; } - public Page pojoPage(String index, String type, SearchSourceBuilder builder, Class clazz) - throws IOException { - SearchResponse response = query(index, type, builder); - if (response == null || response.status() != RestStatus.OK) { - return null; - } - - List content = toPojoList(response, clazz); - SearchHits searchHits = response.getHits(); - int offset = builder.from(); - int size = builder.size(); - int page = offset / size + (offset % size == 0 ? 0 : 1) + 1; - return new Page<>(page, size, searchHits.getTotalHits(), content); - } - public long count(String index, String type, SearchSourceBuilder builder) throws IOException { SearchResponse response = query(index, type, builder); if (response == null || response.status() != RestStatus.OK) { @@ -399,6 +497,12 @@ public long count(String index, String type, SearchSourceBuilder builder) throws return searchHits.getTotalHits(); } + public long count(String index, String type, QueryBuilder queryBuilder) throws IOException { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + return count(index, type, searchSourceBuilder); + } + public SearchResponse query(String index, String type, SearchSourceBuilder builder) throws IOException { SearchRequest request = new SearchRequest(index).types(type); request.source(builder); @@ -409,6 +513,131 @@ public SearchResponse query(SearchRequest request) throws IOException { return client.search(request, RequestOptions.DEFAULT); } + /** + * from+size 分页 + *

+ * 注：在深分页的场景下，效率很低（一般超过 1万条数据就不适用了） + */ + public PageData pojoPage(String index, String type, SearchSourceBuilder builder, Class clazz) + throws IOException { + SearchResponse response = query(index, type, builder); + if (response == null || response.status() != RestStatus.OK) { + return null; + } + + List content = toPojoList(response, clazz); + SearchHits searchHits = response.getHits(); + int from = builder.from(); + int size = builder.size(); + int page = from / size + (from % size == 0 ? 0 : 1) + 1; + return new PageData<>(page, size, searchHits.getTotalHits(), content); + } + + /** + * from+size 分页 + *

+ * 注：在深分页的场景下，效率很低（一般超过 1万条数据就不适用了） + */ + public PageData pojoPage(String index, String type, int from, int size, QueryBuilder queryBuilder, + Class clazz) throws IOException { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(from); + searchSourceBuilder.size(size); + return pojoPage(index, type, searchSourceBuilder, clazz); + } + + /** + * search after 分页 + */ + public ScrollData pojoPageByLastId(String index, String type, String lastId, int size, + QueryBuilder queryBuilder, Class clazz) throws IOException { + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(size); + searchSourceBuilder.sort(BaseEsEntity.DOC_ID, SortOrder.ASC); + if (StrUtil.isNotBlank(lastId)) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(queryBuilder).must(QueryBuilders.rangeQuery(BaseEsEntity.DOC_ID).gt(lastId)); + searchSourceBuilder.query(boolQueryBuilder); + } else { + searchSourceBuilder.query(queryBuilder); + } + + SearchResponse response = query(index, type, searchSourceBuilder); + if (response == null || response.status() != RestStatus.OK) { + return null; + } + List content = toPojoList(response, clazz); + ScrollData scrollData = new ScrollData<>(); + scrollData.setSize(size); + scrollData.setTotal(response.getHits().getTotalHits()); + scrollData.setContent(content); + if (CollectionUtil.isNotEmpty(content)) { + T lastEntity = content.get(content.size() - 1); + scrollData.setScrollId(lastEntity.getDocId()); + } + return scrollData; + } + + /** + * 首次滚动查询批量查询，但是不适用与搜索，仅用于批查询 + **/ + public ScrollData pojoScrollBegin(String index, String type, SearchSourceBuilder searchBuilder, + Class clazz) throws IOException { + + int scrollTime = 10; + final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(scrollTime)); + SearchRequest request = new SearchRequest(index); + request.types(type); + request.source(searchBuilder); + request.scroll(scroll); + SearchResponse response = client.search(request, RequestOptions.DEFAULT); + if (response == null || response.status() != RestStatus.OK) { + return null; + } + List content = toPojoList(response, clazz); + ScrollData scrollData = new ScrollData<>(); + scrollData.setSize(searchBuilder.size()); + scrollData.setTotal(response.getHits().getTotalHits()); + scrollData.setScrollId(response.getScrollId()); + scrollData.setContent(content); + return scrollData; + } + + /** + * 知道ScrollId之后，后续根据scrollId批量查询 + **/ + public ScrollData pojoScroll(String scrollId, SearchSourceBuilder searchBuilder, Class clazz) + throws IOException { + + int scrollTime = 10; + final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(scrollTime)); + SearchScrollRequest request = new SearchScrollRequest(scrollId); + request.scroll(scroll); + SearchResponse response = client.scroll(request, RequestOptions.DEFAULT); + if (response == null || response.status() != RestStatus.OK) { + return null; + } + List content = toPojoList(response, clazz); + ScrollData scrollData = new ScrollData<>(); + scrollData.setSize(searchBuilder.size()); + scrollData.setTotal(response.getHits().getTotalHits()); + scrollData.setScrollId(response.getScrollId()); + scrollData.setContent(content); + return scrollData; + } + + public boolean pojoScrollEnd(String scrollId) throws IOException { + ClearScrollRequest request = new ClearScrollRequest(); + request.addScrollId(scrollId); + ClearScrollResponse response = client.clearScroll(request, RequestOptions.DEFAULT); + if (response != null) { + return response.isSucceeded(); + } + return false; + } + public T toPojo(GetResponse response, Class clazz) { if (null == response) { return null; diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java index bdd2f387..791bbf1c 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/config/ElasticsearchConfig.java @@ -1,12 +1,17 @@ package io.github.dunwu.javadb.elasticsearch.config; +import cn.hutool.core.util.StrUtil; import io.github.dunwu.javadb.elasticsearch.ElasticsearchFactory; import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import java.util.List; + /** * ES 配置 * @@ -17,14 +22,23 @@ @ComponentScan(value = "io.github.dunwu.javadb.elasticsearch.mapper") public class ElasticsearchConfig { + @Value("${es.hosts:#{null}}") + private String hostsConfig; + @Bean("restHighLevelClient") + @ConditionalOnMissingBean public RestHighLevelClient restHighLevelClient() { - return ElasticsearchFactory.newRestHighLevelClient(); + if (hostsConfig == null) { + return ElasticsearchFactory.newRestHighLevelClient(); + } else { + List hosts = StrUtil.split(hostsConfig, ","); + return ElasticsearchFactory.newRestHighLevelClient(hosts); + } } @Bean("elasticsearchTemplate") - public ElasticsearchTemplate elasticsearchTemplate() { - return ElasticsearchFactory.newElasticsearchTemplate(); + public ElasticsearchTemplate elasticsearchTemplate(RestHighLevelClient restHighLevelClient) { + return ElasticsearchFactory.newElasticsearchTemplate(restHighLevelClient); } } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/CodeMsg.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/CodeMsg.java new file mode 100644 index 00000000..96e46f6c --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/CodeMsg.java @@ -0,0 +1,15 @@ +package io.github.dunwu.javadb.elasticsearch.constant; + +/** + * 请求 / 应答状态接口 + * + * @author Zhang Peng + * @since 2019-06-06 + */ +public interface CodeMsg { + + int getCode(); + + String getMsg(); + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/ResultCode.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/ResultCode.java new file mode 100644 index 00000000..d4822fb8 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/constant/ResultCode.java @@ -0,0 +1,97 @@ +package io.github.dunwu.javadb.elasticsearch.constant; + +import cn.hutool.core.util.StrUtil; + +import java.util.stream.Stream; + +/** + * 系统级错误码 + * + * @author Zhang Peng + * @see HTTP 状态码 + * @see 腾讯开放平台错误码 + * @see 新浪开放平台错误码 + * @see 支付宝开放平台API + * @see 微信开放平台错误码 + * @since 2019-04-11 + */ +public enum ResultCode implements CodeMsg { + + OK(0, "成功"), + + PART_OK(1, "部分成功"), + + FAIL(-1, "失败"), + + // ----------------------------------------------------- + // 系统级错误码 + // ----------------------------------------------------- + + ERROR(1000, "服务器错误"), + + PARAM_ERROR(1001, "参数错误"), + + TASK_ERROR(1001, "调度任务错误"), + + CONFIG_ERROR(1003, "配置错误"), + + REQUEST_ERROR(1004, "请求错误"), + + IO_ERROR(1005, "IO 错误"), + + // ----------------------------------------------------- + // 2000 ~ 2999 数据库错误 + // ----------------------------------------------------- + + DATA_ERROR(2000, "数据库错误"), + + // ----------------------------------------------------- + // 3000 ~ 3999 三方错误 + // ----------------------------------------------------- + + THIRD_PART_ERROR(3000, "三方错误"), + + // ----------------------------------------------------- + // 3000 ~ 3999 认证错误 + // ----------------------------------------------------- + + AUTH_ERROR(4000, "认证错误"); + + private final int code; + + private final String msg; + + ResultCode(int code, String msg) { + this.code = code; + this.msg = msg; + } + + @Override + public int getCode() { + return code; + } + + @Override + public String getMsg() { + return msg; + } + + public static String getNameByCode(int code) { + return Stream.of(ResultCode.values()).filter(item -> item.getCode() == code).findFirst() + .map(ResultCode::getMsg).orElse(null); + } + + public static ResultCode getEnumByCode(int code) { + return Stream.of(ResultCode.values()).filter(item -> item.getCode() == code).findFirst().orElse(null); + } + + public static String getTypeInfo() { + StringBuilder sb = new StringBuilder(); + ResultCode[] types = ResultCode.values(); + for (ResultCode type : types) { + sb.append(StrUtil.format("{}:{}, ", type.getCode(), type.getMsg())); + } + return sb.toString(); + } +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java index a7759763..32206066 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/BaseEsEntity.java @@ -4,6 +4,8 @@ import lombok.ToString; import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; /** * ES 实体接口 @@ -15,6 +17,8 @@ @ToString public abstract class BaseEsEntity implements Serializable { + public static final String DOC_ID = "docId"; + /** * 获取版本 */ @@ -24,4 +28,10 @@ public abstract class BaseEsEntity implements Serializable { public abstract String getDocId(); + public static Map getPropertiesMap() { + Map map = new LinkedHashMap<>(1); + map.put(BaseEsEntity.DOC_ID, "keyword"); + return map; + } + } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java deleted file mode 100644 index 8baef8b5..00000000 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/Page.java +++ /dev/null @@ -1,36 +0,0 @@ -package io.github.dunwu.javadb.elasticsearch.entity; - -import lombok.Data; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * 分页实体 - * - * @author Zhang Peng - * @date 2023-06-28 - */ -@Data -public class Page { - - private long total; - private int page; - private int size; - private List content = new ArrayList<>(); - - public Page(int page, int size, long total) { - this.total = total; - this.page = page; - this.size = size; - } - - public Page(int page, int size, long total, Collection list) { - this.total = total; - this.page = page; - this.size = size; - this.content.addAll(list); - } - -} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java index 1594772c..b21b229c 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/User.java @@ -2,32 +2,43 @@ import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; /** - * 用户实体 + * 短剧、长视频消费数据 ES 实体 * * @author Zhang Peng - * @since 2023-06-28 + * @date 2024-04-02 */ -@Data @Builder -@EqualsAndHashCode(callSuper = true) -@AllArgsConstructor +@Getter +@Setter @NoArgsConstructor -public class User extends BaseEsEntity { +@AllArgsConstructor +public class User extends BaseEsEntity implements Serializable { - private Long id; - private String username; - private String password; + private String id; + private String name; private Integer age; - private String email; @Override public String getDocId() { - return String.valueOf(id); + return id; + } + + public static Map getPropertiesMap() { + Map map = new LinkedHashMap<>(); + map.put(BaseEsEntity.DOC_ID, "keyword"); + map.put("id", "long"); + map.put("name", "keyword"); + map.put("age", "integer"); + return map; } } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/PageData.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/PageData.java new file mode 100644 index 00000000..e436f8b6 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/PageData.java @@ -0,0 +1,37 @@ +package io.github.dunwu.javadb.elasticsearch.entity.common; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * 分页实体 + * + * @author Zhang Peng + * @date 2023-06-28 + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class PageData implements Serializable { + + private int page; + private int size; + private long total; + private List content = new ArrayList<>(); + + public PageData(int page, int size, long total) { + this.total = total; + this.page = page; + this.size = size; + } + + private static final long serialVersionUID = 1L; + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/ScrollData.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/ScrollData.java new file mode 100644 index 00000000..4f90cb85 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/entity/common/ScrollData.java @@ -0,0 +1,30 @@ +package io.github.dunwu.javadb.elasticsearch.entity.common; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; +import java.util.Collection; + +/** + * Hbase 滚动数据实体 + * + * @author Zhang Peng + * @date 2023-11-16 + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class ScrollData implements Serializable { + + private String scrollId; + private int size = 10; + private long total = 0L; + private Collection content; + + private static final long serialVersionUID = 1L; + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/CodeMsgException.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/CodeMsgException.java new file mode 100644 index 00000000..98ab1995 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/CodeMsgException.java @@ -0,0 +1,128 @@ +package io.github.dunwu.javadb.elasticsearch.exception; + +import cn.hutool.core.util.StrUtil; +import io.github.dunwu.javadb.elasticsearch.constant.CodeMsg; +import io.github.dunwu.javadb.elasticsearch.constant.ResultCode; + +/** + * 基础异常 + * + * @author Zhang Peng + * @since 2021-09-25 + */ +public class CodeMsgException extends RuntimeException implements CodeMsg { + + private static final long serialVersionUID = 6146660782281445735L; + + /** + * 状态码 + */ + protected int code; + + /** + * 响应信息 + */ + protected String msg; + + /** + * 提示信息 + */ + protected String toast; + + public CodeMsgException() { + this(ResultCode.FAIL); + } + + public CodeMsgException(CodeMsg codeMsg) { + this(codeMsg.getCode(), codeMsg.getMsg()); + } + + public CodeMsgException(CodeMsg codeMsg, String msg) { + this(codeMsg.getCode(), msg, null); + } + + public CodeMsgException(CodeMsg codeMsg, String msg, String toast) { + this(codeMsg.getCode(), msg, toast); + } + + public CodeMsgException(String msg) { + this(ResultCode.FAIL, msg); + } + + public CodeMsgException(int code, String msg) { + this(code, msg, msg); + } + + public CodeMsgException(int code, String msg, String toast) { + super(msg); + setCode(code); + setMsg(msg); + setToast(toast); + } + + public CodeMsgException(Throwable cause) { + this(cause, ResultCode.FAIL); + } + + public CodeMsgException(Throwable cause, String msg) { + this(cause, ResultCode.FAIL, msg); + } + + public CodeMsgException(Throwable cause, CodeMsg codeMsg) { + this(cause, codeMsg.getCode(), codeMsg.getMsg()); + } + + public CodeMsgException(Throwable cause, CodeMsg codeMsg, String msg) { + this(cause, codeMsg.getCode(), msg, null); + } + + public CodeMsgException(Throwable cause, CodeMsg codeMsg, String msg, String toast) { + this(cause, codeMsg.getCode(), msg, toast); + } + + public CodeMsgException(Throwable cause, int code, String msg) { + this(cause, code, msg, null); + } + + public CodeMsgException(Throwable cause, int code, String msg, String toast) { + super(msg, cause); + setCode(code); + setMsg(msg); + setToast(toast); + } + + @Override + public String getMessage() { + if (StrUtil.isNotBlank(msg)) { + return StrUtil.format("[{}]{}", code, msg); + } + return super.getMessage(); + } + + @Override + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + @Override + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public String getToast() { + return toast; + } + + public void setToast(String toast) { + this.toast = toast; + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/DefaultException.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/DefaultException.java new file mode 100644 index 00000000..14908e39 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/exception/DefaultException.java @@ -0,0 +1,72 @@ +package io.github.dunwu.javadb.elasticsearch.exception; + +import io.github.dunwu.javadb.elasticsearch.constant.CodeMsg; +import io.github.dunwu.javadb.elasticsearch.constant.ResultCode; + +/** + * 默认异常 + * + * @author Zhang Peng + * @since 2021-12-30 + */ +public class DefaultException extends CodeMsgException { + + private static final long serialVersionUID = -7027578114976830416L; + + public DefaultException() { + this(ResultCode.FAIL); + } + + public DefaultException(CodeMsg codeMsg) { + this(codeMsg.getCode(), codeMsg.getMsg()); + } + + public DefaultException(CodeMsg codeMsg, String msg) { + this(codeMsg.getCode(), msg, null); + } + + public DefaultException(CodeMsg codeMsg, String msg, String toast) { + this(codeMsg.getCode(), msg, toast); + } + + public DefaultException(String msg) { + this(ResultCode.FAIL, msg); + } + + public DefaultException(int code, String msg) { + this(code, msg, msg); + } + + public DefaultException(int code, String msg, String toast) { + super(code, msg, toast); + } + + public DefaultException(Throwable cause) { + this(cause, ResultCode.FAIL); + } + + public DefaultException(Throwable cause, String msg) { + this(cause, ResultCode.FAIL, msg); + } + + public DefaultException(Throwable cause, CodeMsg codeMsg) { + this(cause, codeMsg.getCode(), codeMsg.getMsg()); + } + + public DefaultException(Throwable cause, CodeMsg codeMsg, String msg) { + this(cause, codeMsg.getCode(), msg, null); + } + + public DefaultException(Throwable cause, CodeMsg codeMsg, String msg, String toast) { + this(cause, codeMsg.getCode(), msg, toast); + } + + public DefaultException(Throwable cause, int code, String msg) { + this(cause, code, msg, null); + } + + public DefaultException(Throwable cause, int code, String msg, String toast) { + super(cause, code, msg, toast); + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java new file mode 100644 index 00000000..710594df --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java @@ -0,0 +1,210 @@ +package io.github.dunwu.javadb.elasticsearch.mapper; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.StrUtil; +import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; +import io.github.dunwu.javadb.elasticsearch.constant.ResultCode; +import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; +import io.github.dunwu.javadb.elasticsearch.entity.common.PageData; +import io.github.dunwu.javadb.elasticsearch.entity.common.ScrollData; +import io.github.dunwu.javadb.elasticsearch.exception.DefaultException; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * 动态 ES Mapper 基础类（以时间为维度动态创建、删除 index），用于数据量特别大，需要按照日期分片的索引。 + *

+ * 注：使用此 Mapper 的索引、别名必须遵循命名格式：索引名 = 别名_yyyyMMdd + * + * @author Zhang Peng + * @date 2024-04-07 + */ +@Slf4j +public abstract class BaseDynamicEsMapper extends BaseEsMapper { + + public BaseDynamicEsMapper(ElasticsearchTemplate elasticsearchTemplate) { + super(elasticsearchTemplate); + } + + // ==================================================================== + // 索引管理操作 + // ==================================================================== + + public String getIndex(String day) { + + String alias = getAlias(); + if (StrUtil.isBlank(day)) { + String msg = StrUtil.format("【ES】获取 {} 索引失败！day 不能为空！", alias); + throw new DefaultException(ResultCode.PARAM_ERROR, msg); + } + + DateTime date; + try { + date = DateUtil.parse(day, DatePattern.NORM_DATE_PATTERN); + } catch (Exception e) { + String msg = StrUtil.format("【ES】获取 {} 索引失败！day: {} 不符合日期格式 {}！", + alias, day, DatePattern.NORM_DATE_PATTERN); + throw new DefaultException(e, ResultCode.PARAM_ERROR, msg); + } + + String formatDate = DateUtil.format(date, DatePattern.PURE_DATE_FORMAT); + return alias + "_" + formatDate; + } + + public boolean isIndexExistsInDay(String day) throws IOException { + return elasticsearchTemplate.isIndexExists(getIndex(day)); + } + + public String createIndexInDay(String day) throws IOException, DefaultException { + String index = getIndex(day); + boolean indexExists = isIndexExistsInDay(day); + if (indexExists) { + return index; + } + elasticsearchTemplate.createIndex(index, getType(), getAlias(), getShard(), getReplica()); + Map map = getPropertiesMap(); + if (MapUtil.isNotEmpty(map)) { + elasticsearchTemplate.setMapping(index, getType(), map); + } + return index; + } + + public void deleteIndexInDay(String day) throws IOException { + elasticsearchTemplate.deleteIndex(getIndex(day)); + } + + public void updateAliasInDay(String day) throws IOException { + elasticsearchTemplate.updateAlias(getIndex(day), getAlias()); + } + + // ==================================================================== + // CRUD 操作 + // ==================================================================== + + public GetResponse getByIdInDay(String day, String id) throws IOException { + return elasticsearchTemplate.getById(getIndex(day), getType(), id, null); + } + + public T pojoByIdInDay(String day, String id) throws IOException { + return elasticsearchTemplate.pojoById(getIndex(day), getType(), id, null, getEntityClass()); + } + + public List pojoListByIdsInDay(String day, Collection ids) throws IOException { + return elasticsearchTemplate.pojoListByIds(getIndex(day), getType(), ids, getEntityClass()); + } + + public long countInDay(String day, SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.count(getIndex(day), getType(), builder); + } + + public SearchResponse queryInDay(String day, SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.query(getIndex(day), getType(), builder); + } + + public PageData pojoPageInDay(String day, SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.pojoPage(getIndex(day), getType(), builder, getEntityClass()); + } + + public ScrollData pojoPageByLastIdInDay(String day, String lastId, int size, QueryBuilder queryBuilder) + throws IOException { + return elasticsearchTemplate.pojoPageByLastId(getIndex(day), getType(), lastId, size, + queryBuilder, getEntityClass()); + } + + public ScrollData pojoScrollBeginInDay(String day, SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.pojoScrollBegin(getIndex(day), getType(), builder, getEntityClass()); + } + + /** + * 根据日期动态选择索引并更新 + * + * @param day 日期，格式为：yyyy-MM-dd + * @param entity 待更新的数据 + * @return / + */ + public boolean saveInDay(String day, T entity) throws IOException, DefaultException { + String index = checkIndex(day); + checkData(entity); + elasticsearchTemplate.save(index, getType(), entity); + return true; + } + + /** + * 根据日期动态选择索引并批量更新 + * + * @param day 日期，格式为：yyyy-MM-dd + * @param list 待更新的数据 + * @return / + */ + public boolean saveBatchInDay(String day, Collection list) throws IOException, DefaultException { + String index = checkIndex(day); + checkData(list); + elasticsearchTemplate.saveBatch(index, getType(), list); + return true; + } + + public void asyncSaveBatchInDay(String day, Collection list) throws IOException { + String index = checkIndex(day); + checkData(list); + ActionListener listener = new ActionListener() { + @Override + public void onResponse(BulkResponse response) { + if (response != null && !response.hasFailures()) { + String msg = StrUtil.format("【ES】按日期异步批量保存 {} 成功！", index); + log.info(msg); + } else { + String msg = StrUtil.format("【ES】按日期异步批量保存 {} 失败！", index); + log.warn(msg); + } + } + + @Override + public void onFailure(Exception e) { + String msg = StrUtil.format("【ES】按日期异步批量保存 {} 异常！", index); + log.error(msg, e); + } + }; + asyncSaveBatchInDay(day, list, listener); + } + + public void asyncSaveBatchInDay(String day, Collection list, ActionListener listener) + throws IOException { + String index = checkIndex(day); + checkData(list); + elasticsearchTemplate.asyncSaveBatch(getIndex(day), getType(), list, listener); + } + + public boolean deleteByIdInDay(String day, String id) throws IOException { + return elasticsearchTemplate.deleteById(getIndex(day), getType(), id); + } + + public boolean deleteBatchIdsInDay(String day, Collection ids) throws IOException { + return elasticsearchTemplate.deleteBatchIds(getIndex(day), getType(), ids); + } + + protected String checkIndex(String day) throws IOException { + if (!enableAutoCreateIndex()) { + return getIndex(day); + } + String index = createIndexInDay(day); + if (StrUtil.isBlank(index)) { + String msg = StrUtil.format("【ES】按日期批量保存 {} 失败！索引找不到且创建失败！", index); + throw new DefaultException(ResultCode.ERROR, msg); + } + return index; + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java index cd7e53a9..3232f9e9 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java @@ -1,23 +1,31 @@ package io.github.dunwu.javadb.elasticsearch.mapper; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.ReflectUtil; +import cn.hutool.core.util.StrUtil; import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; +import io.github.dunwu.javadb.elasticsearch.constant.ResultCode; import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; -import io.github.dunwu.javadb.elasticsearch.entity.Page; +import io.github.dunwu.javadb.elasticsearch.entity.common.PageData; +import io.github.dunwu.javadb.elasticsearch.entity.common.ScrollData; +import io.github.dunwu.javadb.elasticsearch.exception.DefaultException; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.IndicesClient; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * ES Mapper 基础类 @@ -28,7 +36,7 @@ @Slf4j public abstract class BaseEsMapper implements EsMapper { - private BulkProcessor bulkProcessor; + protected BulkProcessor bulkProcessor; protected final ElasticsearchTemplate elasticsearchTemplate; @@ -36,6 +44,21 @@ public BaseEsMapper(ElasticsearchTemplate elasticsearchTemplate) { this.elasticsearchTemplate = elasticsearchTemplate; } + public int getShard() { + return 5; + } + + public int getReplica() { + return 1; + } + + /** + * 如果开启，添加 ES 数据时，如果索引不存在，会自动创建索引 + */ + public boolean enableAutoCreateIndex() { + return true; + } + @Override public RestHighLevelClient getClient() { if (elasticsearchTemplate == null) { @@ -52,14 +75,63 @@ public synchronized BulkProcessor getBulkProcessor() { return bulkProcessor; } + @SuppressWarnings("unchecked") + public Map getPropertiesMap() { + + Class clazz = getEntityClass(); + Method method; + try { + method = clazz.getMethod("getPropertiesMap"); + } catch (NoSuchMethodException e) { + String msg = StrUtil.format("【ES】检查并创建 {} 索引失败！day 不能为空！", getAlias()); + throw new DefaultException(e, ResultCode.ERROR, msg); + } + + Object result = ReflectUtil.invokeStatic(method); + if (result == null) { + return new HashMap<>(0); + } + return (Map) result; + } + + // ==================================================================== + // 索引管理操作 + // ==================================================================== + @Override public boolean isIndexExists() throws IOException { - IndicesClient indicesClient = getClient().indices(); - GetIndexRequest request = new GetIndexRequest(); - request.indices(getIndex()); - return indicesClient.exists(request, RequestOptions.DEFAULT); + return elasticsearchTemplate.isIndexExists(getIndex()); + } + + @Override + public String createIndexIfNotExists() throws IOException { + String index = getIndex(); + boolean exists = elasticsearchTemplate.isIndexExists(index); + if (exists) { + return index; + } + elasticsearchTemplate.createIndex(index, getType(), getAlias(), getShard(), getReplica()); + Map propertiesMap = getPropertiesMap(); + if (MapUtil.isNotEmpty(propertiesMap)) { + elasticsearchTemplate.setMapping(index, getType(), propertiesMap); + } + return index; + } + + @Override + public void deleteIndex() throws IOException { + elasticsearchTemplate.deleteIndex(getIndex()); } + @Override + public void updateAlias() throws IOException { + elasticsearchTemplate.updateAlias(getIndex(), getAlias()); + } + + // ==================================================================== + // CRUD 操作 + // ==================================================================== + @Override public GetResponse getById(String id) throws IOException { return getById(id, null); @@ -85,11 +157,6 @@ public List pojoListByIds(Collection ids) throws IOException { return elasticsearchTemplate.pojoListByIds(getIndex(), getType(), ids, getEntityClass()); } - @Override - public Page pojoPage(SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.pojoPage(getIndex(), getType(), builder, getEntityClass()); - } - @Override public long count(SearchSourceBuilder builder) throws IOException { return elasticsearchTemplate.count(getIndex(), getType(), builder); @@ -99,55 +166,95 @@ public long count(SearchSourceBuilder builder) throws IOException { public SearchResponse query(SearchSourceBuilder builder) throws IOException { return elasticsearchTemplate.query(getIndex(), getType(), builder); } + @Override + public PageData pojoPage(SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.pojoPage(getIndex(), getType(), builder, getEntityClass()); + } + + @Override + public ScrollData pojoPageByLastId(String lastId, int size, QueryBuilder queryBuilder) throws IOException { + return elasticsearchTemplate.pojoPageByLastId(getIndex(), getType(), lastId, size, + queryBuilder, getEntityClass()); + } + + @Override + public ScrollData pojoScrollBegin(SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.pojoScrollBegin(getIndex(), getType(), builder, getEntityClass()); + } + + @Override + public ScrollData pojoScroll(String scrollId, SearchSourceBuilder builder) throws IOException { + return elasticsearchTemplate.pojoScroll(scrollId, builder, getEntityClass()); + } + + @Override + public boolean pojoScrollEnd(String scrollId) throws IOException { + return elasticsearchTemplate.pojoScrollEnd(scrollId); + } + @Override public T save(T entity) throws IOException { - return elasticsearchTemplate.save(getIndex(), getType(), entity); + String index = checkIndex(); + checkData(entity); + return elasticsearchTemplate.save(index, getType(), entity); } @Override - public boolean batchSave(Collection list) throws IOException { - return elasticsearchTemplate.batchSave(getIndex(), getType(), list); + public boolean saveBatch(Collection list) throws IOException { + String index = checkIndex(); + checkData(list); + return elasticsearchTemplate.saveBatch(index, getType(), list); } @Override - public void asyncBatchSave(Collection list) throws IOException { + public void asyncSaveBatch(Collection list) throws IOException { + String index = checkIndex(); + checkData(list); ActionListener listener = new ActionListener() { @Override public void onResponse(BulkResponse response) { if (response != null && !response.hasFailures()) { - log.info("【ES】异步批量插入成功！"); + String msg = StrUtil.format("【ES】异步批量保存 {} 成功！", index); + log.info(msg); } else { - log.warn("【ES】异步批量插入失败！"); + String msg = StrUtil.format("【ES】异步批量保存 {} 失败！", index); + log.warn(msg); } } @Override public void onFailure(Exception e) { - log.error("【ES】异步批量插入异常！", e); + String msg = StrUtil.format("【ES】异步批量保存 {} 异常！", index); + log.error(msg, e); } }; - asyncBatchSave(list, listener); + asyncSaveBatch(list, listener); } @Override - public void asyncBatchSave(Collection list, ActionListener listener) { - elasticsearchTemplate.asyncBatchSave(getIndex(), getType(), list, listener); + public void asyncSaveBatch(Collection list, ActionListener listener) throws IOException { + String index = checkIndex(); + checkData(list); + elasticsearchTemplate.asyncSaveBatch(index, getType(), list, listener); } @Override public T updateById(T entity) throws IOException { + checkData(entity); return elasticsearchTemplate.updateById(getIndex(), getType(), entity); } @Override - public boolean batchUpdateById(Collection list) throws IOException { - return elasticsearchTemplate.batchUpdateById(getIndex(), getType(), list); + public boolean updateBatchIds(Collection list) throws IOException { + checkData(list); + return elasticsearchTemplate.updateBatchIds(getIndex(), getType(), list); } @Override - public void asyncBatchUpdateById(Collection list, ActionListener listener) { - elasticsearchTemplate.asyncBatchUpdateById(getIndex(), getType(), list, listener); + public void asyncUpdateBatchIds(Collection list, ActionListener listener) { + checkData(list); + elasticsearchTemplate.asyncUpdateBatchIds(getIndex(), getType(), list, listener); } @Override @@ -156,12 +263,12 @@ public boolean deleteById(String id) throws IOException { } @Override - public boolean batchDeleteById(Collection ids) throws IOException { - return elasticsearchTemplate.batchDeleteById(getIndex(), getType(), ids); + public boolean deleteBatchIds(Collection ids) throws IOException { + return elasticsearchTemplate.deleteBatchIds(getIndex(), getType(), ids); } @Override - public void asyncBatchDeleteById(Collection ids) throws IOException { + public void asyncDeleteBatchIds(Collection ids) throws IOException { ActionListener listener = new ActionListener() { @Override public void onResponse(BulkResponse response) { @@ -177,12 +284,38 @@ public void onFailure(Exception e) { log.error("【ES】异步批量删除异常！ids: {}", ids, e); } }; - asyncBatchDeleteById(ids, listener); + asyncDeleteBatchIds(ids, listener); } @Override - public void asyncBatchDeleteById(Collection ids, ActionListener listener) throws IOException { - elasticsearchTemplate.asyncBatchDeleteById(getIndex(), getType(), ids, listener); + public void asyncDeleteBatchIds(Collection ids, ActionListener listener) throws IOException { + elasticsearchTemplate.asyncDeleteBatchIds(getIndex(), getType(), ids, listener); + } + + protected String checkIndex() throws IOException { + if (!enableAutoCreateIndex()) { + return getIndex(); + } + String index = createIndexIfNotExists(); + if (StrUtil.isBlank(index)) { + String msg = StrUtil.format("【ES】索引找不到且创建失败！", index); + throw new DefaultException(ResultCode.ERROR, msg); + } + return index; + } + + protected void checkData(Collection list) { + if (CollectionUtil.isEmpty(list)) { + String msg = StrUtil.format("【ES】写入 {} 失败！list 不能为空！", getIndex()); + throw new DefaultException(ResultCode.PARAM_ERROR, msg); + } + } + + protected void checkData(T entity) { + if (entity == null) { + String msg = StrUtil.format("【ES】写入 {} 失败！entity 不能为空！", getIndex()); + throw new DefaultException(ResultCode.PARAM_ERROR, msg); + } } } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java index 53eb9b2c..bacabff6 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java @@ -2,13 +2,15 @@ import cn.hutool.core.collection.CollectionUtil; import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; -import io.github.dunwu.javadb.elasticsearch.entity.Page; +import io.github.dunwu.javadb.elasticsearch.entity.common.PageData; +import io.github.dunwu.javadb.elasticsearch.entity.common.ScrollData; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -25,6 +27,11 @@ */ public interface EsMapper { + /** + * 获取别名 + */ + String getAlias(); + /** * 获取索引名 */ @@ -35,6 +42,16 @@ public interface EsMapper { */ String getType(); + /** + * 获取分片数 + */ + int getShard(); + + /** + * 获取副本数 + */ + int getReplica(); + /** * 获取实体类型 */ @@ -46,6 +63,12 @@ public interface EsMapper { boolean isIndexExists() throws IOException; + String createIndexIfNotExists() throws IOException; + + void deleteIndex() throws IOException; + + void updateAlias() throws IOException; + GetResponse getById(String id) throws IOException; GetResponse getById(String id, Long version) throws IOException; @@ -69,32 +92,40 @@ default Map pojoMapByIds(Collection ids) throws IOException { return map; } - Page pojoPage(SearchSourceBuilder builder) throws IOException; - long count(SearchSourceBuilder builder) throws IOException; SearchResponse query(SearchSourceBuilder builder) throws IOException; + PageData pojoPage(SearchSourceBuilder builder) throws IOException; + + ScrollData pojoPageByLastId(String lastId, int size, QueryBuilder queryBuilder) throws IOException; + + ScrollData pojoScrollBegin(SearchSourceBuilder builder) throws IOException; + + ScrollData pojoScroll(String scrollId, SearchSourceBuilder builder) throws IOException; + + boolean pojoScrollEnd(String scrollId) throws IOException; + T save(T entity) throws IOException; - boolean batchSave(Collection list) throws IOException; + boolean saveBatch(Collection list) throws IOException; - void asyncBatchSave(Collection list) throws IOException; + void asyncSaveBatch(Collection list) throws IOException; - void asyncBatchSave(Collection list, ActionListener listener) throws IOException; + void asyncSaveBatch(Collection list, ActionListener listener) throws IOException; T updateById(T entity) throws IOException; - boolean batchUpdateById(Collection list) throws IOException; + boolean updateBatchIds(Collection list) throws IOException; - void asyncBatchUpdateById(Collection list, ActionListener listener); + void asyncUpdateBatchIds(Collection list, ActionListener listener); boolean deleteById(String id) throws IOException; - boolean batchDeleteById(Collection ids) throws IOException; + boolean deleteBatchIds(Collection ids) throws IOException; - void asyncBatchDeleteById(Collection ids) throws IOException; + void asyncDeleteBatchIds(Collection ids) throws IOException; - void asyncBatchDeleteById(Collection ids, ActionListener listener) throws IOException; + void asyncDeleteBatchIds(Collection ids, ActionListener listener) throws IOException; } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java index 970b1b4d..de6d1b7c 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapper.java @@ -1,27 +1,39 @@ package io.github.dunwu.javadb.elasticsearch.mapper; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; import io.github.dunwu.javadb.elasticsearch.entity.User; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.Date; + /** - * User ES Mapper + * open_applet_consume_yyyyMMdd ES Mapper * * @author Zhang Peng * @date 2023-06-27 */ +@Slf4j @Component -public class UserEsMapper extends BaseEsMapper { +public class UserEsMapper extends BaseDynamicEsMapper { public UserEsMapper(ElasticsearchTemplate elasticsearchTemplate) { super(elasticsearchTemplate); } @Override - public String getIndex() { + public String getAlias() { return "user"; } + @Override + public String getIndex() { + String date = DateUtil.format(new Date(), DatePattern.PURE_DATE_FORMAT); + return getAlias() + "_" + date; + } + @Override public String getType() { return "_doc"; diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java index 8d43d1b8..1fadeff5 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseApplicationTests.java @@ -16,9 +16,6 @@ @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) public abstract class BaseApplicationTests { - // ---------------------------------------------------------------------------- 测试常量数据 - - // ---------------------------------------------------------------------------- protected MockMvc mockMvc; @Autowired @@ -26,7 +23,7 @@ public abstract class BaseApplicationTests { @BeforeEach public void setUp() { - mockMvc = MockMvcBuilders.webAppContextSetup(context).build(); //构造MockMvc + mockMvc = MockMvcBuilders.webAppContextSetup(context).build(); } @BeforeAll diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java new file mode 100644 index 00000000..05b95f7a --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java @@ -0,0 +1,241 @@ +package io.github.dunwu.javadb.elasticsearch; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.StrUtil; +import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; +import io.github.dunwu.javadb.elasticsearch.entity.common.PageData; +import io.github.dunwu.javadb.elasticsearch.entity.common.ScrollData; +import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.Assertions; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * ElasticsearchTemplate 测试 + * + * @author Zhang Peng + * @date 2023-11-13 + */ +@Slf4j +public abstract class BaseElasticsearchTemplateTest { + + static final int FROM = 0; + static final int SIZE = 10; + static final String TEST_ID_01 = "1"; + static final String TEST_ID_02 = "2"; + + protected ElasticsearchTemplate TEMPLATE = ElasticsearchFactory.newElasticsearchTemplate(); + + protected abstract String getAlias(); + + protected abstract String getIndex(); + + protected abstract String getType(); + + protected abstract int getShard(); + + protected abstract int getReplica(); + + protected abstract Class getEntityClass(); + + protected abstract Map getPropertiesMap(); + + protected abstract T getOneMockData(String id); + + protected abstract List getMockList(int num); + + protected void deleteIndex() throws IOException { + boolean exists = TEMPLATE.isIndexExists(getIndex()); + if (!exists) { + return; + } + TEMPLATE.deleteIndex(getIndex()); + exists = TEMPLATE.isIndexExists(getIndex()); + Assertions.assertThat(exists).isFalse(); + } + + protected void createIndex() throws IOException { + boolean exists = TEMPLATE.isIndexExists(getIndex()); + if (exists) { + return; + } + TEMPLATE.createIndex(getIndex(), getType(), getAlias(), getShard(), getReplica()); + TEMPLATE.setMapping(getIndex(), getType(), getPropertiesMap()); + exists = TEMPLATE.isIndexExists(getIndex()); + Assertions.assertThat(exists).isTrue(); + } + + protected void save() throws IOException { + String id = "1"; + T entity = getOneMockData(id); + TEMPLATE.save(getIndex(), getType(), entity); + T newEntity = TEMPLATE.pojoById(getIndex(), getType(), id, getEntityClass()); + log.info("记录：{}", JsonUtil.toString(newEntity)); + Assertions.assertThat(newEntity).isNotNull(); + } + + protected void saveBatch() throws IOException { + int total = 10000; + List> listGroup = CollectionUtil.split(getMockList(total), 1000); + for (List list : listGroup) { + TEMPLATE.saveBatch(getIndex(), getType(), list); + } + long count = TEMPLATE.count(getIndex(), getType(), new SearchSourceBuilder()); + log.info("批量更新记录数: {}", count); + Assertions.assertThat(count).isEqualTo(total); + } + + protected void getById() throws IOException { + GetResponse response = TEMPLATE.getById(getIndex(), getType(), TEST_ID_01); + Assertions.assertThat(response).isNotNull(); + log.info("记录：{}", JsonUtil.toString(response.getSourceAsMap())); + } + + protected void pojoById() throws IOException { + T entity = TEMPLATE.pojoById(getIndex(), getType(), TEST_ID_01, getEntityClass()); + Assertions.assertThat(entity).isNotNull(); + log.info("记录：{}", JsonUtil.toString(entity)); + } + + protected void pojoListByIds() throws IOException { + List ids = Arrays.asList(TEST_ID_01, TEST_ID_02); + List list = TEMPLATE.pojoListByIds(getIndex(), getType(), ids, getEntityClass()); + Assertions.assertThat(list).isNotEmpty(); + Assertions.assertThat(list.size()).isEqualTo(2); + for (T entity : list) { + log.info("记录：{}", JsonUtil.toString(entity)); + } + } + + protected void count() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + long total = TEMPLATE.count(getIndex(), getType(), searchSourceBuilder); + Assertions.assertThat(total).isNotZero(); + log.info("符合条件的记录数：{}", total); + } + + protected void query() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(FROM); + searchSourceBuilder.size(SIZE); + SearchResponse response = TEMPLATE.query(getIndex(), getType(), searchSourceBuilder); + Assertions.assertThat(response).isNotNull(); + Assertions.assertThat(response.getHits()).isNotNull(); + for (SearchHit hit : response.getHits().getHits()) { + log.info("记录：{}", hit.getSourceAsString()); + Map map = hit.getSourceAsMap(); + Assertions.assertThat(map).isNotNull(); + Assertions.assertThat(Integer.valueOf((String) map.get("docId"))).isLessThan(100); + } + } + + protected void pojoPage() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(FROM); + searchSourceBuilder.size(SIZE); + PageData page = TEMPLATE.pojoPage(getIndex(), getType(), searchSourceBuilder, getEntityClass()); + Assertions.assertThat(page).isNotNull(); + Assertions.assertThat(page.getContent()).isNotEmpty(); + for (T entity : page.getContent()) { + log.info("记录：{}", JsonUtil.toString(entity)); + } + } + + protected void pojoPageByLastId() throws IOException { + + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + + long total = TEMPLATE.count(getIndex(), getType(), queryBuilder); + ScrollData scrollData = + TEMPLATE.pojoPageByLastId(getIndex(), getType(), null, SIZE, queryBuilder, getEntityClass()); + if (scrollData == null || scrollData.getScrollId() == null) { + return; + } + Assertions.assertThat(scrollData.getTotal()).isEqualTo(total); + + long count = 0L; + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + + String scrollId = scrollData.getScrollId(); + while (CollectionUtil.isNotEmpty(scrollData.getContent())) { + scrollData = + TEMPLATE.pojoPageByLastId(getIndex(), getType(), scrollId, SIZE, queryBuilder, getEntityClass()); + if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { + break; + } + if (StrUtil.isNotBlank(scrollData.getScrollId())) { + scrollId = scrollData.getScrollId(); + } + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + } + log.info("total: {}", total); + Assertions.assertThat(count).isEqualTo(total); + } + + protected void pojoScroll() throws IOException { + + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(SIZE).query(queryBuilder).trackScores(false); + + long total = TEMPLATE.count(getIndex(), getType(), queryBuilder); + ScrollData scrollData = + TEMPLATE.pojoScrollBegin(getIndex(), getType(), searchSourceBuilder, getEntityClass()); + if (scrollData == null || scrollData.getScrollId() == null) { + return; + } + Assertions.assertThat(scrollData.getTotal()).isEqualTo(total); + + long count = 0L; + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + + String scrollId = scrollData.getScrollId(); + while (CollectionUtil.isNotEmpty(scrollData.getContent())) { + scrollData = TEMPLATE.pojoScroll(scrollId, searchSourceBuilder, getEntityClass()); + if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { + break; + } + if (StrUtil.isNotBlank(scrollData.getScrollId())) { + scrollId = scrollData.getScrollId(); + } + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + } + TEMPLATE.pojoScrollEnd(scrollId); + log.info("total: {}", total); + Assertions.assertThat(count).isEqualTo(total); + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java deleted file mode 100644 index bbddf70d..00000000 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplateTest.java +++ /dev/null @@ -1,193 +0,0 @@ -package io.github.dunwu.javadb.elasticsearch; - -import io.github.dunwu.javadb.elasticsearch.entity.Page; -import io.github.dunwu.javadb.elasticsearch.entity.User; -import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; -import org.assertj.core.api.Assertions; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -/** - * ElasticsearchTemplate 测试 - * - * @author Zhang Peng - * @date 2023-11-13 - */ -public class ElasticsearchTemplateTest { - - public static final String INDEX = "user"; - public static final String TYPE = "_doc"; - public static final String TEST_ID_01 = "1"; - public static final String TEST_ID_02 = "2"; - - private static final ElasticsearchTemplate TEMPLATE; - - static { - TEMPLATE = ElasticsearchFactory.newElasticsearchTemplate(); - } - - @Test - @DisplayName("根据ID精确查询") - public void getById() throws IOException { - GetResponse response = TEMPLATE.getById(INDEX, TYPE, TEST_ID_01); - System.out.println("记录：" + JsonUtil.toString(response.getSourceAsMap())); - } - - @Test - @DisplayName("根据ID精确查询POJO") - public void pojoById() throws IOException { - User entity = TEMPLATE.pojoById(INDEX, TYPE, TEST_ID_01, User.class); - System.out.println("记录：" + JsonUtil.toString(entity)); - } - - @Test - @DisplayName("根据ID精确批量查询POJO") - public void pojoListByIds() throws IOException { - List ids = Arrays.asList(TEST_ID_01, TEST_ID_02); - List list = TEMPLATE.pojoListByIds(INDEX, TYPE, ids, User.class); - Assertions.assertThat(list).isNotEmpty(); - Assertions.assertThat(list.size()).isEqualTo(2); - for (User entity : list) { - System.out.println("记录：" + JsonUtil.toString(entity)); - } - } - - @Test - @DisplayName("分页查询") - public void pojoPage() throws IOException { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("theme", 3)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.from(0); - searchSourceBuilder.size(10); - - Page page = TEMPLATE.pojoPage(INDEX, TYPE, searchSourceBuilder, User.class); - Assertions.assertThat(page).isNotNull(); - Assertions.assertThat(page.getContent()).isNotEmpty(); - for (User entity : page.getContent()) { - System.out.println("记录：" + JsonUtil.toString(entity)); - } - } - - @Test - @DisplayName("条件数量查询") - public void count() throws IOException { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("theme", 3)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.from(0); - searchSourceBuilder.size(10); - long total = TEMPLATE.count(INDEX, TYPE, searchSourceBuilder); - Assertions.assertThat(total).isNotZero(); - System.out.println("符合条件的总记录数：" + total); - } - - @Test - @DisplayName("条件查询") - public void query() throws IOException { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("theme", 3)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.from(0); - searchSourceBuilder.size(10); - SearchResponse response = TEMPLATE.query(INDEX, TYPE, searchSourceBuilder); - Assertions.assertThat(response).isNotNull(); - Assertions.assertThat(response.getHits()).isNotNull(); - for (SearchHit hit : response.getHits().getHits()) { - System.out.println("记录：" + hit.getSourceAsString()); - Map map = hit.getSourceAsMap(); - Assertions.assertThat(map).isNotNull(); - Assertions.assertThat(map.get("theme")).isEqualTo(3); - } - } - - @Nested - @DisplayName("写操作测试") - public class WriteTest { - - String json1 = - "{\"id\":1,\"username\":\"user1\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user1@xxx.com\"}"; - String json2 = - "{\"id\":2,\"username\":\"user2\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user2@xxx.com\"}"; - - @Test - @DisplayName("插入、更新") - public void saveAndUpdate() throws IOException, InterruptedException { - - User origin = JsonUtil.toBean(json1, User.class); - if (origin == null) { - System.err.println("反序列化失败！"); - return; - } - - TEMPLATE.save(INDEX, TYPE, origin); - TimeUnit.SECONDS.sleep(1); - User expectEntity = TEMPLATE.pojoById(INDEX, TYPE, origin.getDocId(), User.class); - Assertions.assertThat(expectEntity).isNotNull(); - - expectEntity.setAge(20); - TEMPLATE.updateById(INDEX, TYPE, expectEntity); - TimeUnit.SECONDS.sleep(18); - User expectEntity2 = - TEMPLATE.pojoById(INDEX, TYPE, origin.getDocId(), User.class); - Assertions.assertThat(expectEntity2).isNotNull(); - Assertions.assertThat(expectEntity2.getAge()).isEqualTo(20); - } - - @Test - @DisplayName("批量插入、更新") - public void batchSaveAndUpdate() throws IOException, InterruptedException { - - User origin1 = JsonUtil.toBean(json1, User.class); - if (origin1 == null) { - System.err.println("反序列化失败！"); - return; - } - - User origin2 = JsonUtil.toBean(json2, User.class); - if (origin2 == null) { - System.err.println("反序列化失败！"); - return; - } - - List list = Arrays.asList(origin1, origin2); - List ids = list.stream().map(User::getDocId).collect(Collectors.toList()); - - TEMPLATE.batchSave(INDEX, TYPE, list); - List newList = TEMPLATE.pojoListByIds(INDEX, TYPE, ids, User.class); - Assertions.assertThat(newList).isNotEmpty(); - - newList.forEach(entity -> { - entity.setAge(20); - }); - TEMPLATE.batchUpdateById(INDEX, TYPE, newList); - TimeUnit.SECONDS.sleep(1); - - List expectList = - TEMPLATE.pojoListByIds(INDEX, TYPE, ids, User.class); - Assertions.assertThat(expectList).isNotEmpty(); - for (User item : expectList) { - Assertions.assertThat(item.getAge()).isEqualTo(20); - } - } - - } - -} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java new file mode 100644 index 00000000..37242460 --- /dev/null +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java @@ -0,0 +1,108 @@ +package io.github.dunwu.javadb.elasticsearch; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.RandomUtil; +import io.github.dunwu.javadb.elasticsearch.entity.User; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * 使用 ElasticsearchTemplate 对 user 索引进行测试 + * + * @author Zhang Peng + * @date 2024-04-09 + */ +@Slf4j +public class UserElasticsearchTemplateTest extends BaseElasticsearchTemplateTest { + + @Override + protected String getAlias() { + return "user"; + } + + @Override + protected String getIndex() { + String date = DateUtil.format(new Date(), DatePattern.PURE_DATE_FORMAT); + return getAlias() + "_" + date; + } + + @Override + protected String getType() { + return "_doc"; + } + + @Override + protected int getShard() { + return 5; + } + + @Override + protected int getReplica() { + return 1; + } + + @Override + protected Class getEntityClass() { + return User.class; + } + + @Override + protected Map getPropertiesMap() { + return User.getPropertiesMap(); + } + + @Override + protected User getOneMockData(String id) { + return User.builder() + .id(id) + .name("测试数据" + id) + .age(RandomUtil.randomInt(1, 100)) + .build(); + } + + @Override + protected List getMockList(int num) { + List list = new LinkedList<>(); + for (int i = 1; i <= num; i++) { + User entity = getOneMockData(String.valueOf(i)); + list.add(entity); + } + return list; + } + + @Test + @DisplayName("索引管理测试") + public void indexTest() throws IOException { + super.deleteIndex(); + super.createIndex(); + } + + @Test + @DisplayName("写数据测试") + protected void writeTest() throws IOException { + super.save(); + super.saveBatch(); + } + + @Test + @DisplayName("读数据测试") + public void readTest() throws IOException { + super.getById(); + super.pojoById(); + super.pojoListByIds(); + super.count(); + super.query(); + super.pojoPage(); + super.pojoPageByLastId(); + super.pojoScroll(); + } + +} diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java index 65c79274..ccc7b393 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java @@ -1,27 +1,31 @@ package io.github.dunwu.javadb.elasticsearch.mapper; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.StrUtil; import io.github.dunwu.javadb.elasticsearch.BaseApplicationTests; -import io.github.dunwu.javadb.elasticsearch.entity.Page; import io.github.dunwu.javadb.elasticsearch.entity.User; +import io.github.dunwu.javadb.elasticsearch.entity.common.PageData; +import io.github.dunwu.javadb.elasticsearch.entity.common.ScrollData; import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; +import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; -import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import java.io.IOException; -import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; /** * ElasticsearchTemplate 测试 @@ -29,163 +33,441 @@ * @author Zhang Peng * @date 2023-11-13 */ +@Slf4j public class UserEsMapperTest extends BaseApplicationTests { + static final int FROM = 0; + static final int SIZE = 10; + private static final String day = "2024-04-07"; + @Autowired private UserEsMapper mapper; - public static final String TEST_ID_01 = "1"; - public static final String TEST_ID_02 = "2"; - - @Test - @DisplayName("根据ID精确查询") - public void getById() throws IOException { - GetResponse response = mapper.getById(TEST_ID_01); - System.out.println("记录：" + JsonUtil.toString(response.getSourceAsMap())); - } - @Test - @DisplayName("根据ID精确查询POJO") - public void pojoById() throws IOException { - User entity = mapper.pojoById(TEST_ID_01); - System.out.println("记录：" + JsonUtil.toString(entity)); - } + @Nested + @DisplayName("删除索引测试") + class DeleteIndexTest { - @Test - @DisplayName("根据ID精确批量查询POJO") - public void pojoListByIds() throws IOException { - List ids = Arrays.asList(TEST_ID_01, TEST_ID_02); - List list = mapper.pojoListByIds(ids); - Assertions.assertThat(list).isNotEmpty(); - Assertions.assertThat(list.size()).isEqualTo(2); - for (User entity : list) { - System.out.println("记录：" + JsonUtil.toString(entity)); + @Test + @DisplayName("删除当天索引") + public void deleteIndex() throws IOException { + String index = mapper.getIndex(); + boolean indexExists = mapper.isIndexExists(); + if (!indexExists) { + log.info("【ES】{} 不存在！", index); + return; + } + mapper.deleteIndex(); + indexExists = mapper.isIndexExists(); + Assertions.assertThat(indexExists).isFalse(); } - } - @Test - @DisplayName("分页查询") - public void pojoPage() throws IOException { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("id", 1)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.from(0); - searchSourceBuilder.size(10); - - Page page = mapper.pojoPage(searchSourceBuilder); - Assertions.assertThat(page).isNotNull(); - Assertions.assertThat(page.getContent()).isNotEmpty(); - for (User entity : page.getContent()) { - System.out.println("记录：" + JsonUtil.toString(entity)); + @Test + @DisplayName("根据日期删除索引") + public void deleteIndexInDay() throws IOException { + String index = mapper.getIndex(day); + boolean indexExists = mapper.isIndexExistsInDay(day); + if (!indexExists) { + log.info("【ES】{} 不存在！", index); + return; + } + mapper.deleteIndexInDay(day); + indexExists = mapper.isIndexExistsInDay(day); + Assertions.assertThat(indexExists).isFalse(); } - } - @Test - @DisplayName("条件数量查询") - public void count() throws IOException { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("id", 1)); - // boolQueryBuilder.must(QueryBuilders.rangeQuery("age") - // .from(18) - // .to(25) - // .includeLower(true) - // .includeUpper(true)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.from(0); - searchSourceBuilder.size(10); - long total = mapper.count(searchSourceBuilder); - Assertions.assertThat(total).isNotZero(); - System.out.println("符合条件的总记录数：" + total); } - @Test - @DisplayName("条件查询") - public void query() throws IOException { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery("id", 1)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.from(0); - searchSourceBuilder.size(10); - SearchResponse response = mapper.query(searchSourceBuilder); - Assertions.assertThat(response).isNotNull(); - Assertions.assertThat(response.getHits()).isNotNull(); - for (SearchHit hit : response.getHits().getHits()) { - System.out.println("记录：" + hit.getSourceAsString()); - Map map = hit.getSourceAsMap(); - Assertions.assertThat(map).isNotNull(); + @Nested + @DisplayName("创建索引测试") + class CreateIndexTest { + + @Test + @DisplayName("创建当天索引") + public void createIndex() throws IOException { + + String index = mapper.getIndex(); + boolean indexExists = mapper.isIndexExists(); + if (indexExists) { + log.info("【ES】{} 已存在！", index); + return; + } + + mapper.createIndexIfNotExists(); + indexExists = mapper.isIndexExists(); + Assertions.assertThat(indexExists).isTrue(); + } + + @Test + @DisplayName("根据日期创建索引") + public void createIndexInDay() throws IOException { + + String index = mapper.getIndex(day); + boolean indexExists = mapper.isIndexExistsInDay(day); + if (indexExists) { + log.info("【ES】{} 已存在！", index); + return; + } + + mapper.createIndexInDay(day); + indexExists = mapper.isIndexExistsInDay(day); + Assertions.assertThat(indexExists).isTrue(); } + } @Nested @DisplayName("写操作测试") - public class WriteTest { + class WriteTest { + + @Test + @DisplayName("保存当天数据") + public void save() throws IOException { + String id = "1"; + User entity = getOneMockData(id); + mapper.save(entity); + User newEntity = mapper.pojoById(id); + log.info("entity: {}", JsonUtil.toString(newEntity)); + Assertions.assertThat(newEntity).isNotNull(); + } + + @Test + @DisplayName("保存指定日期数据") + public void saveInDay() throws IOException { + String id = "1"; + User entity = getOneMockData(id); + mapper.saveInDay(day, entity); + User newEntity = mapper.pojoByIdInDay(day, id); + log.info("entity: {}", JsonUtil.toString(newEntity)); + Assertions.assertThat(newEntity).isNotNull(); + } + + @Test + @DisplayName("批量保存当天数据") + public void batchSave() throws IOException, InterruptedException { + int total = 10000; + List> listGroup = CollectionUtil.split(getMockList(total), 1000); + for (List list : listGroup) { + mapper.asyncSaveBatch(list); + } + TimeUnit.SECONDS.sleep(20); + long count = mapper.count(new SearchSourceBuilder()); + log.info("count: {}", count); + Assertions.assertThat(count).isEqualTo(10 * 1000); + } + + @Test + @DisplayName("批量保存指定日期数据") + public void batchSaveInDay() throws IOException, InterruptedException { + int total = 10000; + List> listGroup = CollectionUtil.split(getMockList(total), 1000); + for (List list : listGroup) { + mapper.asyncSaveBatchInDay(day, list); + } + TimeUnit.SECONDS.sleep(20); + long count = mapper.countInDay(day, new SearchSourceBuilder()); + log.info("count: {}", count); + Assertions.assertThat(count).isEqualTo(10 * 1000); + } + + } + + @Nested + @DisplayName("读操作测试") + class ReadTest { + + @Test + @DisplayName("根据ID查找当日数据") + public void pojoById() throws IOException { + String id = "1"; + User newEntity = mapper.pojoById(id); + log.info("entity: {}", JsonUtil.toString(newEntity)); + Assertions.assertThat(newEntity).isNotNull(); + } + + @Test + @DisplayName("根据ID查找指定日期数据") + public void pojoByIdInDay() throws IOException { + String id = "1"; + User newEntity = mapper.pojoByIdInDay(day, id); + log.info("entity: {}", JsonUtil.toString(newEntity)); + Assertions.assertThat(newEntity).isNotNull(); + } - String json1 = - "{\"id\":1,\"username\":\"user1\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user1@xxx.com\"}"; - String json2 = - "{\"id\":2,\"username\":\"user2\",\"password\":\"xxxxxx\",\"age\":18,\"email\":\"user2@xxx.com\"}"; + @Test + @DisplayName("获取匹配条件的记录数") + public void count() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + long total = mapper.count(searchSourceBuilder); + Assertions.assertThat(total).isNotZero(); + log.info("符合条件的记录数：{}", total); + } @Test - @DisplayName("插入、更新") - public void saveAndUpdate() throws IOException, InterruptedException { + @DisplayName("获取匹配条件的指定日期记录数") + public void countInDay() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + long total = mapper.countInDay(day, searchSourceBuilder); + Assertions.assertThat(total).isNotZero(); + log.info("符合条件的记录数：{}", total); + } - User origin = JsonUtil.toBean(json1, User.class); - if (origin == null) { - System.err.println("反序列化失败！"); + @Test + @DisplayName("获取匹配条件的记录") + public void query() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(FROM); + searchSourceBuilder.size(SIZE); + SearchResponse response = mapper.query(searchSourceBuilder); + Assertions.assertThat(response).isNotNull(); + Assertions.assertThat(response.getHits()).isNotNull(); + for (SearchHit hit : response.getHits().getHits()) { + log.info("记录：{}", hit.getSourceAsString()); + Map map = hit.getSourceAsMap(); + Assertions.assertThat(map).isNotNull(); + Assertions.assertThat(Integer.valueOf((String) map.get("docId"))).isLessThan(100); + } + } + + @Test + @DisplayName("获取匹配条件的指定日期记录") + public void queryInDay() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(FROM); + searchSourceBuilder.size(SIZE); + SearchResponse response = mapper.queryInDay(day, searchSourceBuilder); + Assertions.assertThat(response).isNotNull(); + Assertions.assertThat(response.getHits()).isNotNull(); + for (SearchHit hit : response.getHits().getHits()) { + log.info("记录：{}", hit.getSourceAsString()); + Map map = hit.getSourceAsMap(); + Assertions.assertThat(map).isNotNull(); + Assertions.assertThat(Integer.valueOf((String) map.get("docId"))).isLessThan(100); + } + } + + @Test + @DisplayName("from + size 分页查询当日数据") + public void pojoPage() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(FROM); + searchSourceBuilder.size(SIZE); + PageData page = mapper.pojoPage(searchSourceBuilder); + Assertions.assertThat(page).isNotNull(); + Assertions.assertThat(page.getContent()).isNotEmpty(); + for (User entity : page.getContent()) { + log.info("记录：{}", JsonUtil.toString(entity)); + } + } + + @Test + @DisplayName("from + size 分页查询指定日期数据") + public void pojoPageInDay() throws IOException { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.from(FROM); + searchSourceBuilder.size(SIZE); + PageData page = mapper.pojoPageInDay(day, searchSourceBuilder); + Assertions.assertThat(page).isNotNull(); + Assertions.assertThat(page.getContent()).isNotEmpty(); + for (User entity : page.getContent()) { + log.info("记录：{}", JsonUtil.toString(entity)); + } + } + + @Test + @DisplayName("search after 分页查询当日数据") + protected void pojoPageByLastId() throws IOException { + + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + long total = mapper.count(searchSourceBuilder); + ScrollData scrollData = mapper.pojoPageByLastId(null, SIZE, queryBuilder); + if (scrollData == null || scrollData.getScrollId() == null) { return; } + Assertions.assertThat(scrollData.getTotal()).isEqualTo(total); - mapper.save(origin); - TimeUnit.SECONDS.sleep(1); - User expectEntity = mapper.pojoById(origin.getDocId()); - Assertions.assertThat(expectEntity).isNotNull(); + long count = 0L; + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); - expectEntity.setAge(20); - mapper.updateById(expectEntity); - TimeUnit.SECONDS.sleep(1); - User expectEntity2 = mapper.pojoById(origin.getDocId()); - Assertions.assertThat(expectEntity2).isNotNull(); - Assertions.assertThat(expectEntity2.getAge()).isEqualTo(20); + String scrollId = scrollData.getScrollId(); + while (CollectionUtil.isNotEmpty(scrollData.getContent())) { + scrollData = mapper.pojoPageByLastId(scrollId, SIZE, queryBuilder); + if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { + break; + } + if (StrUtil.isNotBlank(scrollData.getScrollId())) { + scrollId = scrollData.getScrollId(); + } + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + } + log.info("total: {}", total); + Assertions.assertThat(count).isEqualTo(total); } @Test - @DisplayName("批量插入、更新") - public void batchSaveAndUpdate() throws IOException, InterruptedException { + @DisplayName("search after 分页查询指定日期数据") + protected void pojoPageByLastIdInDay() throws IOException { - User origin1 = JsonUtil.toBean(json1, User.class); - if (origin1 == null) { - System.err.println("反序列化失败！"); + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + long total = mapper.count(searchSourceBuilder); + ScrollData scrollData = mapper.pojoPageByLastIdInDay(day, null, SIZE, queryBuilder); + if (scrollData == null || scrollData.getScrollId() == null) { return; } + Assertions.assertThat(scrollData.getTotal()).isEqualTo(total); + + long count = 0L; + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + + String scrollId = scrollData.getScrollId(); + while (CollectionUtil.isNotEmpty(scrollData.getContent())) { + scrollData = mapper.pojoPageByLastIdInDay(day, scrollId, SIZE, queryBuilder); + if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { + break; + } + if (StrUtil.isNotBlank(scrollData.getScrollId())) { + scrollId = scrollData.getScrollId(); + } + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + } + log.info("total: {}", total); + Assertions.assertThat(count).isEqualTo(total); + } + + @Test + @DisplayName("滚动翻页当日数据") + public void pojoScroll() throws IOException { + + final int size = 100; - User origin2 = JsonUtil.toBean(json2, User.class); - if (origin2 == null) { - System.err.println("反序列化失败！"); + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(size).sort("docId", SortOrder.ASC).query(queryBuilder).trackScores(false); + + long total = mapper.count(searchSourceBuilder); + log.info("total: {}", total); + + ScrollData scrollData = mapper.pojoScrollBegin(searchSourceBuilder); + if (scrollData == null || scrollData.getScrollId() == null) { return; } - List list = Arrays.asList(origin1, origin2); - List ids = list.stream().map(User::getDocId).collect(Collectors.toList()); + long count = 0L; + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + Assertions.assertThat(scrollData.getTotal()).isEqualTo(total); + count += scrollData.getContent().size(); - mapper.batchSave(list); - List newList = mapper.pojoListByIds(ids); - Assertions.assertThat(newList).isNotEmpty(); + String scrollId = scrollData.getScrollId(); + while (CollectionUtil.isNotEmpty(scrollData.getContent())) { + scrollData = mapper.pojoScroll(scrollId, searchSourceBuilder); + if (scrollData != null && StrUtil.isNotBlank(scrollData.getScrollId())) { + scrollId = scrollData.getScrollId(); + } + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); + } + mapper.pojoScrollEnd(scrollId); + Assertions.assertThat(count).isEqualTo(total); + } - newList.forEach(entity -> { - entity.setAge(20); + @Test + @DisplayName("滚动翻页指定日期数据") + public void pojoScrollInDay() throws IOException { + + final int size = 100; + + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(size).sort("docId", SortOrder.ASC).query(queryBuilder).trackScores(false); + + long total = mapper.countInDay(day, searchSourceBuilder); + log.info("total: {}", total); + + ScrollData scrollData = mapper.pojoScrollBeginInDay(day, searchSourceBuilder); + if (scrollData == null || scrollData.getScrollId() == null) { + return; + } + + long count = 0L; + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); }); - mapper.batchUpdateById(newList); - TimeUnit.SECONDS.sleep(1); + Assertions.assertThat(scrollData.getTotal()).isEqualTo(total); + count += scrollData.getContent().size(); - List expectList = mapper.pojoListByIds(ids); - Assertions.assertThat(expectList).isNotEmpty(); - for (User item : expectList) { - Assertions.assertThat(item.getAge()).isEqualTo(20); + String scrollId = scrollData.getScrollId(); + while (CollectionUtil.isNotEmpty(scrollData.getContent())) { + scrollData = mapper.pojoScroll(scrollId, searchSourceBuilder); + if (scrollData != null && StrUtil.isNotBlank(scrollData.getScrollId())) { + scrollId = scrollData.getScrollId(); + } + scrollData.getContent().forEach(data -> { + log.info("docId: {}", data.getDocId()); + }); + count += scrollData.getContent().size(); } + mapper.pojoScrollEnd(scrollId); + Assertions.assertThat(count).isEqualTo(total); } } + public User getOneMockData(String id) { + return User.builder() + .id(id) + .name("测试数据" + id) + .age(RandomUtil.randomInt(1, 100)) + .build(); + } + + public List getMockList(int num) { + List list = new LinkedList<>(); + for (int i = 1; i <= num; i++) { + User entity = getOneMockData(String.valueOf(i)); + list.add(entity); + } + return list; + } + } From bd6b17cda7f30760fbd5a7e6acc931bd6841d25b Mon Sep 17 00:00:00 2001 From: dunwu Date: Mon, 15 Apr 2024 07:43:19 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20elasticsearch=206.x=20=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch/ElasticsearchTemplate.java | 41 +- .../mapper/BaseDynamicEsMapper.java | 281 ++++++++---- .../elasticsearch/mapper/BaseEsMapper.java | 411 +++++++++++++----- .../javadb/elasticsearch/mapper/EsMapper.java | 71 +-- .../BaseElasticsearchTemplateTest.java | 57 ++- .../UserElasticsearchTemplateTest.java | 8 + .../mapper/UserEsMapperTest.java | 43 +- 7 files changed, 646 insertions(+), 266 deletions(-) diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java index 730f9f17..749956e8 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -40,8 +41,10 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.GetAliasesResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -65,6 +68,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.stream.Collectors; @@ -194,6 +198,20 @@ public boolean isIndexExists(String index) throws IOException { return client.indices().exists(request.indices(index), RequestOptions.DEFAULT); } + public Set getIndexSet(String alias) throws IOException { + GetAliasesRequest request = new GetAliasesRequest(alias); + GetAliasesResponse response = client.indices().getAlias(request, RequestOptions.DEFAULT); + if (StrUtil.isNotBlank(response.getError())) { + String msg = StrUtil.format("【ES】获取索引失败！alias: {}, error: {}", alias, response.getError()); + throw new ElasticsearchException(msg); + } + if (response.getException() != null) { + throw response.getException(); + } + Map> aliasMap = response.getAliases(); + return aliasMap.keySet(); + } + public void setMapping(String index, String type, Map propertiesMap) throws IOException { if (MapUtil.isEmpty(propertiesMap)) { @@ -459,7 +477,7 @@ public List pojoListByIds(String index, String type, Collection i throws IOException { if (CollectionUtil.isEmpty(ids)) { - return null; + return new ArrayList<>(0); } MultiGetRequest request = new MultiGetRequest(); @@ -471,7 +489,7 @@ public List pojoListByIds(String index, String type, Collection i if (null == multiGetResponse || multiGetResponse.getResponses() == null || multiGetResponse.getResponses().length <= 0) { - return new ArrayList<>(); + return new ArrayList<>(0); } List list = new ArrayList<>(); @@ -491,7 +509,7 @@ public List pojoListByIds(String index, String type, Collection i public long count(String index, String type, SearchSourceBuilder builder) throws IOException { SearchResponse response = query(index, type, builder); if (response == null || response.status() != RestStatus.OK) { - return -1L; + return 0L; } SearchHits searchHits = response.getHits(); return searchHits.getTotalHits(); @@ -550,15 +568,15 @@ public PageData pojoPage(String index, String type, int from, int size, Q /** * search after 分页 */ - public ScrollData pojoPageByLastId(String index, String type, String lastId, int size, + public ScrollData pojoPageByScrollId(String index, String type, String scrollId, int size, QueryBuilder queryBuilder, Class clazz) throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(size); searchSourceBuilder.sort(BaseEsEntity.DOC_ID, SortOrder.ASC); - if (StrUtil.isNotBlank(lastId)) { + if (StrUtil.isNotBlank(scrollId)) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(queryBuilder).must(QueryBuilders.rangeQuery(BaseEsEntity.DOC_ID).gt(lastId)); + boolQueryBuilder.must(queryBuilder).must(QueryBuilders.rangeQuery(BaseEsEntity.DOC_ID).gt(scrollId)); searchSourceBuilder.query(boolQueryBuilder); } else { searchSourceBuilder.query(queryBuilder); @@ -639,9 +657,7 @@ public boolean pojoScrollEnd(String scrollId) throws IOException { } public T toPojo(GetResponse response, Class clazz) { - if (null == response) { - return null; - } else if (StrUtil.isBlank(response.getSourceAsString())) { + if (null == response || StrUtil.isBlank(response.getSourceAsString())) { return null; } else { return JsonUtil.toBean(response.getSourceAsString(), clazz); @@ -649,15 +665,12 @@ public T toPojo(GetResponse response, Class clazz) { } public List toPojoList(SearchResponse response, Class clazz) { - if (response == null || response.status() != RestStatus.OK) { - return new ArrayList<>(); + return new ArrayList<>(0); } - if (ArrayUtil.isEmpty(response.getHits().getHits())) { - return new ArrayList<>(); + return new ArrayList<>(0); } - return Stream.of(response.getHits().getHits()) .map(hit -> JsonUtil.toBean(hit.getSourceAsString(), clazz)) .collect(Collectors.toList()); diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java index 710594df..c75c6cc6 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseDynamicEsMapper.java @@ -1,10 +1,11 @@ package io.github.dunwu.javadb.elasticsearch.mapper; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.DatePattern; import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateUtil; -import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; import io.github.dunwu.javadb.elasticsearch.constant.ResultCode; import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; @@ -20,9 +21,9 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Map; /** * 动态 ES Mapper 基础类（以时间为维度动态创建、删除 index），用于数据量特别大，需要按照日期分片的索引。 @@ -39,6 +40,11 @@ public BaseDynamicEsMapper(ElasticsearchTemplate elasticsearchTemplate) { super(elasticsearchTemplate); } + @Override + public boolean enableAutoCreateIndex() { + return true; + } + // ==================================================================== // 索引管理操作 // ==================================================================== @@ -64,68 +70,139 @@ public String getIndex(String day) { return alias + "_" + formatDate; } - public boolean isIndexExistsInDay(String day) throws IOException { - return elasticsearchTemplate.isIndexExists(getIndex(day)); + public boolean isIndexExistsInDay(String day) { + if (StrUtil.isBlank(day)) { + return false; + } + String index = getIndex(day); + try { + return elasticsearchTemplate.isIndexExists(getIndex(day)); + } catch (Exception e) { + log.error("【ES】判断索引是否存在异常！index: {}", index, e); + return false; + } } - public String createIndexInDay(String day) throws IOException, DefaultException { + public String createIndexIfNotExistsInDay(String day) { String index = getIndex(day); - boolean indexExists = isIndexExistsInDay(day); - if (indexExists) { - return index; - } - elasticsearchTemplate.createIndex(index, getType(), getAlias(), getShard(), getReplica()); - Map map = getPropertiesMap(); - if (MapUtil.isNotEmpty(map)) { - elasticsearchTemplate.setMapping(index, getType(), map); - } - return index; + String type = getType(); + String alias = getAlias(); + int shard = getShard(); + int replica = getReplica(); + return createIndex(index, type, alias, shard, replica); } - public void deleteIndexInDay(String day) throws IOException { - elasticsearchTemplate.deleteIndex(getIndex(day)); + public void deleteIndexInDay(String day) { + String index = getIndex(day); + try { + log.info("【ES】删除索引成功！index: {}", index); + elasticsearchTemplate.deleteIndex(index); + } catch (Exception e) { + log.error("【ES】删除索引异常！index: {}", index, e); + } } - public void updateAliasInDay(String day) throws IOException { - elasticsearchTemplate.updateAlias(getIndex(day), getAlias()); + public void updateAliasInDay(String day) { + String index = getIndex(day); + String alias = getAlias(); + try { + log.info("【ES】更新别名成功！alias: {} -> index: {}", alias, index); + elasticsearchTemplate.updateAlias(index, alias); + } catch (IOException e) { + log.error("【ES】更新别名异常！alias: {} -> index: {}", alias, index, e); + } } // ==================================================================== // CRUD 操作 // ==================================================================== - public GetResponse getByIdInDay(String day, String id) throws IOException { - return elasticsearchTemplate.getById(getIndex(day), getType(), id, null); + public GetResponse getByIdInDay(String day, String id) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.getById(index, type, id, null); + } catch (IOException e) { + log.error("【ES】根据ID查询异常！index: {}, type: {}, id: {}", index, type, id, e); + return null; + } } - public T pojoByIdInDay(String day, String id) throws IOException { - return elasticsearchTemplate.pojoById(getIndex(day), getType(), id, null, getEntityClass()); + public T pojoByIdInDay(String day, String id) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.pojoById(index, type, id, null, getEntityClass()); + } catch (IOException e) { + log.error("【ES】根据ID查询POJO异常！index: {}, type: {}, id: {}", index, type, id, e); + return null; + } } - public List pojoListByIdsInDay(String day, Collection ids) throws IOException { - return elasticsearchTemplate.pojoListByIds(getIndex(day), getType(), ids, getEntityClass()); + public List pojoListByIdsInDay(String day, Collection ids) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.pojoListByIds(index, type, ids, getEntityClass()); + } catch (IOException e) { + log.error("【ES】根据ID查询POJO列表异常！index: {}, type: {}, ids: {}", index, type, ids, e); + return new ArrayList<>(0); + } } - public long countInDay(String day, SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.count(getIndex(day), getType(), builder); + public long countInDay(String day, SearchSourceBuilder builder) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.count(index, type, builder); + } catch (IOException e) { + log.error("【ES】获取匹配记录数异常！index: {}, type: {}", index, type, e); + return 0L; + } } - public SearchResponse queryInDay(String day, SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.query(getIndex(day), getType(), builder); + public SearchResponse queryInDay(String day, SearchSourceBuilder builder) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.query(index, type, builder); + } catch (IOException e) { + log.error("【ES】条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } - public PageData pojoPageInDay(String day, SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.pojoPage(getIndex(day), getType(), builder, getEntityClass()); + public PageData pojoPageInDay(String day, SearchSourceBuilder builder) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.pojoPage(index, type, builder, getEntityClass()); + } catch (IOException e) { + log.error("【ES】from + size 分页条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } - public ScrollData pojoPageByLastIdInDay(String day, String lastId, int size, QueryBuilder queryBuilder) - throws IOException { - return elasticsearchTemplate.pojoPageByLastId(getIndex(day), getType(), lastId, size, - queryBuilder, getEntityClass()); + public ScrollData pojoPageByLastIdInDay(String day, String scrollId, int size, QueryBuilder queryBuilder) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.pojoPageByScrollId(index, type, scrollId, size, queryBuilder, getEntityClass()); + } catch (IOException e) { + log.error("【ES】search after 分页条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } - public ScrollData pojoScrollBeginInDay(String day, SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.pojoScrollBegin(getIndex(day), getType(), builder, getEntityClass()); + public ScrollData pojoScrollBeginInDay(String day, SearchSourceBuilder builder) { + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.pojoScrollBegin(index, type, builder, getEntityClass()); + } catch (IOException e) { + log.error("【ES】开启滚动分页条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } /** @@ -135,11 +212,20 @@ public ScrollData pojoScrollBeginInDay(String day, SearchSourceBuilder builde * @param entity 待更新的数据 * @return / */ - public boolean saveInDay(String day, T entity) throws IOException, DefaultException { - String index = checkIndex(day); - checkData(entity); - elasticsearchTemplate.save(index, getType(), entity); - return true; + public T saveInDay(String day, T entity) { + if (StrUtil.isBlank(day) || entity == null) { + return null; + } + String index = getIndex(day); + String type = getType(); + try { + checkIndex(day); + checkData(entity); + return elasticsearchTemplate.save(index, getType(), entity); + } catch (IOException e) { + log.error("【ES】添加数据异常！index: {}, type: {}, entity: {}", index, type, JSONUtil.toJsonStr(entity), e); + return null; + } } /** @@ -149,59 +235,94 @@ public boolean saveInDay(String day, T entity) throws IOException, DefaultExcept * @param list 待更新的数据 * @return / */ - public boolean saveBatchInDay(String day, Collection list) throws IOException, DefaultException { - String index = checkIndex(day); - checkData(list); - elasticsearchTemplate.saveBatch(index, getType(), list); - return true; + public boolean saveBatchInDay(String day, Collection list) { + if (StrUtil.isBlank(day) || CollectionUtil.isEmpty(list)) { + return false; + } + String index = getIndex(day); + String type = getType(); + try { + checkIndex(day); + checkData(list); + return elasticsearchTemplate.saveBatch(index, type, list); + } catch (IOException e) { + log.error("【ES】批量添加数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + return false; + } + } + + public void asyncSaveBatchInDay(String day, Collection list) { + asyncSaveBatchInDay(day, list, DEFAULT_BULK_LISTENER); } - public void asyncSaveBatchInDay(String day, Collection list) throws IOException { - String index = checkIndex(day); - checkData(list); - ActionListener listener = new ActionListener() { - @Override - public void onResponse(BulkResponse response) { - if (response != null && !response.hasFailures()) { - String msg = StrUtil.format("【ES】按日期异步批量保存 {} 成功！", index); - log.info(msg); - } else { - String msg = StrUtil.format("【ES】按日期异步批量保存 {} 失败！", index); - log.warn(msg); - } - } + public void asyncSaveBatchInDay(String day, Collection list, ActionListener listener) { + if (StrUtil.isBlank(day) || CollectionUtil.isEmpty(list)) { + return; + } + String index = getIndex(day); + String type = getType(); + try { + checkIndex(day); + checkData(list); + elasticsearchTemplate.asyncSaveBatch(index, type, list, listener); + } catch (Exception e) { + log.error("【ES】异步批量添加数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + } + } - @Override - public void onFailure(Exception e) { - String msg = StrUtil.format("【ES】按日期异步批量保存 {} 异常！", index); - log.error(msg, e); - } - }; - asyncSaveBatchInDay(day, list, listener); + public void asyncUpdateBatchIdsInDay(String day, Collection list) { + asyncUpdateBatchIdsInDay(day, list, DEFAULT_BULK_LISTENER); } - public void asyncSaveBatchInDay(String day, Collection list, ActionListener listener) - throws IOException { - String index = checkIndex(day); - checkData(list); - elasticsearchTemplate.asyncSaveBatch(getIndex(day), getType(), list, listener); + public void asyncUpdateBatchIdsInDay(String day, Collection list, ActionListener listener) { + if (StrUtil.isBlank(day) || CollectionUtil.isEmpty(list)) { + return; + } + String index = getIndex(day); + String type = getType(); + try { + checkData(list); + elasticsearchTemplate.asyncUpdateBatchIds(index, type, list, listener); + } catch (Exception e) { + log.error("【ES】异步批量更新数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + } } - public boolean deleteByIdInDay(String day, String id) throws IOException { - return elasticsearchTemplate.deleteById(getIndex(day), getType(), id); + public boolean deleteByIdInDay(String day, String id) { + if (StrUtil.isBlank(day) || StrUtil.isBlank(id)) { + return false; + } + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.deleteById(index, type, id); + } catch (IOException e) { + log.error("【ES】根据ID删除数据异常！index: {}, type: {}, id: {}", index, type, id, e); + return false; + } } - public boolean deleteBatchIdsInDay(String day, Collection ids) throws IOException { - return elasticsearchTemplate.deleteBatchIds(getIndex(day), getType(), ids); + public boolean deleteBatchIdsInDay(String day, Collection ids) { + if (StrUtil.isBlank(day) || CollectionUtil.isEmpty(ids)) { + return false; + } + String index = getIndex(day); + String type = getType(); + try { + return elasticsearchTemplate.deleteBatchIds(index, type, ids); + } catch (IOException e) { + log.error("【ES】根据ID批量删除数据异常！index: {}, type: {}, ids: {}", index, type, ids, e); + return false; + } } - protected String checkIndex(String day) throws IOException { + protected String checkIndex(String day) { if (!enableAutoCreateIndex()) { return getIndex(day); } - String index = createIndexInDay(day); + String index = createIndexIfNotExistsInDay(day); if (StrUtil.isBlank(index)) { - String msg = StrUtil.format("【ES】按日期批量保存 {} 失败！索引找不到且创建失败！", index); + String msg = StrUtil.format("【ES】索引 {}_{} 找不到且创建失败！", getAlias(), day); throw new DefaultException(ResultCode.ERROR, msg); } return index; diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java index 3232f9e9..b125bea9 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/BaseEsMapper.java @@ -4,6 +4,7 @@ import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.ReflectUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; import io.github.dunwu.javadb.elasticsearch.ElasticsearchTemplate; import io.github.dunwu.javadb.elasticsearch.constant.ResultCode; import io.github.dunwu.javadb.elasticsearch.entity.BaseEsEntity; @@ -20,12 +21,14 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import java.io.IOException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * ES Mapper 基础类 @@ -52,13 +55,6 @@ public int getReplica() { return 1; } - /** - * 如果开启，添加 ES 数据时，如果索引不存在，会自动创建索引 - */ - public boolean enableAutoCreateIndex() { - return true; - } - @Override public RestHighLevelClient getClient() { if (elasticsearchTemplate == null) { @@ -77,14 +73,13 @@ public synchronized BulkProcessor getBulkProcessor() { @SuppressWarnings("unchecked") public Map getPropertiesMap() { - Class clazz = getEntityClass(); Method method; try { method = clazz.getMethod("getPropertiesMap"); } catch (NoSuchMethodException e) { - String msg = StrUtil.format("【ES】检查并创建 {} 索引失败！day 不能为空！", getAlias()); - throw new DefaultException(e, ResultCode.ERROR, msg); + log.error("【ES】{} 中不存在 getPropertiesMap 方法！", clazz.getCanonicalName()); + return new HashMap<>(0); } Object result = ReflectUtil.invokeStatic(method); @@ -99,33 +94,80 @@ public Map getPropertiesMap() { // ==================================================================== @Override - public boolean isIndexExists() throws IOException { - return elasticsearchTemplate.isIndexExists(getIndex()); + public boolean isIndexExists() { + String index = getIndex(); + try { + return elasticsearchTemplate.isIndexExists(index); + } catch (Exception e) { + log.error("【ES】判断索引是否存在异常！index: {}", index, e); + return false; + } } @Override - public String createIndexIfNotExists() throws IOException { + public String createIndexIfNotExists() { String index = getIndex(); - boolean exists = elasticsearchTemplate.isIndexExists(index); - if (exists) { + String type = getType(); + String alias = getAlias(); + int shard = getShard(); + int replica = getReplica(); + return createIndex(index, type, alias, shard, replica); + } + + protected String createIndex(String index, String type, String alias, int shard, int replica) { + try { + if (elasticsearchTemplate.isIndexExists(index)) { + return index; + } + elasticsearchTemplate.createIndex(index, type, alias, shard, replica); + log.info("【ES】创建索引成功！index: {}, type: {}, alias: {}, shard: {}, replica: {}", + index, type, alias, shard, replica); + Map propertiesMap = getPropertiesMap(); + if (MapUtil.isNotEmpty(propertiesMap)) { + elasticsearchTemplate.setMapping(index, type, propertiesMap); + log.error("【ES】设置索引 mapping 成功！index: {}, type: {}, propertiesMap: {}", + index, type, JSONUtil.toJsonStr(propertiesMap)); + } return index; + } catch (Exception e) { + log.error("【ES】创建索引异常！index: {}, type: {}, alias: {}, shard: {}, replica: {}", + index, type, alias, shard, replica, e); + return null; } - elasticsearchTemplate.createIndex(index, getType(), getAlias(), getShard(), getReplica()); - Map propertiesMap = getPropertiesMap(); - if (MapUtil.isNotEmpty(propertiesMap)) { - elasticsearchTemplate.setMapping(index, getType(), propertiesMap); + } + + @Override + public void deleteIndex() { + String index = getIndex(); + try { + log.info("【ES】删除索引成功！index: {}", index); + elasticsearchTemplate.deleteIndex(index); + } catch (Exception e) { + log.error("【ES】删除索引异常！index: {}", index, e); } - return index; } @Override - public void deleteIndex() throws IOException { - elasticsearchTemplate.deleteIndex(getIndex()); + public void updateAlias() { + String index = getIndex(); + String alias = getAlias(); + try { + log.info("【ES】更新别名成功！alias: {} -> index: {}", alias, index); + elasticsearchTemplate.updateAlias(index, alias); + } catch (Exception e) { + log.error("【ES】更新别名异常！alias: {} -> index: {}", alias, index, e); + } } @Override - public void updateAlias() throws IOException { - elasticsearchTemplate.updateAlias(getIndex(), getAlias()); + public Set getIndexSet() { + String alias = getAlias(); + try { + return elasticsearchTemplate.getIndexSet(alias); + } catch (Exception e) { + log.error("【ES】获取别名的所有索引异常！alias: {}", alias, e); + return new HashSet<>(0); + } } // ==================================================================== @@ -133,172 +175,295 @@ public void updateAlias() throws IOException { // ==================================================================== @Override - public GetResponse getById(String id) throws IOException { + public GetResponse getById(String id) { return getById(id, null); } @Override - public GetResponse getById(String id, Long version) throws IOException { - return elasticsearchTemplate.getById(getIndex(), getType(), id, version); + public GetResponse getById(String id, Long version) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.getById(index, type, id, version); + } catch (Exception e) { + log.error("【ES】根据ID查询异常！index: {}, type: {}, id: {}, version: {}", index, type, id, version, e); + return null; + } } @Override - public T pojoById(String id) throws IOException { + public T pojoById(String id) { return pojoById(id, null); } @Override - public T pojoById(String id, Long version) throws IOException { - return elasticsearchTemplate.pojoById(getIndex(), getType(), id, version, getEntityClass()); + public T pojoById(String id, Long version) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.pojoById(index, type, id, version, getEntityClass()); + } catch (Exception e) { + log.error("【ES】根据ID查询POJO异常！index: {}, type: {}, id: {}, version: {}", index, type, id, version, e); + return null; + } } @Override - public List pojoListByIds(Collection ids) throws IOException { - return elasticsearchTemplate.pojoListByIds(getIndex(), getType(), ids, getEntityClass()); + public List pojoListByIds(Collection ids) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.pojoListByIds(index, type, ids, getEntityClass()); + } catch (Exception e) { + log.error("【ES】根据ID查询POJO列表异常！index: {}, type: {}, ids: {}", index, type, ids, e); + return new ArrayList<>(0); + } } @Override - public long count(SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.count(getIndex(), getType(), builder); + public long count(SearchSourceBuilder builder) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.count(index, type, builder); + } catch (Exception e) { + log.error("【ES】获取匹配记录数异常！index: {}, type: {}", index, type, e); + return 0L; + } } @Override - public SearchResponse query(SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.query(getIndex(), getType(), builder); + public SearchResponse query(SearchSourceBuilder builder) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.query(index, type, builder); + } catch (Exception e) { + log.error("【ES】条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } + @Override - public PageData pojoPage(SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.pojoPage(getIndex(), getType(), builder, getEntityClass()); + public PageData pojoPage(SearchSourceBuilder builder) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.pojoPage(index, type, builder, getEntityClass()); + } catch (Exception e) { + log.error("【ES】from + size 分页条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } @Override - public ScrollData pojoPageByLastId(String lastId, int size, QueryBuilder queryBuilder) throws IOException { - return elasticsearchTemplate.pojoPageByLastId(getIndex(), getType(), lastId, size, - queryBuilder, getEntityClass()); + public ScrollData pojoPageByLastId(String scrollId, int size, QueryBuilder queryBuilder) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.pojoPageByScrollId(index, type, scrollId, size, queryBuilder, + getEntityClass()); + } catch (Exception e) { + log.error("【ES】search after 分页条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } @Override - public ScrollData pojoScrollBegin(SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.pojoScrollBegin(getIndex(), getType(), builder, getEntityClass()); + public ScrollData pojoScrollBegin(SearchSourceBuilder builder) { + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.pojoScrollBegin(index, type, builder, getEntityClass()); + } catch (Exception e) { + log.error("【ES】开启滚动分页条件查询异常！index: {}, type: {}", index, type, e); + return null; + } } @Override - public ScrollData pojoScroll(String scrollId, SearchSourceBuilder builder) throws IOException { - return elasticsearchTemplate.pojoScroll(scrollId, builder, getEntityClass()); + public ScrollData pojoScroll(String scrollId, SearchSourceBuilder builder) { + try { + return elasticsearchTemplate.pojoScroll(scrollId, builder, getEntityClass()); + } catch (Exception e) { + log.error("【ES】滚动分页条件查询异常！scrollId: {}", scrollId, e); + return null; + } } @Override - public boolean pojoScrollEnd(String scrollId) throws IOException { - return elasticsearchTemplate.pojoScrollEnd(scrollId); + public boolean pojoScrollEnd(String scrollId) { + try { + return elasticsearchTemplate.pojoScrollEnd(scrollId); + } catch (Exception e) { + log.error("【ES】关闭滚动分页条件查询异常！scrollId: {}", scrollId, e); + return false; + } } - @Override - public T save(T entity) throws IOException { - String index = checkIndex(); - checkData(entity); - return elasticsearchTemplate.save(index, getType(), entity); + public T save(T entity) { + if (entity == null) { + return null; + } + String index = getIndex(); + String type = getType(); + try { + checkIndex(); + checkData(entity); + return elasticsearchTemplate.save(index, type, entity); + } catch (Exception e) { + log.error("【ES】添加数据异常！index: {}, type: {}, entity: {}", index, type, JSONUtil.toJsonStr(entity), e); + return null; + } } @Override - public boolean saveBatch(Collection list) throws IOException { - String index = checkIndex(); - checkData(list); - return elasticsearchTemplate.saveBatch(index, getType(), list); + public boolean saveBatch(Collection list) { + if (CollectionUtil.isEmpty(list)) { + return false; + } + String index = getIndex(); + String type = getType(); + try { + checkIndex(); + checkData(list); + return elasticsearchTemplate.saveBatch(index, type, list); + } catch (Exception e) { + log.error("【ES】批量添加数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + return false; + } } @Override - public void asyncSaveBatch(Collection list) throws IOException { - String index = checkIndex(); - checkData(list); - ActionListener listener = new ActionListener() { - @Override - public void onResponse(BulkResponse response) { - if (response != null && !response.hasFailures()) { - String msg = StrUtil.format("【ES】异步批量保存 {} 成功！", index); - log.info(msg); - } else { - String msg = StrUtil.format("【ES】异步批量保存 {} 失败！", index); - log.warn(msg); - } - } + public void asyncSaveBatch(Collection list) { + asyncSaveBatch(list, DEFAULT_BULK_LISTENER); + } - @Override - public void onFailure(Exception e) { - String msg = StrUtil.format("【ES】异步批量保存 {} 异常！", index); - log.error(msg, e); - } - }; - asyncSaveBatch(list, listener); + @Override + public void asyncSaveBatch(Collection list, ActionListener listener) { + if (CollectionUtil.isEmpty(list)) { + return; + } + String index = getIndex(); + String type = getType(); + try { + checkIndex(); + checkData(list); + elasticsearchTemplate.asyncSaveBatch(index, getType(), list, listener); + } catch (Exception e) { + log.error("【ES】异步批量添加数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + } } @Override - public void asyncSaveBatch(Collection list, ActionListener listener) throws IOException { - String index = checkIndex(); - checkData(list); - elasticsearchTemplate.asyncSaveBatch(index, getType(), list, listener); + public T updateById(T entity) { + if (entity == null) { + return null; + } + String index = getIndex(); + String type = getType(); + try { + checkData(entity); + return elasticsearchTemplate.updateById(index, type, entity); + } catch (Exception e) { + log.error("【ES】更新数据异常！index: {}, type: {}", index, type, e); + return null; + } } @Override - public T updateById(T entity) throws IOException { - checkData(entity); - return elasticsearchTemplate.updateById(getIndex(), getType(), entity); + public boolean updateBatchIds(Collection list) { + if (CollectionUtil.isEmpty(list)) { + return false; + } + String index = getIndex(); + String type = getType(); + try { + checkData(list); + return elasticsearchTemplate.updateBatchIds(index, type, list); + } catch (Exception e) { + log.error("【ES】批量更新数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + return false; + } } @Override - public boolean updateBatchIds(Collection list) throws IOException { - checkData(list); - return elasticsearchTemplate.updateBatchIds(getIndex(), getType(), list); + public void asyncUpdateBatchIds(Collection list) { + asyncUpdateBatchIds(list, DEFAULT_BULK_LISTENER); } @Override public void asyncUpdateBatchIds(Collection list, ActionListener listener) { - checkData(list); - elasticsearchTemplate.asyncUpdateBatchIds(getIndex(), getType(), list, listener); + if (CollectionUtil.isEmpty(list)) { + return; + } + String index = getIndex(); + String type = getType(); + try { + checkData(list); + elasticsearchTemplate.asyncUpdateBatchIds(index, type, list, listener); + } catch (Exception e) { + log.error("【ES】异步批量更新数据异常！index: {}, type: {}, size: {}", index, type, list.size(), e); + } } @Override - public boolean deleteById(String id) throws IOException { - return elasticsearchTemplate.deleteById(getIndex(), getType(), id); + public boolean deleteById(String id) { + if (StrUtil.isBlank(id)) { + return false; + } + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.deleteById(index, type, id); + } catch (Exception e) { + log.error("【ES】根据ID删除数据异常！index: {}, type: {}, id: {}", index, type, id, e); + return false; + } } @Override - public boolean deleteBatchIds(Collection ids) throws IOException { - return elasticsearchTemplate.deleteBatchIds(getIndex(), getType(), ids); + public boolean deleteBatchIds(Collection ids) { + if (CollectionUtil.isEmpty(ids)) { + return false; + } + String index = getIndex(); + String type = getType(); + try { + return elasticsearchTemplate.deleteBatchIds(index, type, ids); + } catch (Exception e) { + log.error("【ES】根据ID批量删除数据异常！index: {}, type: {}, ids: {}", index, type, ids, e); + return false; + } } @Override - public void asyncDeleteBatchIds(Collection ids) throws IOException { - ActionListener listener = new ActionListener() { - @Override - public void onResponse(BulkResponse response) { - if (response != null && !response.hasFailures()) { - log.info("【ES】异步批量删除成功！"); - } else { - log.warn("【ES】异步批量删除失败！ids: {}", ids); - } - } - - @Override - public void onFailure(Exception e) { - log.error("【ES】异步批量删除异常！ids: {}", ids, e); - } - }; - asyncDeleteBatchIds(ids, listener); + public void asyncDeleteBatchIds(Collection ids) { + asyncDeleteBatchIds(ids, DEFAULT_BULK_LISTENER); } @Override - public void asyncDeleteBatchIds(Collection ids, ActionListener listener) throws IOException { - elasticsearchTemplate.asyncDeleteBatchIds(getIndex(), getType(), ids, listener); + public void asyncDeleteBatchIds(Collection ids, ActionListener listener) { + if (CollectionUtil.isEmpty(ids)) { + return; + } + String index = getIndex(); + String type = getType(); + try { + elasticsearchTemplate.asyncDeleteBatchIds(index, type, ids, listener); + } catch (Exception e) { + log.error("【ES】异步根据ID批量删除数据异常！index: {}, type: {}, ids: {}", index, type, ids, e); + } } - protected String checkIndex() throws IOException { + protected String checkIndex() { if (!enableAutoCreateIndex()) { return getIndex(); } String index = createIndexIfNotExists(); if (StrUtil.isBlank(index)) { - String msg = StrUtil.format("【ES】索引找不到且创建失败！", index); + String msg = StrUtil.format("【ES】索引 {} 找不到且创建失败！", index); throw new DefaultException(ResultCode.ERROR, msg); } return index; @@ -318,4 +483,20 @@ protected void checkData(T entity) { } } + protected final ActionListener DEFAULT_BULK_LISTENER = new ActionListener() { + @Override + public void onResponse(BulkResponse response) { + if (response != null && !response.hasFailures()) { + log.info("【ES】异步批量写数据成功！index: {}, type: {}", getIndex(), getType()); + } else { + log.warn("【ES】异步批量写数据失败！index: {}, type: {}", getIndex(), getType()); + } + } + + @Override + public void onFailure(Exception e) { + log.error("【ES】异步批量写数据异常！index: {}, type: {}", getIndex(), getType()); + } + }; + } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java index bacabff6..5630d664 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/mapper/EsMapper.java @@ -13,11 +13,11 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import java.io.IOException; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * ES Mapper @@ -57,29 +57,38 @@ public interface EsMapper { */ Class getEntityClass(); - RestHighLevelClient getClient() throws IOException; + /** + * 如果开启，添加 ES 数据时，如果索引不存在，会自动创建索引 + */ + default boolean enableAutoCreateIndex() { + return false; + } - BulkProcessor getBulkProcessor() throws IOException; + RestHighLevelClient getClient(); - boolean isIndexExists() throws IOException; + BulkProcessor getBulkProcessor(); - String createIndexIfNotExists() throws IOException; + boolean isIndexExists(); - void deleteIndex() throws IOException; + String createIndexIfNotExists(); - void updateAlias() throws IOException; + void deleteIndex(); - GetResponse getById(String id) throws IOException; + void updateAlias(); - GetResponse getById(String id, Long version) throws IOException; + Set getIndexSet(); - T pojoById(String id) throws IOException; + GetResponse getById(String id); - T pojoById(String id, Long version) throws IOException; + GetResponse getById(String id, Long version); - List pojoListByIds(Collection ids) throws IOException; + T pojoById(String id); - default Map pojoMapByIds(Collection ids) throws IOException { + T pojoById(String id, Long version); + + List pojoListByIds(Collection ids); + + default Map pojoMapByIds(Collection ids) { List list = pojoListByIds(ids); if (CollectionUtil.isEmpty(list)) { return new HashMap<>(0); @@ -92,40 +101,42 @@ default Map pojoMapByIds(Collection ids) throws IOException { return map; } - long count(SearchSourceBuilder builder) throws IOException; + long count(SearchSourceBuilder builder); + + SearchResponse query(SearchSourceBuilder builder); - SearchResponse query(SearchSourceBuilder builder) throws IOException; + PageData pojoPage(SearchSourceBuilder builder); - PageData pojoPage(SearchSourceBuilder builder) throws IOException; + ScrollData pojoPageByLastId(String scrollId, int size, QueryBuilder queryBuilder); - ScrollData pojoPageByLastId(String lastId, int size, QueryBuilder queryBuilder) throws IOException; + ScrollData pojoScrollBegin(SearchSourceBuilder builder); - ScrollData pojoScrollBegin(SearchSourceBuilder builder) throws IOException; + ScrollData pojoScroll(String scrollId, SearchSourceBuilder builder); - ScrollData pojoScroll(String scrollId, SearchSourceBuilder builder) throws IOException; + boolean pojoScrollEnd(String scrollId); - boolean pojoScrollEnd(String scrollId) throws IOException; + T save(T entity); - T save(T entity) throws IOException; + boolean saveBatch(Collection list); - boolean saveBatch(Collection list) throws IOException; + void asyncSaveBatch(Collection list); - void asyncSaveBatch(Collection list) throws IOException; + void asyncSaveBatch(Collection list, ActionListener listener); - void asyncSaveBatch(Collection list, ActionListener listener) throws IOException; + T updateById(T entity); - T updateById(T entity) throws IOException; + boolean updateBatchIds(Collection list); - boolean updateBatchIds(Collection list) throws IOException; + void asyncUpdateBatchIds(Collection list); void asyncUpdateBatchIds(Collection list, ActionListener listener); - boolean deleteById(String id) throws IOException; + boolean deleteById(String id); - boolean deleteBatchIds(Collection ids) throws IOException; + boolean deleteBatchIds(Collection ids); - void asyncDeleteBatchIds(Collection ids) throws IOException; + void asyncDeleteBatchIds(Collection ids); - void asyncDeleteBatchIds(Collection ids, ActionListener listener) throws IOException; + void asyncDeleteBatchIds(Collection ids, ActionListener listener); } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java index 05b95f7a..67277952 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java @@ -8,6 +8,8 @@ import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -19,6 +21,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; /** * ElasticsearchTemplate 测试 @@ -75,8 +79,34 @@ protected void createIndex() throws IOException { Assertions.assertThat(exists).isTrue(); } + public void getIndexList() throws IOException { + Set set = TEMPLATE.getIndexSet(getAlias()); + log.info("alias: {}, indexList: {}", getAlias(), set); + Assertions.assertThat(set).isNotEmpty(); + } + protected void save() throws IOException { String id = "1"; + T oldEntity = getOneMockData(id); + TEMPLATE.save(getIndex(), getType(), oldEntity); + T newEntity = TEMPLATE.pojoById(getIndex(), getType(), id, getEntityClass()); + log.info("记录：{}", JsonUtil.toString(newEntity)); + Assertions.assertThat(newEntity).isNotNull(); + } + + protected void saveBatch() throws IOException { + int total = 5000; + List> listGroup = CollectionUtil.split(getMockList(total), 1000); + for (List list : listGroup) { + TEMPLATE.saveBatch(getIndex(), getType(), list); + } + long count = TEMPLATE.count(getIndex(), getType(), new SearchSourceBuilder()); + log.info("批量更新记录数: {}", count); + Assertions.assertThat(count).isEqualTo(total); + } + + protected void asyncSave() throws IOException { + String id = "10000"; T entity = getOneMockData(id); TEMPLATE.save(getIndex(), getType(), entity); T newEntity = TEMPLATE.pojoById(getIndex(), getType(), id, getEntityClass()); @@ -84,12 +114,13 @@ protected void save() throws IOException { Assertions.assertThat(newEntity).isNotNull(); } - protected void saveBatch() throws IOException { + protected void asyncSaveBatch() throws IOException, InterruptedException { int total = 10000; List> listGroup = CollectionUtil.split(getMockList(total), 1000); for (List list : listGroup) { - TEMPLATE.saveBatch(getIndex(), getType(), list); + TEMPLATE.asyncSaveBatch(getIndex(), getType(), list, DEFAULT_BULK_LISTENER); } + TimeUnit.SECONDS.sleep(20); long count = TEMPLATE.count(getIndex(), getType(), new SearchSourceBuilder()); log.info("批量更新记录数: {}", count); Assertions.assertThat(count).isEqualTo(total); @@ -167,7 +198,7 @@ protected void pojoPageByLastId() throws IOException { long total = TEMPLATE.count(getIndex(), getType(), queryBuilder); ScrollData scrollData = - TEMPLATE.pojoPageByLastId(getIndex(), getType(), null, SIZE, queryBuilder, getEntityClass()); + TEMPLATE.pojoPageByScrollId(getIndex(), getType(), null, SIZE, queryBuilder, getEntityClass()); if (scrollData == null || scrollData.getScrollId() == null) { return; } @@ -181,8 +212,8 @@ protected void pojoPageByLastId() throws IOException { String scrollId = scrollData.getScrollId(); while (CollectionUtil.isNotEmpty(scrollData.getContent())) { - scrollData = - TEMPLATE.pojoPageByLastId(getIndex(), getType(), scrollId, SIZE, queryBuilder, getEntityClass()); + scrollData = TEMPLATE.pojoPageByScrollId(getIndex(), getType(), scrollId, SIZE, + queryBuilder, getEntityClass()); if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { break; } @@ -238,4 +269,20 @@ protected void pojoScroll() throws IOException { Assertions.assertThat(count).isEqualTo(total); } + final ActionListener DEFAULT_BULK_LISTENER = new ActionListener() { + @Override + public void onResponse(BulkResponse response) { + if (response != null && !response.hasFailures()) { + log.info("【ES】异步批量写数据成功！index: {}, type: {}", getIndex(), getType()); + } else { + log.warn("【ES】异步批量写数据失败！index: {}, type: {}", getIndex(), getType()); + } + } + + @Override + public void onFailure(Exception e) { + log.error("【ES】异步批量写数据异常！index: {}, type: {}", getIndex(), getType()); + } + }; + } diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java index 37242460..aa8aab5c 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/UserElasticsearchTemplateTest.java @@ -83,6 +83,7 @@ protected List getMockList(int num) { public void indexTest() throws IOException { super.deleteIndex(); super.createIndex(); + super.getIndexList(); } @Test @@ -92,6 +93,13 @@ protected void writeTest() throws IOException { super.saveBatch(); } + @Test + @DisplayName("异步写数据测试") + public void asyncWriteTest() throws IOException, InterruptedException { + super.asyncSave(); + super.asyncSaveBatch(); + } + @Test @DisplayName("读数据测试") public void readTest() throws IOException { diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java index ccc7b393..a287be67 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/mapper/UserEsMapperTest.java @@ -21,7 +21,6 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import java.io.IOException; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -49,7 +48,7 @@ class DeleteIndexTest { @Test @DisplayName("删除当天索引") - public void deleteIndex() throws IOException { + public void deleteIndex() { String index = mapper.getIndex(); boolean indexExists = mapper.isIndexExists(); if (!indexExists) { @@ -63,7 +62,7 @@ public void deleteIndex() throws IOException { @Test @DisplayName("根据日期删除索引") - public void deleteIndexInDay() throws IOException { + public void deleteIndexInDay() { String index = mapper.getIndex(day); boolean indexExists = mapper.isIndexExistsInDay(day); if (!indexExists) { @@ -83,7 +82,7 @@ class CreateIndexTest { @Test @DisplayName("创建当天索引") - public void createIndex() throws IOException { + public void createIndex() { String index = mapper.getIndex(); boolean indexExists = mapper.isIndexExists(); @@ -99,7 +98,7 @@ public void createIndex() throws IOException { @Test @DisplayName("根据日期创建索引") - public void createIndexInDay() throws IOException { + public void createIndexInDay() { String index = mapper.getIndex(day); boolean indexExists = mapper.isIndexExistsInDay(day); @@ -108,7 +107,7 @@ public void createIndexInDay() throws IOException { return; } - mapper.createIndexInDay(day); + mapper.createIndexIfNotExistsInDay(day); indexExists = mapper.isIndexExistsInDay(day); Assertions.assertThat(indexExists).isTrue(); } @@ -121,7 +120,7 @@ class WriteTest { @Test @DisplayName("保存当天数据") - public void save() throws IOException { + public void save() { String id = "1"; User entity = getOneMockData(id); mapper.save(entity); @@ -132,7 +131,7 @@ public void save() throws IOException { @Test @DisplayName("保存指定日期数据") - public void saveInDay() throws IOException { + public void saveInDay() { String id = "1"; User entity = getOneMockData(id); mapper.saveInDay(day, entity); @@ -143,7 +142,7 @@ public void saveInDay() throws IOException { @Test @DisplayName("批量保存当天数据") - public void batchSave() throws IOException, InterruptedException { + public void batchSave() throws InterruptedException { int total = 10000; List> listGroup = CollectionUtil.split(getMockList(total), 1000); for (List list : listGroup) { @@ -157,7 +156,7 @@ public void batchSave() throws IOException, InterruptedException { @Test @DisplayName("批量保存指定日期数据") - public void batchSaveInDay() throws IOException, InterruptedException { + public void batchSaveInDay() throws InterruptedException { int total = 10000; List> listGroup = CollectionUtil.split(getMockList(total), 1000); for (List list : listGroup) { @@ -177,7 +176,7 @@ class ReadTest { @Test @DisplayName("根据ID查找当日数据") - public void pojoById() throws IOException { + public void pojoById() { String id = "1"; User newEntity = mapper.pojoById(id); log.info("entity: {}", JsonUtil.toString(newEntity)); @@ -186,7 +185,7 @@ public void pojoById() throws IOException { @Test @DisplayName("根据ID查找指定日期数据") - public void pojoByIdInDay() throws IOException { + public void pojoByIdInDay() { String id = "1"; User newEntity = mapper.pojoByIdInDay(day, id); log.info("entity: {}", JsonUtil.toString(newEntity)); @@ -195,7 +194,7 @@ public void pojoByIdInDay() throws IOException { @Test @DisplayName("获取匹配条件的记录数") - public void count() throws IOException { + public void count() { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -207,7 +206,7 @@ public void count() throws IOException { @Test @DisplayName("获取匹配条件的指定日期记录数") - public void countInDay() throws IOException { + public void countInDay() { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -219,7 +218,7 @@ public void countInDay() throws IOException { @Test @DisplayName("获取匹配条件的记录") - public void query() throws IOException { + public void query() { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -239,7 +238,7 @@ public void query() throws IOException { @Test @DisplayName("获取匹配条件的指定日期记录") - public void queryInDay() throws IOException { + public void queryInDay() { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -259,7 +258,7 @@ public void queryInDay() throws IOException { @Test @DisplayName("from + size 分页查询当日数据") - public void pojoPage() throws IOException { + public void pojoPage() { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -276,7 +275,7 @@ public void pojoPage() throws IOException { @Test @DisplayName("from + size 分页查询指定日期数据") - public void pojoPageInDay() throws IOException { + public void pojoPageInDay() { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -293,7 +292,7 @@ public void pojoPageInDay() throws IOException { @Test @DisplayName("search after 分页查询当日数据") - protected void pojoPageByLastId() throws IOException { + protected void pojoPageByLastId() { BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); @@ -332,7 +331,7 @@ protected void pojoPageByLastId() throws IOException { @Test @DisplayName("search after 分页查询指定日期数据") - protected void pojoPageByLastIdInDay() throws IOException { + protected void pojoPageByLastIdInDay() { BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); queryBuilder.must(QueryBuilders.rangeQuery("docId").lt("100")); @@ -371,7 +370,7 @@ protected void pojoPageByLastIdInDay() throws IOException { @Test @DisplayName("滚动翻页当日数据") - public void pojoScroll() throws IOException { + public void pojoScroll() { final int size = 100; @@ -412,7 +411,7 @@ public void pojoScroll() throws IOException { @Test @DisplayName("滚动翻页指定日期数据") - public void pojoScrollInDay() throws IOException { + public void pojoScrollInDay() { final int size = 100; From 47d7d1552f1be6cde030d44b19d976553965886a Mon Sep 17 00:00:00 2001 From: dunwu Date: Wed, 9 Oct 2024 07:16:02 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20=E7=A4=BA=E4=BE=8B=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch/ElasticsearchTemplate.java | 96 ++++++++++--------- .../BaseElasticsearchTemplateTest.java | 19 ++-- codes/javadb/redis/pom.xml | 22 +---- .../springboot/RedisAutoConfiguration.java | 26 ++++- .../javadb/redis/RedissonStandaloneTest.java | 84 +++++++++++++++- .../javadb/redis/jedis/JedisPoolDemoTest.java | 5 +- .../test/resources/redisson-standalone.xml | 2 - 7 files changed, 175 insertions(+), 79 deletions(-) diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java index 749956e8..5e627cbe 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/main/java/io/github/dunwu/javadb/elasticsearch/ElasticsearchTemplate.java @@ -280,7 +280,7 @@ public T save(String index, String type, T entity) thro || response.getResult() == DocWriteResponse.Result.UPDATED) { return entity; } else { - log.warn("【ES】save 响应结果无效！result: {}", response.getResult()); + log.warn("【ES】save 失败，result: {}！", response.getResult()); return null; } } @@ -292,45 +292,25 @@ public boolean saveBatch(String index, String type, Col return true; } - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - for (T entity : list) { - Map map = toMap(entity); - if (MapUtil.isEmpty(map)) { - continue; - } - IndexRequest request = new IndexRequest(index, type).source(map); - if (entity.getDocId() != null) { - request.id(entity.getDocId()); - } - bulkRequest.add(request); - } - + BulkRequest bulkRequest = toBulkIndexRequest(index, type, list); BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); - return response != null && !response.hasFailures(); + if (response == null) { + log.warn("【ES】saveBatch 失败，result 为空！list: {}", JsonUtil.toString(list)); + return false; + } + if (response.hasFailures()) { + log.warn("【ES】saveBatch 失败，result: {}！", response.buildFailureMessage()); + return false; + } + return true; } public void asyncSaveBatch(String index, String type, Collection list, ActionListener listener) { - if (CollectionUtil.isEmpty(list)) { return; } - - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - for (T entity : list) { - Map map = toMap(entity); - if (MapUtil.isEmpty(map)) { - continue; - } - IndexRequest request = new IndexRequest(index, type).source(map); - if (entity.getDocId() != null) { - request.id(entity.getDocId()); - } - bulkRequest.add(request); - } - + BulkRequest bulkRequest = toBulkIndexRequest(index, type, list); client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); } @@ -362,7 +342,7 @@ public T updateById(String index, String type, T entity if (response.getResult() == DocWriteResponse.Result.UPDATED) { return entity; } else { - log.warn("【ES】updateById 响应结果无效！result: {}", response.getResult()); + log.warn("【ES】updateById 响应结果无效，result: {}！", response.getResult()); return null; } } @@ -374,23 +354,49 @@ public boolean updateBatchIds(String index, String type return true; } - BulkRequest bulkRequest = toUpdateBulkRequest(index, type, list); + BulkRequest bulkRequest = toBulkUpdateRequest(index, type, list); BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); - return response != null && !response.hasFailures(); + if (response == null) { + log.warn("【ES】updateBatchIds 失败，result 为空！list: {}", JsonUtil.toString(list)); + return false; + } + if (response.hasFailures()) { + log.warn("【ES】updateBatchIds 失败，result: {}！", response.buildFailureMessage()); + return false; + } + return true; } public void asyncUpdateBatchIds(String index, String type, Collection list, ActionListener listener) { - if (CollectionUtil.isEmpty(list)) { return; } - - BulkRequest bulkRequest = toUpdateBulkRequest(index, type, list); + BulkRequest bulkRequest = toBulkUpdateRequest(index, type, list); client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); } - private BulkRequest toUpdateBulkRequest(String index, String type, Collection list) { + private BulkRequest toBulkIndexRequest(String index, String type, Collection list) { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (T entity : list) { + if (entity == null) { + continue; + } + Map map = toMap(entity); + if (MapUtil.isEmpty(map)) { + continue; + } + IndexRequest request = new IndexRequest(index, type).source(map); + if (entity.getDocId() != null) { + request.id(entity.getDocId()); + } + bulkRequest.add(request); + } + return bulkRequest; + } + + private BulkRequest toBulkUpdateRequest(String index, String type, Collection list) { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (T entity : list) { @@ -426,11 +432,14 @@ public boolean deleteBatchIds(String index, String type, Collection ids) BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); if (response == null) { - log.warn("【ES】batchDeleteById 响应结果为空！"); + log.warn("【ES】deleteBatchIds 失败，result 为空！ids: {}", JsonUtil.toString(ids)); return false; } - - return !response.hasFailures(); + if (response.hasFailures()) { + log.warn("【ES】deleteBatchIds 失败，result: {}！", response.buildFailureMessage()); + return false; + } + return true; } public void asyncDeleteBatchIds(String index, String type, Collection ids, @@ -568,7 +577,8 @@ public PageData pojoPage(String index, String type, int from, int size, Q /** * search after 分页 */ - public ScrollData pojoPageByScrollId(String index, String type, String scrollId, int size, + public ScrollData pojoPageByScrollId(String index, String type, String scrollId, + int size, QueryBuilder queryBuilder, Class clazz) throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); diff --git a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java index 67277952..4ec530a1 100644 --- a/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java +++ b/codes/javadb/elasticsearch/elasticsearch6/src/test/java/io/github/dunwu/javadb/elasticsearch/BaseElasticsearchTemplateTest.java @@ -8,6 +8,7 @@ import io.github.dunwu.javadb.elasticsearch.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; @@ -59,12 +60,18 @@ public abstract class BaseElasticsearchTemplateTest { protected abstract List getMockList(int num); protected void deleteIndex() throws IOException { - boolean exists = TEMPLATE.isIndexExists(getIndex()); - if (!exists) { - return; + try { + Set set = TEMPLATE.getIndexSet(getAlias()); + if (CollectionUtil.isNotEmpty(set)) { + for (String index : set) { + log.info("删除 alias: {}, index: {}", getAlias(), index); + TEMPLATE.deleteIndex(index); + } + } + } catch (IOException | ElasticsearchException e) { + log.error("删除索引失败！", e); } - TEMPLATE.deleteIndex(getIndex()); - exists = TEMPLATE.isIndexExists(getIndex()); + boolean exists = TEMPLATE.isIndexExists(getIndex()); Assertions.assertThat(exists).isFalse(); } @@ -98,7 +105,7 @@ protected void saveBatch() throws IOException { int total = 5000; List> listGroup = CollectionUtil.split(getMockList(total), 1000); for (List list : listGroup) { - TEMPLATE.saveBatch(getIndex(), getType(), list); + Assertions.assertThat(TEMPLATE.saveBatch(getIndex(), getType(), list)).isTrue(); } long count = TEMPLATE.count(getIndex(), getType(), new SearchSourceBuilder()); log.info("批量更新记录数: {}", count); diff --git a/codes/javadb/redis/pom.xml b/codes/javadb/redis/pom.xml index e847e9ce..3150ffba 100644 --- a/codes/javadb/redis/pom.xml +++ b/codes/javadb/redis/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 2.6.3 + 2.7.18 io.github.dunwu @@ -36,7 +36,7 @@ cn.hutool hutool-all - 5.5.9 + 5.8.27 org.projectlombok @@ -51,27 +51,11 @@ org.redisson redisson - 3.16.8 + 3.29.0 - - - junit - junit - test - - - - - org.redisson - redisson - ${redisson.version} - - - - diff --git a/codes/javadb/redis/src/main/java/io/github/dunwu/javadb/redis/springboot/RedisAutoConfiguration.java b/codes/javadb/redis/src/main/java/io/github/dunwu/javadb/redis/springboot/RedisAutoConfiguration.java index 6ea387bf..eb7c40e1 100644 --- a/codes/javadb/redis/src/main/java/io/github/dunwu/javadb/redis/springboot/RedisAutoConfiguration.java +++ b/codes/javadb/redis/src/main/java/io/github/dunwu/javadb/redis/springboot/RedisAutoConfiguration.java @@ -1,14 +1,24 @@ package io.github.dunwu.javadb.redis.springboot; +import cn.hutool.core.util.StrUtil; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.*; +import org.springframework.data.redis.core.HashOperations; +import org.springframework.data.redis.core.ListOperations; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.SetOperations; +import org.springframework.data.redis.core.ValueOperations; +import org.springframework.data.redis.core.ZSetOperations; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; @@ -22,6 +32,19 @@ public class RedisAutoConfiguration { @Autowired private ObjectMapper objectMapper; + @Value("${spring.redis.host:localhost}") + private String host; + + @Value("${spring.redis.port:6379}") + private String port; + + @Bean + public RedissonClient redissonClient() { + Config config = new Config(); + config.useSingleServer().setAddress(StrUtil.format("redis://{}:{}", host, port)); + return Redisson.create(config); + } + @Bean public HashOperations hashOperations(RedisTemplate redisTemplate) { return redisTemplate.opsForHash(); @@ -44,7 +67,6 @@ public RedisTemplate redisTemplate(RedisConnectionFactory factor // 使用Jackson2JsonRedisSerializer来序列化和反序列化redis的value值（默认使用JDK的序列化方式） Jackson2JsonRedisSerializer serializer = new Jackson2JsonRedisSerializer(Object.class); serializer.setObjectMapper(objectMapper); - RedisTemplate template = new RedisTemplate<>(); // 配置连接工厂 template.setConnectionFactory(factory); diff --git a/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java b/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java index f483c9a1..ed3c13a3 100644 --- a/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java +++ b/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java @@ -1,26 +1,104 @@ package io.github.dunwu.javadb.redis; +import cn.hutool.core.thread.ThreadUtil; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.redisson.api.RBucket; +import org.redisson.api.RLock; import org.redisson.api.RedissonClient; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + /** * @author Zhang Peng * @since 2018/6/19 */ +@Slf4j public class RedissonStandaloneTest { + private static RedissonClient redissonClient; + + static { + ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:redisson-standalone.xml"); + redissonClient = (RedissonClient) applicationContext.getBean("standalone"); + } + @Test + @DisplayName("测试连接") public void testRedissonConnect() { - ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:redisson-standalone.xml"); - RedissonClient redisson = (RedissonClient) applicationContext.getBean("standalone"); // 首先获取redis中的key-value对象，key不存在没关系 - RBucket keyObject = redisson.getBucket("key"); + RBucket keyObject = redissonClient.getBucket("key"); // 如果key存在，就设置key的值为新值value // 如果key不存在，就设置key的值为value keyObject.set("value"); + String value = keyObject.get(); + System.out.println("value=" + value); + } + + @Test + @DisplayName("分布式锁测试") + public void testLock() { + // 两个线程任务都是不断再尝试获取或，直到成功获取锁后才推出任务 + // 第一个线程获取到锁后，第二个线程需要等待 5 秒超时后才能获取到锁 + CountDownLatch latch = new CountDownLatch(2); + ExecutorService executorService = ThreadUtil.newFixedExecutor(2, "获取锁", true); + executorService.submit(new Task(latch)); + executorService.submit(new Task(latch)); + + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + // 输出： + // 17:59:25.896 [获取锁1] [INFO ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁成功 + // 17:59:26.888 [获取锁0] [WARN ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁失败 + // 17:59:27.889 [获取锁0] [WARN ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁失败 + // 17:59:28.891 [获取锁0] [WARN ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁失败 + // 17:59:29.892 [获取锁0] [WARN ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁失败 + // 17:59:30.895 [获取锁0] [WARN ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁失败 + // 17:59:30.896 [获取锁0] [INFO ] i.g.d.j.redis.RedissonStandaloneTest.run - + // 获取分布式锁成功 + + static class Task implements Runnable { + + private CountDownLatch latch; + + public Task(CountDownLatch latch) { + this.latch = latch; + } + + @Override + public void run() { + while (true) { + RLock lock = redissonClient.getLock("test_lock"); + try { + boolean isLock = lock.tryLock(1, 5, TimeUnit.SECONDS); + if (isLock) { + log.info("获取分布式锁成功"); + break; + } else { + log.warn("获取分布式锁失败"); + } + } catch (Exception e) { + log.error("获取分布式锁异常", e); + } + } + latch.countDown(); + } + } } diff --git a/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java b/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java index 9b693bad..e52d65ba 100644 --- a/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java +++ b/codes/javadb/redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java @@ -1,13 +1,11 @@ package io.github.dunwu.javadb.redis.jedis; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; @@ -18,7 +16,6 @@ /** * @author Zhang Peng */ -@RunWith(SpringJUnit4ClassRunner.class) @ActiveProfiles("dev") @ContextConfiguration(locations = {"classpath:/applicationContext.xml"}) public class JedisPoolDemoTest { diff --git a/codes/javadb/redis/src/test/resources/redisson-standalone.xml b/codes/javadb/redis/src/test/resources/redisson-standalone.xml index 462abd6c..47607374 100644 --- a/codes/javadb/redis/src/test/resources/redisson-standalone.xml +++ b/codes/javadb/redis/src/test/resources/redisson-standalone.xml @@ -14,8 +14,6 @@ idle-connection-timeout="10000" connect-timeout="10000" timeout="3000" - ping-timeout="30000" - reconnection-timeout="30000" database="0"/>