From 399b94f406207cf25c695b66e8217c1d1cebe1e8 Mon Sep 17 00:00:00 2001 From: Alex <97039406+chengyouling@users.noreply.github.com> Date: Sat, 19 Oct 2024 19:03:30 +0800 Subject: [PATCH] [#4562] fixed dual-AZ engine disk abnormal scenario registration center intermittently delete instance problem (#4563) --- .../config/center/client/ConfigCenterManager.java | 9 ++++----- .../servicecomb/config/kie/client/KieClient.java | 2 +- .../config/kie/client/KieConfigManager.java | 11 +++++------ .../http/client/common/AbstractAddressManager.java | 4 ++-- .../servicecomb/http/client/task/AbstractTask.java | 14 ++++++++++++++ .../service/center/client/ServiceCenterClient.java | 5 ++--- .../center/client/ServiceCenterOperation.java | 5 +---- .../center/client/ServiceCenterRawClient.java | 6 +++--- .../center/client/ServiceCenterRegistration.java | 11 ++++------- 9 files changed, 36 insertions(+), 31 deletions(-) diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java index 5b8c434045c..4d65465b3b9 100644 --- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java +++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java @@ -64,7 +64,8 @@ public void setQueryConfigurationsRequest(QueryConfigurationsRequest queryConfig public void startConfigCenterManager() { this.startTask(new PollConfigurationTask(0)); - this.startTask(new CheckConfigCenterAddressTask()); + schedulerCheckAddressAvailable("cc-addr-check", new CheckConfigCenterAddressTask(), + configCenterConfiguration.getRefreshIntervalInMillis()); } class PollConfigurationTask implements Task { @@ -94,9 +95,9 @@ public void execute() { } } - class CheckConfigCenterAddressTask implements Task { + class CheckConfigCenterAddressTask implements Runnable { @Override - public void execute() { + public void run() { List isolationAddresses = configCenterAddressManager.getIsolationAddresses(); if (isolationAddresses.isEmpty()) { return; @@ -104,8 +105,6 @@ public void execute() { for (String address : isolationAddresses) { configCenterClient.checkAddressAvailable(queryConfigurationsRequest, address); } - startTask(new BackOffSleepTask(configCenterConfiguration.getRefreshIntervalInMillis(), - new CheckConfigCenterAddressTask())); } } } diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java index e85a7c65874..b44ed0b1a25 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java @@ -125,7 +125,7 @@ public void checkAddressAvailable(ConfigurationsRequest request, String address) addressManager.recoverIsolatedAddress(address); } } catch (IOException e) { - LOGGER.error("check kie config isolation address {} available error!", address, e); + LOGGER.error("check kie config isolation address {} available error!", address); } } diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java index bd5ff875c7f..59274314b6a 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java @@ -125,7 +125,8 @@ protected void initTaskPool(String taskName) { public void startConfigKieManager() { this.configurationsRequests.forEach((t) -> this.startTask(new PollConfigurationTask(0, t))); - startTask(new CheckKieConfigAddressTask(configurationsRequests.get(0))); + schedulerCheckAddressAvailable("kie-addr-check", new CheckKieAddressTask(configurationsRequests.get(0)), + kieConfiguration.getRefreshIntervalInMillis()); } class PollConfigurationTask implements Task { @@ -161,15 +162,15 @@ public void execute() { } } - class CheckKieConfigAddressTask implements Task { + class CheckKieAddressTask implements Runnable { ConfigurationsRequest configurationsRequest; - public CheckKieConfigAddressTask(ConfigurationsRequest configurationsRequest) { + public CheckKieAddressTask(ConfigurationsRequest configurationsRequest) { this.configurationsRequest = configurationsRequest; } @Override - public void execute() { + public void run() { List isolationAddresses = kieAddressManager.getIsolationAddresses(); if (isolationAddresses.isEmpty()) { return; @@ -177,8 +178,6 @@ public void execute() { for (String address : isolationAddresses) { configKieClient.checkAddressAvailable(this.configurationsRequest, address); } - startTask(new BackOffSleepTask(kieConfiguration.getRefreshIntervalInMillis(), - new CheckKieConfigAddressTask(this.configurationsRequest))); } } } diff --git a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java index 59cb3e5be8c..c5587b6ed60 100644 --- a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java +++ b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java @@ -198,7 +198,7 @@ public void recoverIsolatedAddress(String address) { recordSuccessState(address); if (addressAutoRefreshed) { if (isolationZoneAddress.remove(address)) { - LOGGER.warn("restore default address [{}]", address); + LOGGER.warn("restore same region address [{}]", address); if (eventBus != null && availableZone.isEmpty()) { eventBus.post(new EngineConnectChangedEvent()); } @@ -212,7 +212,7 @@ public void recoverIsolatedAddress(String address) { return; } if (defaultIsolationAddress.remove(address)) { - LOGGER.warn("restore same region address [{}]", address); + LOGGER.warn("restore default address [{}]", address); addresses.add(address); } } diff --git a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java index deba2d8521d..a2527140b65 100644 --- a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java +++ b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java @@ -20,6 +20,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; @@ -63,6 +64,8 @@ public void execute() { private volatile boolean running = true; + private ScheduledExecutorService addrCheckExecutor; + protected AbstractTask(String taskName) { initTaskPool(taskName); Runtime.getRuntime().addShutdownHook(new Thread(AbstractTask.this::stop, taskName + "-shutdown-hook")); @@ -73,6 +76,13 @@ protected void initTaskPool(String taskName) { new Thread(task, taskName)); } + protected void schedulerCheckAddressAvailable(String taskName, Runnable task, long delayTime) { + if (addrCheckExecutor == null) { + addrCheckExecutor = Executors.newScheduledThreadPool(1, (t) -> new Thread(t, taskName)); + } + addrCheckExecutor.scheduleWithFixedDelay(task, delayTime, delayTime, TimeUnit.MILLISECONDS); + } + protected void startTask(Task task) { if (!running) { return; @@ -96,6 +106,10 @@ public void stop() { running = false; this.taskPool.shutdown(); this.taskPool.awaitTermination(10, TimeUnit.SECONDS); + if (addrCheckExecutor != null) { + this.addrCheckExecutor.shutdown(); + this.addrCheckExecutor.awaitTermination(10, TimeUnit.SECONDS); + } } catch (InterruptedException e) { LOGGER.warn("tasks not shutdown in time {}", e.getMessage()); } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java index c9948615124..45c4c27dc01 100755 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java @@ -555,14 +555,13 @@ public boolean updateMicroserviceProperties(String serviceId, Map isolationAddresses = addressManager.getIsolationAddresses(); if (isolationAddresses.isEmpty()) { return; } for (String address : isolationAddresses) { - httpClient.checkServiceCenterAddressAvailable("/registry/microservices/" + serviceId + "/instances/" + instanceId + - "/heartbeat", null, null, address); + httpClient.checkAddressAvailable("/registry/microservices", null, null, address); } } } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java index cd91b240b8a..f296e66fae9 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java @@ -196,9 +196,6 @@ boolean updateMicroserviceInstanceStatus(String serviceId, String instanceId, /** * Check serviceCenter isolation address available - * - * @param serviceId serviceId - * @param instanceId instanceId */ - void checkIsolationAddressAvailable(String serviceId, String instanceId); + void checkIsolationAddressAvailable(); } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java index c58e5dc8bac..e95240bb3c5 100755 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java @@ -93,15 +93,15 @@ private HttpResponse doHttpRequest(String url, boolean absoluteUrl, Map headers, String content, + public void checkAddressAvailable(String url, Map headers, String content, String address) { String formatUrl = addressManager.formatUrl(url, false, address); - HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content, HttpRequest.PUT); + HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content, HttpRequest.GET); try { httpTransport.doRequest(httpRequest); addressManager.recoverIsolatedAddress(address); } catch (IOException e) { - LOGGER.error("check service center isolation address {} available error!", address, e); + LOGGER.error("check service center isolation address {} available error!", address); } } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java index 2a1296d3728..332171943fc 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java @@ -104,6 +104,7 @@ public ServiceCenterRegistration setSchemaInfos(List schemaInfos) { public void startRegistration() { startTask(new RegisterMicroserviceTask(0)); + schedulerCheckAddressAvailable("sc-addr-check", new CheckAddressTask(), heartBeatInterval); } class RegisterMicroserviceTask implements Task { @@ -248,7 +249,6 @@ public void execute() { microserviceInstance.getInstanceId()); eventBus.post(new MicroserviceInstanceRegistrationEvent(true, microservice, microserviceInstance)); startTask(new SendHeartBeatTask(0)); - startTask(new CheckServiceCenterAddressTask()); } } catch (Exception e) { LOGGER.error("register microservice instance failed, and will try again.", e); @@ -294,13 +294,10 @@ public void execute() { } } - class CheckServiceCenterAddressTask implements Task { + class CheckAddressTask implements Runnable { @Override - public void execute() { - serviceCenterClient.checkIsolationAddressAvailable(microservice.getServiceId(), - microserviceInstance.getInstanceId()); - startTask(new BackOffSleepTask(Math.max(heartBeatInterval, heartBeatRequestTimeout), - new CheckServiceCenterAddressTask())); + public void run() { + serviceCenterClient.checkIsolationAddressAvailable(); } } }