Задача Егора
This commit is contained in:
dipatrik10
@@ -1,6 +1,9 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -8,6 +11,7 @@ import org.springframework.stereotype.Service;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLParameters;
|
||||
import javax.net.ssl.TrustManagerFactory;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
@@ -19,19 +23,29 @@ import java.nio.file.Path;
|
||||
import java.security.KeyStore;
|
||||
import java.security.cert.Certificate;
|
||||
import java.security.cert.CertificateFactory;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Обновляет Kubernetes Secret с конфигурацией тенантов через Kubernetes REST API.
|
||||
* Атомарно изменяет Kubernetes Secret с конфигурацией тенантов через Kubernetes REST API.
|
||||
*
|
||||
* <p>Работает только внутри Kubernetes pod и использует service-account token и
|
||||
* service-account CA. В локальной разработке персистенция пропускается.</p>
|
||||
* <p>Каждая мутация сначала читает актуальный Secret, применяет изменение к полученному
|
||||
* списку и отправляет полный объект обратно условным {@code PUT} с
|
||||
* {@code metadata.resourceVersion}. При конфликте операция повторно читает актуальное
|
||||
* состояние и заново применяет только свою мутацию.</p>
|
||||
*/
|
||||
@Service
|
||||
public class KubernetesTenantSecretUpdater {
|
||||
public class KubernetesTenantSecretUpdater implements TenantConfigStore {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(KubernetesTenantSecretUpdater.class);
|
||||
|
||||
@@ -43,6 +57,15 @@ public class KubernetesTenantSecretUpdater {
|
||||
Path.of("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
private static final String DEFAULT_API_BASE = "https://kubernetes.default.svc";
|
||||
private static final String DEFAULT_SECRET_NAME = "tenants-secret";
|
||||
private static final int DEFAULT_MAX_ATTEMPTS = 3;
|
||||
private static final Duration DEFAULT_RETRY_DELAY = Duration.ofMillis(50);
|
||||
private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(5);
|
||||
private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(10);
|
||||
private static final Pattern DOMAIN_PATTERN = Pattern.compile(
|
||||
"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"
|
||||
);
|
||||
private static final TypeReference<List<TenantConfig>> TENANT_LIST_TYPE = new TypeReference<>() {
|
||||
};
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final Path tokenPath;
|
||||
@@ -51,6 +74,10 @@ public class KubernetesTenantSecretUpdater {
|
||||
private final String apiBase;
|
||||
private final String secretName;
|
||||
private final boolean runningInKubernetes;
|
||||
private final int maxAttempts;
|
||||
private final Duration retryDelay;
|
||||
|
||||
private volatile HttpClient httpClient;
|
||||
|
||||
public KubernetesTenantSecretUpdater() {
|
||||
this(
|
||||
@@ -69,79 +96,572 @@ public class KubernetesTenantSecretUpdater {
|
||||
String apiBase,
|
||||
String secretName,
|
||||
ObjectMapper objectMapper) {
|
||||
this.tokenPath = tokenPath;
|
||||
this.namespacePath = namespacePath;
|
||||
this.caPath = caPath;
|
||||
this.apiBase = apiBase.endsWith("/") ? apiBase.substring(0, apiBase.length() - 1) : apiBase;
|
||||
this.secretName = secretName;
|
||||
this.objectMapper = objectMapper;
|
||||
this.runningInKubernetes = Files.exists(tokenPath);
|
||||
this(
|
||||
tokenPath,
|
||||
namespacePath,
|
||||
caPath,
|
||||
apiBase,
|
||||
secretName,
|
||||
objectMapper,
|
||||
DEFAULT_MAX_ATTEMPTS,
|
||||
DEFAULT_RETRY_DELAY
|
||||
);
|
||||
}
|
||||
|
||||
KubernetesTenantSecretUpdater(Path tokenPath,
|
||||
Path namespacePath,
|
||||
Path caPath,
|
||||
String apiBase,
|
||||
String secretName,
|
||||
ObjectMapper objectMapper,
|
||||
int maxAttempts,
|
||||
Duration retryDelay) {
|
||||
this(
|
||||
tokenPath,
|
||||
namespacePath,
|
||||
caPath,
|
||||
apiBase,
|
||||
secretName,
|
||||
objectMapper,
|
||||
maxAttempts,
|
||||
retryDelay,
|
||||
detectKubernetesEnvironment(tokenPath)
|
||||
);
|
||||
}
|
||||
|
||||
KubernetesTenantSecretUpdater(Path tokenPath,
|
||||
Path namespacePath,
|
||||
Path caPath,
|
||||
String apiBase,
|
||||
String secretName,
|
||||
ObjectMapper objectMapper,
|
||||
int maxAttempts,
|
||||
Duration retryDelay,
|
||||
boolean kubernetesEnvironment) {
|
||||
this.tokenPath = Objects.requireNonNull(tokenPath, "Путь к ServiceAccount token не задан");
|
||||
this.namespacePath = Objects.requireNonNull(namespacePath, "Путь к namespace не задан");
|
||||
this.caPath = Objects.requireNonNull(caPath, "Путь к ServiceAccount CA не задан");
|
||||
this.apiBase = normalizeApiBase(apiBase);
|
||||
this.secretName = requireText(secretName, "Имя tenant Secret не задано");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "ObjectMapper не задан");
|
||||
if (maxAttempts < 1) {
|
||||
throw new IllegalArgumentException("Количество попыток должно быть положительным");
|
||||
}
|
||||
if (retryDelay == null || retryDelay.isNegative()) {
|
||||
throw new IllegalArgumentException("Задержка повтора не может быть отрицательной");
|
||||
}
|
||||
this.maxAttempts = maxAttempts;
|
||||
this.retryDelay = retryDelay;
|
||||
this.runningInKubernetes = kubernetesEnvironment || Files.exists(tokenPath);
|
||||
|
||||
if (!runningInKubernetes) {
|
||||
log.info("Приложение запущено вне Kubernetes — обновление tenant Secret будет пропущено");
|
||||
log.info("Приложение запущено вне Kubernetes — постоянное изменение tenant Secret отключено");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Сохраняет полный список тенантов в ключе {@code tenants.json} Kubernetes Secret.
|
||||
*
|
||||
* @return {@code true}, если Secret обновлён или приложение запущено вне Kubernetes
|
||||
*/
|
||||
public boolean updateTenantsConfig(List<TenantConfig> tenants) {
|
||||
@Override
|
||||
public TenantSecretUpdateReceipt upsertTenant(TenantConfig tenant) {
|
||||
TenantConfig requested = TenantSecretUpdateReceipt.copyOf(
|
||||
Objects.requireNonNull(tenant, "Конфигурация тенанта не задана")
|
||||
);
|
||||
String domain = normalizeDomain(requested.getDomain());
|
||||
requested.setDomain(domain);
|
||||
|
||||
if (!runningInKubernetes) {
|
||||
log.warn("Приложение запущено вне Kubernetes, персистенция tenant Secret пропущена");
|
||||
return true;
|
||||
log.warn("Приложение запущено вне Kubernetes, постоянное добавление тенанта пропущено");
|
||||
return new TenantSecretUpdateReceipt(false, false, null, List.of(), List.of(requested));
|
||||
}
|
||||
|
||||
try {
|
||||
String token = Files.readString(tokenPath).trim();
|
||||
String namespace = Files.readString(namespacePath).trim();
|
||||
if (token.isBlank() || namespace.isBlank()) {
|
||||
throw new IllegalStateException("ServiceAccount token или namespace не настроены");
|
||||
return mutate(tenants -> {
|
||||
List<TenantConfig> updated = new ArrayList<>(tenants);
|
||||
updated.removeIf(existing -> domain.equals(existing.getDomain()));
|
||||
updated.add(requested);
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public TenantSecretUpdateReceipt removeTenant(String domain) {
|
||||
String normalizedDomain = normalizeDomain(domain);
|
||||
if (!runningInKubernetes) {
|
||||
log.warn("Приложение запущено вне Kubernetes, постоянное удаление тенанта пропущено");
|
||||
return new TenantSecretUpdateReceipt(false, false, null, List.of(), List.of());
|
||||
}
|
||||
|
||||
return mutate(tenants -> {
|
||||
List<TenantConfig> updated = new ArrayList<>(tenants);
|
||||
updated.removeIf(existing -> normalizedDomain.equals(existing.getDomain()));
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public TenantSecretCompensationResult compensate(TenantSecretUpdateReceipt receipt) {
|
||||
Objects.requireNonNull(receipt, "Квитанция изменения tenant Secret не задана");
|
||||
if (!receipt.persisted() || !receipt.changed()) {
|
||||
return TenantSecretCompensationResult.NOT_REQUIRED;
|
||||
}
|
||||
if (!runningInKubernetes) {
|
||||
return TenantSecretCompensationResult.NOT_REQUIRED;
|
||||
}
|
||||
|
||||
String committedVersion = requireText(
|
||||
receipt.committedResourceVersion(),
|
||||
"В квитанции отсутствует resourceVersion сохранённого tenant Secret"
|
||||
);
|
||||
RequestContext context = requestContext();
|
||||
TenantConfigPersistenceException lastFailure = null;
|
||||
|
||||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
CurrentSecret current;
|
||||
try {
|
||||
current = readCurrent(context);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Компенсация tenant Secret прервана", e);
|
||||
} catch (IOException e) {
|
||||
lastFailure = persistenceFailure("Не удалось прочитать tenant Secret для компенсации", e);
|
||||
if (!prepareRetry(attempt, "чтение перед компенсацией")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
String tenantsJson = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(tenants);
|
||||
String encodedTenants = Base64.getEncoder()
|
||||
.encodeToString(tenantsJson.getBytes(StandardCharsets.UTF_8));
|
||||
String patchBody = objectMapper.writeValueAsString(Map.of(
|
||||
"data", Map.of("tenants.json", encodedTenants)
|
||||
));
|
||||
if (!committedVersion.equals(current.resourceVersion())) {
|
||||
log.warn("Компенсация tenant Secret пропущена: после исходной операции Secret уже изменён");
|
||||
return TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE;
|
||||
}
|
||||
if (!sameTenants(current.tenants(), receipt.committedTenants())) {
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Содержимое tenant Secret не соответствует resourceVersion из квитанции"
|
||||
);
|
||||
}
|
||||
|
||||
HttpResponse<String> response;
|
||||
try {
|
||||
response = putSecret(context, current, receipt.previousTenants());
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Компенсация tenant Secret прервана", e);
|
||||
} catch (IOException e) {
|
||||
TenantSecretCompensationResult reconciled = reconcileCompensation(context, receipt);
|
||||
if (reconciled != null) {
|
||||
return reconciled;
|
||||
}
|
||||
lastFailure = persistenceFailure("Не удалось подтвердить компенсацию tenant Secret", e);
|
||||
if (!prepareRetry(attempt, "компенсация после сетевой ошибки")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isSuccess(response.statusCode())) {
|
||||
TenantSecretCompensationResult confirmed = confirmCompensation(
|
||||
context,
|
||||
receipt,
|
||||
response.body()
|
||||
);
|
||||
if (confirmed != null) {
|
||||
if (confirmed == TenantSecretCompensationResult.RESTORED) {
|
||||
log.info("Конфигурация tenant Secret восстановлена: tenantCount={}",
|
||||
receipt.previousTenants().size());
|
||||
}
|
||||
return confirmed;
|
||||
}
|
||||
lastFailure = new TenantConfigPersistenceException(
|
||||
"Kubernetes API не подтвердил содержимое после компенсации tenant Secret"
|
||||
);
|
||||
if (!prepareRetry(attempt, "проверка компенсации tenant Secret")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (response.statusCode() == 409) {
|
||||
log.warn("Компенсация tenant Secret пропущена: обнаружено конкурентное изменение");
|
||||
return TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE;
|
||||
}
|
||||
if (isRetryable(response.statusCode())) {
|
||||
TenantSecretCompensationResult reconciled = reconcileCompensation(context, receipt);
|
||||
if (reconciled != null) {
|
||||
return reconciled;
|
||||
}
|
||||
lastFailure = new TenantConfigPersistenceException(
|
||||
"Kubernetes API временно отклонил компенсацию tenant Secret: HTTP "
|
||||
+ response.statusCode()
|
||||
);
|
||||
if (!prepareRetry(attempt, "компенсация tenant Secret")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Kubernetes API отклонил компенсацию tenant Secret: HTTP " + response.statusCode()
|
||||
);
|
||||
}
|
||||
|
||||
throw lastFailure == null
|
||||
? new TenantConfigPersistenceException("Не удалось компенсировать tenant Secret")
|
||||
: lastFailure;
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt mutate(Function<List<TenantConfig>, List<TenantConfig>> mutation) {
|
||||
RequestContext context = requestContext();
|
||||
TenantConfigPersistenceException lastFailure = null;
|
||||
|
||||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
CurrentSecret current;
|
||||
try {
|
||||
current = readCurrent(context);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Изменение tenant Secret прервано", e);
|
||||
} catch (IOException e) {
|
||||
lastFailure = persistenceFailure("Не удалось прочитать актуальный tenant Secret", e);
|
||||
if (!prepareRetry(attempt, "чтение tenant Secret")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
List<TenantConfig> previous = normalizeTenants(current.tenants());
|
||||
List<TenantConfig> committed = normalizeTenants(mutation.apply(previous));
|
||||
if (sameTenants(previous, committed)) {
|
||||
return new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
false,
|
||||
current.resourceVersion(),
|
||||
previous,
|
||||
committed
|
||||
);
|
||||
}
|
||||
|
||||
HttpResponse<String> response;
|
||||
try {
|
||||
response = putSecret(context, current, committed);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Изменение tenant Secret прервано", e);
|
||||
} catch (IOException e) {
|
||||
TenantSecretUpdateReceipt reconciled = reconcileMutation(context, previous, committed);
|
||||
if (reconciled != null) {
|
||||
return reconciled;
|
||||
}
|
||||
lastFailure = persistenceFailure("Не удалось подтвердить изменение tenant Secret", e);
|
||||
if (!prepareRetry(attempt, "изменение после сетевой ошибки")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isSuccess(response.statusCode())) {
|
||||
TenantSecretUpdateReceipt confirmed = confirmMutation(
|
||||
context,
|
||||
current,
|
||||
previous,
|
||||
committed,
|
||||
response.body()
|
||||
);
|
||||
if (confirmed != null) {
|
||||
log.info("Tenant Secret атомарно изменён: tenantCount={}", committed.size());
|
||||
return confirmed;
|
||||
}
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Kubernetes API не подтвердил сохранённое содержимое tenant Secret"
|
||||
);
|
||||
}
|
||||
|
||||
if (response.statusCode() == 409) {
|
||||
lastFailure = new TenantConfigPersistenceException(
|
||||
"Исчерпаны попытки изменения tenant Secret из-за конкурентных обновлений"
|
||||
);
|
||||
if (!prepareRetry(attempt, "конфликт resourceVersion")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isRetryable(response.statusCode())) {
|
||||
TenantSecretUpdateReceipt reconciled = reconcileMutation(context, previous, committed);
|
||||
if (reconciled != null) {
|
||||
return reconciled;
|
||||
}
|
||||
lastFailure = new TenantConfigPersistenceException(
|
||||
"Kubernetes API временно отклонил изменение tenant Secret: HTTP "
|
||||
+ response.statusCode()
|
||||
);
|
||||
if (!prepareRetry(attempt, "изменение tenant Secret")) {
|
||||
throw lastFailure;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Kubernetes API отклонил изменение tenant Secret: HTTP " + response.statusCode()
|
||||
);
|
||||
}
|
||||
|
||||
throw lastFailure == null
|
||||
? new TenantConfigPersistenceException("Не удалось изменить tenant Secret")
|
||||
: lastFailure;
|
||||
}
|
||||
|
||||
private CurrentSecret readCurrent(RequestContext context) throws IOException, InterruptedException {
|
||||
HttpRequest request = requestBuilder(context)
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<String> response = secureClient().send(
|
||||
request,
|
||||
HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)
|
||||
);
|
||||
if (response.statusCode() == 200) {
|
||||
return parseSecret(response.body());
|
||||
}
|
||||
if (isRetryable(response.statusCode())) {
|
||||
throw new RetryableApiException(
|
||||
"Kubernetes API временно не вернул tenant Secret: HTTP " + response.statusCode()
|
||||
);
|
||||
}
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Kubernetes API не вернул tenant Secret: HTTP " + response.statusCode()
|
||||
);
|
||||
}
|
||||
|
||||
private HttpResponse<String> putSecret(RequestContext context,
|
||||
CurrentSecret current,
|
||||
List<TenantConfig> tenants)
|
||||
throws IOException, InterruptedException {
|
||||
ObjectNode body = current.secret().deepCopy();
|
||||
ObjectNode metadata = objectNode(body, "metadata");
|
||||
metadata.put("resourceVersion", current.resourceVersion());
|
||||
ObjectNode data = objectNode(body, "data");
|
||||
data.put("tenants.json", encodeTenants(tenants));
|
||||
|
||||
HttpRequest request = requestBuilder(context)
|
||||
.header("Content-Type", "application/json")
|
||||
.PUT(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body)))
|
||||
.build();
|
||||
return secureClient().send(
|
||||
request,
|
||||
HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)
|
||||
);
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt reconcileMutation(RequestContext context,
|
||||
List<TenantConfig> previous,
|
||||
List<TenantConfig> committed) {
|
||||
try {
|
||||
CurrentSecret observed = readCurrent(context);
|
||||
if (sameTenants(observed.tenants(), committed)) {
|
||||
log.info("Изменение tenant Secret подтверждено повторным чтением: tenantCount={}",
|
||||
committed.size());
|
||||
return new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
false,
|
||||
observed.resourceVersion(),
|
||||
previous,
|
||||
observed.tenants()
|
||||
);
|
||||
}
|
||||
return null;
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Проверка изменения tenant Secret прервана", e);
|
||||
} catch (IOException e) {
|
||||
log.warn("Не удалось подтвердить состояние tenant Secret повторным чтением");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt confirmMutation(RequestContext context,
|
||||
CurrentSecret previousSecret,
|
||||
List<TenantConfig> previous,
|
||||
List<TenantConfig> committed,
|
||||
String responseBody) {
|
||||
try {
|
||||
CurrentSecret confirmed = parseSecret(responseBody);
|
||||
if (sameTenants(confirmed.tenants(), committed)
|
||||
&& !Objects.equals(confirmed.resourceVersion(), previousSecret.resourceVersion())) {
|
||||
return new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
true,
|
||||
confirmed.resourceVersion(),
|
||||
previous,
|
||||
confirmed.tenants()
|
||||
);
|
||||
}
|
||||
log.warn("Ответ Kubernetes API после изменения tenant Secret не совпал с ожидаемым состоянием");
|
||||
} catch (TenantConfigPersistenceException invalidResponse) {
|
||||
log.warn("Kubernetes API вернул неполный ответ после изменения tenant Secret, выполняется повторное чтение");
|
||||
log.debug("Технические детали неполного ответа tenant Secret", invalidResponse);
|
||||
}
|
||||
return reconcileMutation(context, previous, committed);
|
||||
}
|
||||
|
||||
private TenantSecretCompensationResult confirmCompensation(RequestContext context,
|
||||
TenantSecretUpdateReceipt receipt,
|
||||
String responseBody) {
|
||||
try {
|
||||
CurrentSecret confirmed = parseSecret(responseBody);
|
||||
if (sameTenants(confirmed.tenants(), receipt.previousTenants())) {
|
||||
return TenantSecretCompensationResult.RESTORED;
|
||||
}
|
||||
log.warn("Ответ Kubernetes API после компенсации tenant Secret не совпал с ожидаемым состоянием");
|
||||
} catch (TenantConfigPersistenceException invalidResponse) {
|
||||
log.warn("Kubernetes API вернул неполный ответ после компенсации tenant Secret, выполняется повторное чтение");
|
||||
log.debug("Технические детали неполного ответа компенсации tenant Secret", invalidResponse);
|
||||
}
|
||||
return reconcileCompensation(context, receipt);
|
||||
}
|
||||
|
||||
private TenantSecretCompensationResult reconcileCompensation(RequestContext context,
|
||||
TenantSecretUpdateReceipt receipt) {
|
||||
try {
|
||||
CurrentSecret observed = readCurrent(context);
|
||||
if (sameTenants(observed.tenants(), receipt.previousTenants())) {
|
||||
log.info("Компенсация tenant Secret подтверждена повторным чтением");
|
||||
return TenantSecretCompensationResult.RESTORED;
|
||||
}
|
||||
if (!Objects.equals(observed.resourceVersion(), receipt.committedResourceVersion())) {
|
||||
log.warn("Компенсация tenant Secret остановлена из-за конкурентного изменения");
|
||||
return TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE;
|
||||
}
|
||||
return null;
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Проверка компенсации tenant Secret прервана", e);
|
||||
} catch (IOException e) {
|
||||
log.warn("Не удалось подтвердить компенсацию tenant Secret повторным чтением");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private CurrentSecret parseSecret(String responseBody) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(responseBody);
|
||||
if (!(root instanceof ObjectNode secret)) {
|
||||
throw new TenantConfigPersistenceException("Kubernetes API вернул некорректный tenant Secret");
|
||||
}
|
||||
String resourceVersion = requireText(
|
||||
root.path("metadata").path("resourceVersion").asText(null),
|
||||
"В tenant Secret отсутствует metadata.resourceVersion"
|
||||
);
|
||||
JsonNode data = root.path("data");
|
||||
if (!data.isObject() || !data.hasNonNull("tenants.json")) {
|
||||
throw new TenantConfigPersistenceException(
|
||||
"В tenant Secret отсутствует обязательный ключ data.tenants.json"
|
||||
);
|
||||
}
|
||||
String encoded = requireText(
|
||||
data.path("tenants.json").asText(null),
|
||||
"В tenant Secret ключ data.tenants.json не содержит конфигурацию"
|
||||
);
|
||||
byte[] decoded = Base64.getDecoder().decode(encoded);
|
||||
List<TenantConfig> tenants = objectMapper.readValue(decoded, TENANT_LIST_TYPE);
|
||||
return new CurrentSecret(secret, resourceVersion, normalizeTenants(tenants));
|
||||
} catch (TenantConfigPersistenceException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Не удалось разобрать конфигурацию из tenant Secret",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private List<TenantConfig> normalizeTenants(List<TenantConfig> tenants) {
|
||||
if (tenants == null) {
|
||||
throw new TenantConfigPersistenceException("Список конфигураций в tenant Secret не задан");
|
||||
}
|
||||
List<TenantConfig> normalized = new ArrayList<>(tenants.size());
|
||||
Set<String> domains = new HashSet<>();
|
||||
for (TenantConfig source : tenants) {
|
||||
TenantConfig tenant = TenantSecretUpdateReceipt.copyOf(
|
||||
Objects.requireNonNull(source, "В tenant Secret обнаружена пустая конфигурация")
|
||||
);
|
||||
String domain = normalizeDomain(tenant.getDomain());
|
||||
tenant.setDomain(domain);
|
||||
if (!domains.add(domain)) {
|
||||
throw new TenantConfigPersistenceException(
|
||||
"В tenant Secret обнаружены повторяющиеся домены"
|
||||
);
|
||||
}
|
||||
normalized.add(tenant);
|
||||
}
|
||||
normalized.sort(Comparator.comparing(TenantConfig::getDomain));
|
||||
return List.copyOf(normalized);
|
||||
}
|
||||
|
||||
private boolean sameTenants(List<TenantConfig> first, List<TenantConfig> second) {
|
||||
List<TenantConfig> left = normalizeTenants(first);
|
||||
List<TenantConfig> right = normalizeTenants(second);
|
||||
if (left.size() != right.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < left.size(); i++) {
|
||||
if (!sameTenant(left.get(i), right.get(i))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean sameTenant(TenantConfig first, TenantConfig second) {
|
||||
return Objects.equals(first.getName(), second.getName())
|
||||
&& Objects.equals(first.getDomain(), second.getDomain())
|
||||
&& Objects.equals(first.getUrl(), second.getUrl())
|
||||
&& Objects.equals(first.getUsername(), second.getUsername())
|
||||
&& Objects.equals(first.getPassword(), second.getPassword());
|
||||
}
|
||||
|
||||
private RequestContext requestContext() {
|
||||
try {
|
||||
String token = requireText(
|
||||
Files.readString(tokenPath).trim(),
|
||||
"ServiceAccount token не настроен"
|
||||
);
|
||||
String namespace = requireText(
|
||||
Files.readString(namespacePath).trim(),
|
||||
"ServiceAccount namespace не настроен"
|
||||
);
|
||||
URI uri = URI.create(String.format(
|
||||
"%s/api/v1/namespaces/%s/secrets/%s",
|
||||
apiBase,
|
||||
namespace,
|
||||
secretName
|
||||
));
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(uri)
|
||||
.header("Authorization", "Bearer " + token)
|
||||
.header("Content-Type", "application/strategic-merge-patch+json")
|
||||
.method("PATCH", HttpRequest.BodyPublishers.ofString(patchBody))
|
||||
.build();
|
||||
|
||||
HttpResponse<Void> response = createSecureClient(caPath)
|
||||
.send(request, HttpResponse.BodyHandlers.discarding());
|
||||
if (response.statusCode() == 200) {
|
||||
log.info("Tenant Secret успешно обновлён: tenantCount={}", tenants.size());
|
||||
return true;
|
||||
}
|
||||
|
||||
log.error("Kubernetes API отклонил обновление tenant Secret: httpStatus={}", response.statusCode());
|
||||
return false;
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.error("Обновление tenant Secret прервано");
|
||||
log.debug("Технические детали прерывания tenant Secret", e);
|
||||
return false;
|
||||
return new RequestContext(token, uri);
|
||||
} catch (TenantConfigPersistenceException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.error("Не удалось безопасно обновить tenant Secret: errorType={}",
|
||||
e.getClass().getSimpleName());
|
||||
log.debug("Технические детали ошибки tenant Secret", e);
|
||||
return false;
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Не удалось прочитать параметры ServiceAccount для tenant Secret",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private HttpRequest.Builder requestBuilder(RequestContext context) {
|
||||
return HttpRequest.newBuilder()
|
||||
.uri(context.uri())
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.header("Authorization", "Bearer " + context.token())
|
||||
.header("Accept", "application/json");
|
||||
}
|
||||
|
||||
private synchronized HttpClient secureClient() {
|
||||
if (httpClient == null) {
|
||||
try {
|
||||
httpClient = createSecureClient(caPath);
|
||||
} catch (Exception e) {
|
||||
throw new TenantConfigPersistenceException(
|
||||
"Не удалось настроить защищённое соединение с Kubernetes API",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
HttpClient createSecureClient(Path serviceAccountCaPath) throws Exception {
|
||||
CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
|
||||
Collection<? extends Certificate> certificates;
|
||||
@@ -171,8 +691,102 @@ public class KubernetesTenantSecretUpdater {
|
||||
sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
|
||||
|
||||
return HttpClient.newBuilder()
|
||||
.connectTimeout(CONNECT_TIMEOUT)
|
||||
.sslContext(sslContext)
|
||||
.sslParameters(sslParameters)
|
||||
.build();
|
||||
}
|
||||
|
||||
private boolean prepareRetry(int attempt, String operation) {
|
||||
if (attempt >= maxAttempts) {
|
||||
return false;
|
||||
}
|
||||
log.warn("Временная ошибка tenant Secret, операция будет повторена: операция={}, попытка={}/{}",
|
||||
operation,
|
||||
attempt,
|
||||
maxAttempts);
|
||||
try {
|
||||
long delayMillis = Math.multiplyExact(retryDelay.toMillis(), attempt);
|
||||
if (delayMillis > 0) {
|
||||
Thread.sleep(delayMillis);
|
||||
}
|
||||
return true;
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new TenantConfigPersistenceException("Ожидание повтора tenant Secret прервано", e);
|
||||
} catch (ArithmeticException e) {
|
||||
throw new TenantConfigPersistenceException("Некорректная задержка повтора tenant Secret", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String encodeTenants(List<TenantConfig> tenants) throws IOException {
|
||||
String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(tenants);
|
||||
return Base64.getEncoder().encodeToString(json.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
private ObjectNode objectNode(ObjectNode parent, String fieldName) {
|
||||
JsonNode existing = parent.get(fieldName);
|
||||
if (existing instanceof ObjectNode objectNode) {
|
||||
return objectNode;
|
||||
}
|
||||
ObjectNode created = objectMapper.createObjectNode();
|
||||
parent.set(fieldName, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
private static boolean isSuccess(int statusCode) {
|
||||
return statusCode >= 200 && statusCode < 300;
|
||||
}
|
||||
|
||||
private static boolean isRetryable(int statusCode) {
|
||||
return statusCode == 408 || statusCode == 409 || statusCode == 429 || statusCode >= 500;
|
||||
}
|
||||
|
||||
private static boolean detectKubernetesEnvironment(Path tokenPath) {
|
||||
String kubernetesServiceHost = System.getenv("KUBERNETES_SERVICE_HOST");
|
||||
return (tokenPath != null && Files.exists(tokenPath))
|
||||
|| (kubernetesServiceHost != null && !kubernetesServiceHost.isBlank());
|
||||
}
|
||||
|
||||
private static String requireText(String value, String message) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new TenantConfigPersistenceException(message);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private static String normalizeDomain(String value) {
|
||||
String domain = requireText(value, "Домен тенанта не задан").toLowerCase(Locale.ROOT);
|
||||
if (!DOMAIN_PATTERN.matcher(domain).matches()) {
|
||||
throw new TenantConfigPersistenceException("Домен тенанта содержит недопустимые символы");
|
||||
}
|
||||
return domain;
|
||||
}
|
||||
|
||||
private static String normalizeApiBase(String apiBase) {
|
||||
String normalized = requireText(apiBase, "Адрес Kubernetes API не задан");
|
||||
return normalized.endsWith("/")
|
||||
? normalized.substring(0, normalized.length() - 1)
|
||||
: normalized;
|
||||
}
|
||||
|
||||
private static TenantConfigPersistenceException persistenceFailure(String message, Exception cause) {
|
||||
return cause instanceof TenantConfigPersistenceException persistenceException
|
||||
? persistenceException
|
||||
: new TenantConfigPersistenceException(message, cause);
|
||||
}
|
||||
|
||||
private record RequestContext(String token, URI uri) {
|
||||
}
|
||||
|
||||
private record CurrentSecret(ObjectNode secret,
|
||||
String resourceVersion,
|
||||
List<TenantConfig> tenants) {
|
||||
}
|
||||
|
||||
private static final class RetryableApiException extends IOException {
|
||||
private RetryableApiException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
/**
|
||||
* Ошибка безопасного чтения или изменения постоянной конфигурации тенантов.
|
||||
*/
|
||||
public class TenantConfigPersistenceException extends RuntimeException {
|
||||
|
||||
public TenantConfigPersistenceException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public TenantConfigPersistenceException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
/**
|
||||
* Хранилище конфигурации тенантов с защитой от потерянных конкурентных обновлений.
|
||||
*/
|
||||
public interface TenantConfigStore {
|
||||
|
||||
TenantSecretUpdateReceipt upsertTenant(TenantConfig tenant);
|
||||
|
||||
TenantSecretUpdateReceipt removeTenant(String domain);
|
||||
|
||||
TenantSecretCompensationResult compensate(TenantSecretUpdateReceipt receipt);
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package com.magistr.app.config.tenant;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -29,16 +30,23 @@ public class TenantConfigWatcher {
|
||||
private static final Logger log = LoggerFactory.getLogger(TenantConfigWatcher.class);
|
||||
|
||||
private final TenantLifecycleService tenantLifecycleService;
|
||||
private final TenantReadinessRegistry readinessRegistry;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Value("${app.tenants.config-path:tenants.json}")
|
||||
private String tenantsConfigPath;
|
||||
|
||||
@Value("${app.tenants.config-required:false}")
|
||||
private boolean tenantsConfigRequired;
|
||||
|
||||
// Хеш последнего прочитанного конфига — чтобы не перезагружать зря
|
||||
private String lastConfigHash = "";
|
||||
private boolean configurationUnavailable;
|
||||
|
||||
public TenantConfigWatcher(TenantLifecycleService tenantLifecycleService) {
|
||||
public TenantConfigWatcher(TenantLifecycleService tenantLifecycleService,
|
||||
TenantReadinessRegistry readinessRegistry) {
|
||||
this.tenantLifecycleService = tenantLifecycleService;
|
||||
this.readinessRegistry = readinessRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -52,12 +60,21 @@ public class TenantConfigWatcher {
|
||||
private void watchForChangesSerialized() {
|
||||
try {
|
||||
File file = new File(tenantsConfigPath);
|
||||
if (!file.exists()) return;
|
||||
if (!file.exists()) {
|
||||
if (tenantsConfigRequired) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
if (!configurationUnavailable) {
|
||||
log.error("Обязательный файл конфигурации тенантов недоступен");
|
||||
}
|
||||
configurationUnavailable = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
String content = new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
|
||||
String hash = configHash(content);
|
||||
|
||||
if (hash.equals(lastConfigHash)) {
|
||||
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
|
||||
return; // Ничего не изменилось
|
||||
}
|
||||
|
||||
@@ -65,8 +82,11 @@ public class TenantConfigWatcher {
|
||||
List<TenantConfig> newTenants = objectMapper.readValue(content, new TypeReference<>() {});
|
||||
syncTenants(newTenants);
|
||||
lastConfigHash = hash;
|
||||
configurationUnavailable = false;
|
||||
|
||||
} catch (Exception e) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
log.error("Ошибка при проверке конфига тенантов: errorType={}", e.getClass().getSimpleName());
|
||||
log.debug("Технические детали синхронизации конфига тенантов", e);
|
||||
}
|
||||
@@ -85,8 +105,13 @@ public class TenantConfigWatcher {
|
||||
if (file.exists()) {
|
||||
String content = new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
|
||||
lastConfigHash = configHash(content);
|
||||
} else if (tenantsConfigRequired) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
log.warn("Не удалось обновить хеш конфига тенантов: errorType={}",
|
||||
e.getClass().getSimpleName());
|
||||
log.debug("Технические детали обновления хеша тенантов", e);
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.magistr.app.config.tenant;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.magistr.app.service.TenantDatabaseMigrationService;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.slf4j.Logger;
|
||||
@@ -18,13 +19,12 @@ import org.springframework.transaction.PlatformTransactionManager;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import javax.sql.DataSource;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Конфигурация мультитенантного DataSource.
|
||||
* Загружает тенанты из JSON-файла (mounted ConfigMap).
|
||||
* Загружает тенанты из JSON-файла (смонтированный Kubernetes Secret).
|
||||
*
|
||||
* Если нет ни одного настроенного тенанта — создаёт H2 in-memory БД
|
||||
* как заглушку, чтобы Spring JPA мог инициализироваться.
|
||||
@@ -37,6 +37,9 @@ public class TenantDataSourceConfig {
|
||||
@Value("${app.tenants.config-path:tenants.json}")
|
||||
private String tenantsConfigPath;
|
||||
|
||||
@Value("${app.tenants.config-required:false}")
|
||||
private boolean tenantsConfigRequired;
|
||||
|
||||
@Value("${spring.datasource.url:}")
|
||||
private String defaultDbUrl;
|
||||
|
||||
@@ -48,11 +51,13 @@ public class TenantDataSourceConfig {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public DataSource dataSource(TenantDatabaseMigrationService migrationService) {
|
||||
public DataSource dataSource(TenantDatabaseMigrationService migrationService,
|
||||
TenantReadinessRegistry readinessRegistry) {
|
||||
TenantRoutingDataSource routingDataSource = new TenantRoutingDataSource();
|
||||
|
||||
// Загружаем тенантов из JSON (read-only ConfigMap mount)
|
||||
List<TenantConfig> tenants = loadTenantsFromFile();
|
||||
// Загружаем тенантов из JSON (read-only Secret mount)
|
||||
TenantConfigLoadResult loadResult = loadTenantsFromFile();
|
||||
List<TenantConfig> tenants = new ArrayList<>(loadResult.tenants());
|
||||
|
||||
// Если нет тенантов и есть дефолтный datasource — создаём "default" тенант
|
||||
if (tenants.isEmpty() && defaultDbUrl != null && !defaultDbUrl.isBlank()) {
|
||||
@@ -63,9 +68,15 @@ public class TenantDataSourceConfig {
|
||||
log.info("Конфигурация тенантов отсутствует, используется default DataSource");
|
||||
}
|
||||
|
||||
readinessRegistry.clearConfigurationFailure();
|
||||
readinessRegistry.replaceDesired(tenants);
|
||||
if (loadResult.failed() || (tenantsConfigRequired && loadResult.tenants().isEmpty())) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
}
|
||||
|
||||
// Регистрируем тенантов
|
||||
for (TenantConfig tenant : tenants) {
|
||||
registerPreparedTenant(routingDataSource, migrationService, tenant);
|
||||
registerPreparedTenant(routingDataSource, migrationService, readinessRegistry, tenant, true);
|
||||
}
|
||||
|
||||
// Если всё ещё нет ни одного тенанта — H2 in-memory заглушка
|
||||
@@ -79,7 +90,13 @@ public class TenantDataSourceConfig {
|
||||
"jdbc:h2:mem:placeholder;DB_CLOSE_DELAY=-1",
|
||||
"sa", ""
|
||||
);
|
||||
if (!registerPreparedTenant(routingDataSource, migrationService, h2Fallback)) {
|
||||
if (!registerPreparedTenant(
|
||||
routingDataSource,
|
||||
migrationService,
|
||||
readinessRegistry,
|
||||
h2Fallback,
|
||||
false
|
||||
)) {
|
||||
throw new IllegalStateException("Не удалось создать резервный H2 DataSource");
|
||||
}
|
||||
}
|
||||
@@ -118,31 +135,43 @@ public class TenantDataSourceConfig {
|
||||
return new JpaTransactionManager(emf);
|
||||
}
|
||||
|
||||
private List<TenantConfig> loadTenantsFromFile() {
|
||||
private TenantConfigLoadResult loadTenantsFromFile() {
|
||||
File file = new File(tenantsConfigPath);
|
||||
if (!file.exists()) {
|
||||
log.info("Файл конфигурации тенантов не найден");
|
||||
return new ArrayList<>();
|
||||
if (tenantsConfigRequired) {
|
||||
log.error("Обязательный файл конфигурации тенантов не найден");
|
||||
} else {
|
||||
log.info("Файл конфигурации тенантов не найден");
|
||||
}
|
||||
return new TenantConfigLoadResult(List.of(), tenantsConfigRequired);
|
||||
}
|
||||
|
||||
try {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
List<TenantConfig> list = mapper.readValue(file, new TypeReference<>() {});
|
||||
if (list == null) {
|
||||
throw new IllegalArgumentException("Корневое значение конфигурации тенантов должно быть массивом");
|
||||
}
|
||||
log.info("Загружено конфигураций тенантов: {}", list.size());
|
||||
return list;
|
||||
} catch (IOException e) {
|
||||
return new TenantConfigLoadResult(List.copyOf(list), false);
|
||||
} catch (Exception e) {
|
||||
log.error("Не удалось прочитать конфигурацию тенантов: errorType={}",
|
||||
e.getClass().getSimpleName());
|
||||
log.debug("Технические детали чтения конфигурации тенантов", e);
|
||||
return new ArrayList<>();
|
||||
return new TenantConfigLoadResult(List.of(), true);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean registerPreparedTenant(TenantRoutingDataSource routingDataSource,
|
||||
TenantDatabaseMigrationService migrationService,
|
||||
TenantConfig tenant) {
|
||||
TenantReadinessRegistry readinessRegistry,
|
||||
TenantConfig tenant,
|
||||
boolean requiredForReadiness) {
|
||||
HikariDataSource candidate = null;
|
||||
boolean activated = false;
|
||||
if (requiredForReadiness) {
|
||||
readinessRegistry.markMigrationStarted(tenant);
|
||||
}
|
||||
try {
|
||||
candidate = routingDataSource.prepareTenantDataSource(tenant);
|
||||
try (Connection connection = candidate.getConnection()) {
|
||||
@@ -155,10 +184,17 @@ public class TenantDataSourceConfig {
|
||||
}
|
||||
TenantRoutingDataSource.TenantState previous = routingDataSource.swapTenant(tenant, candidate);
|
||||
activated = true;
|
||||
if (requiredForReadiness) {
|
||||
readinessRegistry.markMigrationSucceeded(tenant);
|
||||
readinessRegistry.markConnectivity(tenant.getDomain(), true);
|
||||
}
|
||||
closeDataSource(previous == null ? null : previous.dataSource());
|
||||
log.info("Tenant-БД '{}' проверена и активирована", tenant.getDomain());
|
||||
return true;
|
||||
} catch (Exception startupFailure) {
|
||||
if (requiredForReadiness) {
|
||||
readinessRegistry.markMigrationFailed(tenant);
|
||||
}
|
||||
log.error("Не удалось безопасно активировать tenant-БД '{}': errorType={}",
|
||||
tenant.getDomain(), startupFailure.getClass().getSimpleName());
|
||||
log.debug("Технические детали startup lifecycle tenant-БД", startupFailure);
|
||||
@@ -181,4 +217,7 @@ public class TenantDataSourceConfig {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private record TenantConfigLoadResult(List<TenantConfig> tenants, boolean failed) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
/**
|
||||
* Результат компенсирующего изменения конфигурации тенантов.
|
||||
*/
|
||||
public enum TenantSecretCompensationResult {
|
||||
RESTORED,
|
||||
NOT_REQUIRED,
|
||||
SKIPPED_CONCURRENT_CHANGE
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Квитанция об атомарном изменении tenant Secret.
|
||||
*
|
||||
* <p>Снимки конфигурации копируются глубоко и сортируются по домену. Это позволяет
|
||||
* безопасно использовать квитанцию для компенсации и проверки ожидаемого состояния.</p>
|
||||
*/
|
||||
public final class TenantSecretUpdateReceipt {
|
||||
|
||||
private static final Comparator<TenantConfig> BY_DOMAIN = Comparator.comparing(
|
||||
TenantConfig::getDomain,
|
||||
Comparator.nullsFirst(String::compareTo)
|
||||
);
|
||||
|
||||
private final boolean persisted;
|
||||
private final boolean changed;
|
||||
private final String committedResourceVersion;
|
||||
private final List<TenantConfig> previousTenants;
|
||||
private final List<TenantConfig> committedTenants;
|
||||
|
||||
public TenantSecretUpdateReceipt(boolean persisted,
|
||||
boolean changed,
|
||||
String committedResourceVersion,
|
||||
List<TenantConfig> previousTenants,
|
||||
List<TenantConfig> committedTenants) {
|
||||
this.persisted = persisted;
|
||||
this.changed = changed;
|
||||
this.committedResourceVersion = committedResourceVersion;
|
||||
this.previousTenants = copyAndSort(previousTenants);
|
||||
this.committedTenants = copyAndSort(committedTenants);
|
||||
}
|
||||
|
||||
public boolean persisted() {
|
||||
return persisted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Возвращает {@code true}, только если хранилище доказало владение подтверждённой
|
||||
* записанной версией. После неоднозначного результата повторное чтение может подтвердить
|
||||
* целевое состояние, но оставляет этот признак {@code false}, чтобы запретить опасную
|
||||
* автоматическую компенсацию.
|
||||
*/
|
||||
public boolean changed() {
|
||||
return changed;
|
||||
}
|
||||
|
||||
public String committedResourceVersion() {
|
||||
return committedResourceVersion;
|
||||
}
|
||||
|
||||
public List<TenantConfig> previousTenants() {
|
||||
return copyAndSort(previousTenants);
|
||||
}
|
||||
|
||||
public List<TenantConfig> committedTenants() {
|
||||
return copyAndSort(committedTenants);
|
||||
}
|
||||
|
||||
static List<TenantConfig> copyAndSort(List<TenantConfig> tenants) {
|
||||
Objects.requireNonNull(tenants, "Список конфигураций тенантов не задан");
|
||||
List<TenantConfig> result = new ArrayList<>(tenants.size());
|
||||
for (TenantConfig tenant : tenants) {
|
||||
result.add(copyOf(Objects.requireNonNull(tenant, "Конфигурация тенанта не задана")));
|
||||
}
|
||||
result.sort(BY_DOMAIN);
|
||||
return List.copyOf(result);
|
||||
}
|
||||
|
||||
static TenantConfig copyOf(TenantConfig source) {
|
||||
return new TenantConfig(
|
||||
source.getName(),
|
||||
source.getDomain(),
|
||||
source.getUrl(),
|
||||
source.getUsername(),
|
||||
source.getPassword()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TenantSecretUpdateReceipt{" +
|
||||
"persisted=" + persisted +
|
||||
", changed=" + changed +
|
||||
", committedResourceVersion='" + committedResourceVersion + '\'' +
|
||||
", previousTenantCount=" + previousTenants.size() +
|
||||
", committedTenantCount=" + committedTenants.size() +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,9 @@ public class TenantWebMvcConfig implements WebMvcConfigurer {
|
||||
|
||||
@Override
|
||||
public void addInterceptors(InterceptorRegistry registry) {
|
||||
registry.addInterceptor(tenantInterceptor()).addPathPatterns("/**");
|
||||
registry.addInterceptor(tenantInterceptor())
|
||||
.addPathPatterns("/**")
|
||||
.excludePathPatterns("/actuator/**");
|
||||
registry.addInterceptor(authorizationInterceptor).addPathPatterns("/api/**");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* Фоновая проверка доступности tenant-БД с ограниченным числом параллельных задач.
|
||||
*/
|
||||
@Component
|
||||
public class TenantDatabaseHealthMonitor {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TenantDatabaseHealthMonitor.class);
|
||||
|
||||
private final TenantReadinessRegistry registry;
|
||||
private final TenantRoutingDataSource routingDataSource;
|
||||
private final int parallelism;
|
||||
private final Duration checkTimeout;
|
||||
private final Duration maxStaleness;
|
||||
private final Clock clock;
|
||||
private final ExecutorService executor;
|
||||
private final AtomicBoolean refreshInProgress = new AtomicBoolean(false);
|
||||
|
||||
public TenantDatabaseHealthMonitor(
|
||||
TenantReadinessRegistry registry,
|
||||
TenantRoutingDataSource routingDataSource,
|
||||
@Value("${app.health.tenant-database.parallelism:4}") int parallelism,
|
||||
@Value("${app.health.tenant-database.check-timeout-ms:6000}") long checkTimeoutMillis,
|
||||
@Value("${app.health.tenant-database.max-staleness-ms:30000}") long maxStalenessMillis) {
|
||||
this(registry, routingDataSource, parallelism,
|
||||
Duration.ofMillis(checkTimeoutMillis),
|
||||
Duration.ofMillis(maxStalenessMillis),
|
||||
Clock.systemUTC());
|
||||
}
|
||||
|
||||
public TenantDatabaseHealthMonitor(TenantReadinessRegistry registry,
|
||||
TenantRoutingDataSource routingDataSource,
|
||||
int parallelism,
|
||||
Duration checkTimeout,
|
||||
Duration maxStaleness,
|
||||
Clock clock) {
|
||||
this.registry = Objects.requireNonNull(registry, "Реестр готовности обязателен");
|
||||
this.routingDataSource = Objects.requireNonNull(routingDataSource, "Маршрутизатор tenant-БД обязателен");
|
||||
if (parallelism < 1 || parallelism > 32) {
|
||||
throw new IllegalArgumentException("Параллелизм проверки должен быть от 1 до 32");
|
||||
}
|
||||
this.parallelism = parallelism;
|
||||
this.checkTimeout = requirePositive(checkTimeout, "Таймаут проверки должен быть положительным");
|
||||
this.maxStaleness = requirePositive(maxStaleness, "Максимальный возраст проверки должен быть положительным");
|
||||
this.clock = Objects.requireNonNull(clock, "Часы монитора обязательны");
|
||||
this.executor = Executors.newFixedThreadPool(parallelism, new HealthThreadFactory());
|
||||
}
|
||||
|
||||
@Scheduled(
|
||||
fixedDelayString = "${app.health.tenant-database.interval-ms:10000}",
|
||||
initialDelayString = "${app.health.tenant-database.initial-delay-ms:1000}")
|
||||
public void refreshScheduled() {
|
||||
refreshNow();
|
||||
}
|
||||
|
||||
/**
|
||||
* Выполняет немедленный проход. Одновременно может выполняться только один проход.
|
||||
*/
|
||||
public RefreshSummary refreshNow() {
|
||||
if (!refreshInProgress.compareAndSet(false, true)) {
|
||||
return new RefreshSummary(0, 0, 0, true);
|
||||
}
|
||||
|
||||
try {
|
||||
List<TenantReadinessRegistry.ConnectivityTarget> targets = registry.connectivityTargets();
|
||||
int available = 0;
|
||||
int unavailable = 0;
|
||||
|
||||
for (int offset = 0; offset < targets.size(); offset += parallelism) {
|
||||
int end = Math.min(offset + parallelism, targets.size());
|
||||
List<TenantReadinessRegistry.ConnectivityTarget> batch = targets.subList(offset, end);
|
||||
List<Callable<CheckResult>> checks = new ArrayList<>(batch.size());
|
||||
|
||||
for (TenantReadinessRegistry.ConnectivityTarget target : batch) {
|
||||
checks.add(() -> check(target));
|
||||
}
|
||||
|
||||
List<Future<CheckResult>> futures;
|
||||
try {
|
||||
futures = executor.invokeAll(checks, checkTimeout.toNanos(), TimeUnit.NANOSECONDS);
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
Instant interruptedAt = clock.instant();
|
||||
log.warn("Пакетная проверка tenant-БД была прервана");
|
||||
for (int remaining = offset; remaining < targets.size(); remaining++) {
|
||||
TenantReadinessRegistry.ConnectivityTarget target = targets.get(remaining);
|
||||
if (registry.markConnectivity(
|
||||
target.domain(), target.fingerprint(), false, interruptedAt)) {
|
||||
unavailable++;
|
||||
}
|
||||
}
|
||||
return new RefreshSummary(targets.size(), available, unavailable, false);
|
||||
}
|
||||
|
||||
for (int index = 0; index < batch.size(); index++) {
|
||||
TenantReadinessRegistry.ConnectivityTarget target = batch.get(index);
|
||||
CheckResult result = completedResult(target, futures.get(index));
|
||||
boolean applied = registry.markConnectivity(
|
||||
target.domain(), target.fingerprint(), result.connected(), result.checkedAt());
|
||||
if (applied && result.connected()) {
|
||||
available++;
|
||||
} else if (applied) {
|
||||
unavailable++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new RefreshSummary(targets.size(), available, unavailable, false);
|
||||
} finally {
|
||||
refreshInProgress.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
public Duration maxStaleness() {
|
||||
return maxStaleness;
|
||||
}
|
||||
|
||||
public int parallelism() {
|
||||
return parallelism;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
private CheckResult check(TenantReadinessRegistry.ConnectivityTarget target) {
|
||||
try {
|
||||
boolean connected = routingDataSource.testConnection(target.domain());
|
||||
return new CheckResult(connected, clock.instant());
|
||||
} catch (RuntimeException checkFailure) {
|
||||
log.warn("Проверка tenant-БД '{}' завершилась ошибкой: errorType={}",
|
||||
target.domain(), checkFailure.getClass().getSimpleName());
|
||||
log.debug("Технические детали фоновой проверки tenant-БД", checkFailure);
|
||||
return new CheckResult(false, clock.instant());
|
||||
}
|
||||
}
|
||||
|
||||
private CheckResult completedResult(TenantReadinessRegistry.ConnectivityTarget target,
|
||||
Future<CheckResult> future) {
|
||||
if (future.isCancelled()) {
|
||||
log.warn("Проверка tenant-БД '{}' превысила допустимое время", target.domain());
|
||||
return new CheckResult(false, clock.instant());
|
||||
}
|
||||
|
||||
try {
|
||||
return future.get();
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
future.cancel(true);
|
||||
log.warn("Проверка tenant-БД '{}' была прервана", target.domain());
|
||||
return new CheckResult(false, clock.instant());
|
||||
} catch (ExecutionException failure) {
|
||||
log.warn("Не удалось проверить tenant-БД '{}': errorType={}",
|
||||
target.domain(), failure.getCause() == null
|
||||
? failure.getClass().getSimpleName()
|
||||
: failure.getCause().getClass().getSimpleName());
|
||||
return new CheckResult(false, clock.instant());
|
||||
}
|
||||
}
|
||||
|
||||
private Duration requirePositive(Duration value, String message) {
|
||||
Objects.requireNonNull(value, message);
|
||||
if (value.isNegative() || value.isZero()) {
|
||||
throw new IllegalArgumentException(message);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public record RefreshSummary(int requested,
|
||||
int available,
|
||||
int unavailable,
|
||||
boolean skipped) {
|
||||
}
|
||||
|
||||
private record CheckResult(boolean connected, Instant checkedAt) {
|
||||
private CheckResult {
|
||||
Objects.requireNonNull(checkedAt, "Время завершения проверки tenant-БД обязательно");
|
||||
}
|
||||
}
|
||||
|
||||
private static final class HealthThreadFactory implements ThreadFactory {
|
||||
private final AtomicInteger sequence = new AtomicInteger();
|
||||
|
||||
@Override
|
||||
public Thread newThread(Runnable task) {
|
||||
Thread thread = new Thread(task,
|
||||
"проверка-готовности-tenant-бд-" + sequence.incrementAndGet());
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import org.springframework.boot.actuate.health.Health;
|
||||
import org.springframework.boot.actuate.health.HealthIndicator;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Агрегированный индикатор readiness без раскрытия доменов и реквизитов tenant-БД.
|
||||
*/
|
||||
@Component("tenantReadiness")
|
||||
public class TenantReadinessHealthIndicator implements HealthIndicator {
|
||||
|
||||
private final TenantReadinessRegistry registry;
|
||||
private final TenantDatabaseHealthMonitor monitor;
|
||||
|
||||
public TenantReadinessHealthIndicator(TenantReadinessRegistry registry,
|
||||
TenantDatabaseHealthMonitor monitor) {
|
||||
this.registry = Objects.requireNonNull(registry, "Реестр готовности обязателен");
|
||||
this.monitor = Objects.requireNonNull(monitor, "Монитор tenant-БД обязателен");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Health health() {
|
||||
return readiness().ready()
|
||||
? Health.up().build()
|
||||
: Health.down().build();
|
||||
}
|
||||
|
||||
public TenantReadinessRegistry.AggregateReadiness readiness() {
|
||||
return registry.aggregate(monitor.maxStaleness());
|
||||
}
|
||||
|
||||
public boolean isReady() {
|
||||
return readiness().ready();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.SecureRandom;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.UnaryOperator;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Потокобезопасный реестр готовности обязательных tenant-БД.
|
||||
*
|
||||
* <p>Каждое изменение публикуется единым неизменяемым снимком. Результат фоновой
|
||||
* проверки соединения привязан к непрозрачному fingerprint конфигурации, поэтому
|
||||
* запоздалый результат для старых реквизитов не изменит состояние новой конфигурации.</p>
|
||||
*/
|
||||
@Component
|
||||
public class TenantReadinessRegistry {
|
||||
|
||||
private static final String FINGERPRINT_ALGORITHM = "HmacSHA256";
|
||||
private static final Pattern DOMAIN_PATTERN = Pattern.compile(
|
||||
"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"
|
||||
);
|
||||
|
||||
private final AtomicReference<RegistryState> state = new AtomicReference<>(RegistryState.empty());
|
||||
private final Clock clock;
|
||||
private final byte[] fingerprintKey;
|
||||
|
||||
public TenantReadinessRegistry() {
|
||||
this(Clock.systemUTC(), randomFingerprintKey());
|
||||
}
|
||||
|
||||
TenantReadinessRegistry(Clock clock, byte[] fingerprintKey) {
|
||||
this.clock = Objects.requireNonNull(clock, "Часы реестра готовности обязательны");
|
||||
if (fingerprintKey == null || fingerprintKey.length < 16) {
|
||||
throw new IllegalArgumentException("Ключ fingerprint должен содержать не менее 16 байт");
|
||||
}
|
||||
this.fingerprintKey = fingerprintKey.clone();
|
||||
}
|
||||
|
||||
/**
|
||||
* Полностью заменяет список обязательных конфигураций.
|
||||
* Состояние неизменившегося tenant сохраняется, изменившегося — сбрасывается.
|
||||
*/
|
||||
public void replaceDesired(List<TenantConfig> desired) {
|
||||
boolean missingDesired = desired == null;
|
||||
List<TenantConfig> safeDesired = desired == null ? List.of() : new ArrayList<>(desired);
|
||||
mutate(current -> {
|
||||
Map<String, TenantStatus> replacement = new LinkedHashMap<>();
|
||||
boolean invalidConfiguration = missingDesired;
|
||||
|
||||
for (TenantConfig config : safeDesired) {
|
||||
NormalizedConfig normalized = normalize(config);
|
||||
if (normalized == null || replacement.containsKey(normalized.domain())) {
|
||||
invalidConfiguration = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
String fingerprint = fingerprint(normalized);
|
||||
TenantStatus previous = current.tenants().get(normalized.domain());
|
||||
if (previous != null && previous.fingerprint().equals(fingerprint)) {
|
||||
replacement.put(normalized.domain(), previous);
|
||||
} else {
|
||||
replacement.put(normalized.domain(), TenantStatus.pending(
|
||||
normalized.domain(), fingerprint, normalized.h2Placeholder(), clock.instant()));
|
||||
}
|
||||
}
|
||||
|
||||
return new RegistryState(replacement,
|
||||
current.configurationFailure() || invalidConfiguration);
|
||||
});
|
||||
}
|
||||
|
||||
/** Удаляет tenant из обязательного набора. */
|
||||
public void removeDesired(String domain) {
|
||||
String normalizedDomain = normalizeDomain(domain);
|
||||
mutate(current -> {
|
||||
if (!current.tenants().containsKey(normalizedDomain)) {
|
||||
return current;
|
||||
}
|
||||
Map<String, TenantStatus> updated = new LinkedHashMap<>(current.tenants());
|
||||
updated.remove(normalizedDomain);
|
||||
return new RegistryState(updated, current.configurationFailure());
|
||||
});
|
||||
}
|
||||
|
||||
public void markMigrationStarted(TenantConfig config) {
|
||||
updateByConfig(config, MigrationState.IN_PROGRESS, ConnectivityState.UNKNOWN);
|
||||
}
|
||||
|
||||
public void markMigrationSucceeded(TenantConfig config) {
|
||||
updateByConfig(config, MigrationState.SUCCEEDED, ConnectivityState.UNKNOWN);
|
||||
}
|
||||
|
||||
public void markMigrationFailed(TenantConfig config) {
|
||||
updateByConfig(config, MigrationState.FAILED, ConnectivityState.UNKNOWN);
|
||||
}
|
||||
|
||||
public void markMigrationStarted(String domain) {
|
||||
updateExisting(domain, MigrationState.IN_PROGRESS, ConnectivityState.UNKNOWN);
|
||||
}
|
||||
|
||||
public void markMigrationSucceeded(String domain) {
|
||||
updateExisting(domain, MigrationState.SUCCEEDED, ConnectivityState.UNKNOWN);
|
||||
}
|
||||
|
||||
public void markMigrationFailed(String domain) {
|
||||
updateExisting(domain, MigrationState.FAILED, ConnectivityState.UNKNOWN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Сохраняет результат проверки текущей конфигурации tenant.
|
||||
*/
|
||||
public boolean markConnectivity(String domain, boolean connected) {
|
||||
String normalizedDomain = normalizeDomain(domain);
|
||||
TenantStatus current = state.get().tenants().get(normalizedDomain);
|
||||
if (current == null) {
|
||||
return false;
|
||||
}
|
||||
return markConnectivity(normalizedDomain, current.fingerprint(), connected, clock.instant());
|
||||
}
|
||||
|
||||
/**
|
||||
* Сохраняет результат только если fingerprint всё ещё соответствует проверенной конфигурации.
|
||||
*/
|
||||
public boolean markConnectivity(String domain,
|
||||
String expectedFingerprint,
|
||||
boolean connected,
|
||||
Instant checkedAt) {
|
||||
String normalizedDomain = normalizeDomain(domain);
|
||||
Objects.requireNonNull(expectedFingerprint, "Fingerprint проверяемой конфигурации обязателен");
|
||||
Objects.requireNonNull(checkedAt, "Время проверки соединения обязательно");
|
||||
|
||||
AtomicReference<Boolean> applied = new AtomicReference<>(false);
|
||||
mutate(current -> {
|
||||
applied.set(false);
|
||||
TenantStatus previous = current.tenants().get(normalizedDomain);
|
||||
if (previous == null || !previous.fingerprint().equals(expectedFingerprint)
|
||||
|| previous.migrationState() != MigrationState.SUCCEEDED
|
||||
|| previous.h2Placeholder()) {
|
||||
return current;
|
||||
}
|
||||
|
||||
if (previous.connectivityCheckedAt() != null
|
||||
&& checkedAt.isBefore(previous.connectivityCheckedAt())) {
|
||||
return current;
|
||||
}
|
||||
|
||||
Map<String, TenantStatus> updated = new LinkedHashMap<>(current.tenants());
|
||||
updated.put(normalizedDomain, previous.withConnectivity(
|
||||
connected ? ConnectivityState.AVAILABLE : ConnectivityState.UNAVAILABLE,
|
||||
checkedAt,
|
||||
clock.instant()));
|
||||
applied.set(true);
|
||||
return new RegistryState(updated, current.configurationFailure());
|
||||
});
|
||||
return applied.get();
|
||||
}
|
||||
|
||||
/** Отмечает ошибку чтения или разбора общей конфигурации без сохранения её текста. */
|
||||
public void markConfigurationFailure() {
|
||||
mutate(current -> new RegistryState(current.tenants(), true));
|
||||
}
|
||||
|
||||
public void clearConfigurationFailure() {
|
||||
mutate(current -> new RegistryState(current.tenants(), false));
|
||||
}
|
||||
|
||||
/** Возвращает безопасные задания для фоновой проверки соединений. */
|
||||
public List<ConnectivityTarget> connectivityTargets() {
|
||||
List<ConnectivityTarget> targets = new ArrayList<>();
|
||||
state.get().tenants().values().stream()
|
||||
.filter(status -> status.migrationState() == MigrationState.SUCCEEDED)
|
||||
.filter(status -> !status.h2Placeholder())
|
||||
.forEach(status -> targets.add(new ConnectivityTarget(status.domain(), status.fingerprint())));
|
||||
return List.copyOf(targets);
|
||||
}
|
||||
|
||||
/** Формирует агрегированную готовность на текущий момент. */
|
||||
public AggregateReadiness aggregate(Duration maxStaleness) {
|
||||
Objects.requireNonNull(maxStaleness, "Максимальный возраст проверки обязателен");
|
||||
if (maxStaleness.isNegative() || maxStaleness.isZero()) {
|
||||
throw new IllegalArgumentException("Максимальный возраст проверки должен быть положительным");
|
||||
}
|
||||
|
||||
RegistryState snapshot = state.get();
|
||||
Instant assessedAt = clock.instant();
|
||||
Instant oldestAllowed = assessedAt.minus(maxStaleness);
|
||||
int ready = 0;
|
||||
int migrationFailures = 0;
|
||||
int migrationPending = 0;
|
||||
int staleConnectivity = 0;
|
||||
int unavailableConnectivity = 0;
|
||||
int unsupported = 0;
|
||||
|
||||
for (TenantStatus status : snapshot.tenants().values()) {
|
||||
if (status.h2Placeholder()) {
|
||||
unsupported++;
|
||||
continue;
|
||||
}
|
||||
if (status.migrationState() == MigrationState.FAILED) {
|
||||
migrationFailures++;
|
||||
continue;
|
||||
}
|
||||
if (status.migrationState() != MigrationState.SUCCEEDED) {
|
||||
migrationPending++;
|
||||
continue;
|
||||
}
|
||||
if (status.connectivityCheckedAt() == null
|
||||
|| status.connectivityCheckedAt().isBefore(oldestAllowed)) {
|
||||
staleConnectivity++;
|
||||
continue;
|
||||
}
|
||||
if (status.connectivityState() != ConnectivityState.AVAILABLE) {
|
||||
unavailableConnectivity++;
|
||||
continue;
|
||||
}
|
||||
ready++;
|
||||
}
|
||||
|
||||
int desiredCount = snapshot.tenants().size();
|
||||
boolean isReady = !snapshot.configurationFailure()
|
||||
&& desiredCount > 0
|
||||
&& unsupported == 0
|
||||
&& ready == desiredCount;
|
||||
return new AggregateReadiness(
|
||||
isReady,
|
||||
isReady ? "ГОТОВ" : "НЕ ГОТОВ",
|
||||
desiredCount,
|
||||
ready,
|
||||
desiredCount - ready,
|
||||
migrationPending,
|
||||
migrationFailures,
|
||||
unavailableConnectivity,
|
||||
staleConnectivity,
|
||||
unsupported,
|
||||
snapshot.configurationFailure(),
|
||||
assessedAt
|
||||
);
|
||||
}
|
||||
|
||||
/** Неизменяемый диагностический снимок для внутренних тестов и интеграции. */
|
||||
public Map<String, TenantStatus> snapshot() {
|
||||
return state.get().tenants();
|
||||
}
|
||||
|
||||
private void updateByConfig(TenantConfig config,
|
||||
MigrationState migrationState,
|
||||
ConnectivityState connectivityState) {
|
||||
NormalizedConfig normalized = normalize(config);
|
||||
if (normalized == null) {
|
||||
markConfigurationFailure();
|
||||
return;
|
||||
}
|
||||
String fingerprint = fingerprint(normalized);
|
||||
mutate(current -> {
|
||||
TenantStatus previous = current.tenants().get(normalized.domain());
|
||||
TenantStatus base = previous != null && previous.fingerprint().equals(fingerprint)
|
||||
? previous
|
||||
: TenantStatus.pending(normalized.domain(), fingerprint, normalized.h2Placeholder(), clock.instant());
|
||||
Map<String, TenantStatus> updated = new LinkedHashMap<>(current.tenants());
|
||||
updated.put(normalized.domain(), base.withMigration(migrationState, connectivityState, clock.instant()));
|
||||
return new RegistryState(updated, current.configurationFailure());
|
||||
});
|
||||
}
|
||||
|
||||
private void updateExisting(String domain,
|
||||
MigrationState migrationState,
|
||||
ConnectivityState connectivityState) {
|
||||
String normalizedDomain = normalizeDomain(domain);
|
||||
mutate(current -> {
|
||||
TenantStatus previous = current.tenants().get(normalizedDomain);
|
||||
if (previous == null) {
|
||||
return current;
|
||||
}
|
||||
Map<String, TenantStatus> updated = new LinkedHashMap<>(current.tenants());
|
||||
updated.put(normalizedDomain, previous.withMigration(migrationState, connectivityState, clock.instant()));
|
||||
return new RegistryState(updated, current.configurationFailure());
|
||||
});
|
||||
}
|
||||
|
||||
private void mutate(UnaryOperator<RegistryState> mutation) {
|
||||
RegistryState current;
|
||||
RegistryState updated;
|
||||
do {
|
||||
current = state.get();
|
||||
updated = Objects.requireNonNull(mutation.apply(current), "Новое состояние реестра обязательно");
|
||||
if (updated == current) {
|
||||
return;
|
||||
}
|
||||
} while (!state.compareAndSet(current, updated));
|
||||
}
|
||||
|
||||
private NormalizedConfig normalize(TenantConfig config) {
|
||||
if (config == null || config.getDomain() == null || config.getDomain().isBlank()
|
||||
|| config.getUrl() == null || config.getUrl().isBlank()) {
|
||||
return null;
|
||||
}
|
||||
String domain = config.getDomain().trim().toLowerCase(Locale.ROOT);
|
||||
String url = config.getUrl().trim();
|
||||
if (!DOMAIN_PATTERN.matcher(domain).matches()) {
|
||||
return null;
|
||||
}
|
||||
String name = config.getName() == null || config.getName().isBlank()
|
||||
? domain
|
||||
: config.getName().trim();
|
||||
return new NormalizedConfig(
|
||||
name,
|
||||
domain,
|
||||
url,
|
||||
config.getUsername(),
|
||||
config.getPassword(),
|
||||
url.startsWith("jdbc:h2:")
|
||||
);
|
||||
}
|
||||
|
||||
private String normalizeDomain(String domain) {
|
||||
if (domain == null || domain.isBlank()) {
|
||||
throw new IllegalArgumentException("Домен tenant не может быть пустым");
|
||||
}
|
||||
return domain.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String fingerprint(NormalizedConfig normalized) {
|
||||
try {
|
||||
Mac mac = Mac.getInstance(FINGERPRINT_ALGORITHM);
|
||||
mac.init(new SecretKeySpec(fingerprintKey, FINGERPRINT_ALGORITHM));
|
||||
updateFingerprint(mac, normalized.name());
|
||||
updateFingerprint(mac, normalized.domain());
|
||||
updateFingerprint(mac, normalized.url());
|
||||
updateFingerprint(mac, normalized.username());
|
||||
updateFingerprint(mac, normalized.password());
|
||||
return Base64.getUrlEncoder().withoutPadding().encodeToString(mac.doFinal());
|
||||
} catch (GeneralSecurityException cryptoFailure) {
|
||||
throw new IllegalStateException("Не удалось сформировать безопасный fingerprint конфигурации", cryptoFailure);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateFingerprint(Mac mac, String value) {
|
||||
byte[] bytes = value == null ? new byte[0] : value.getBytes(StandardCharsets.UTF_8);
|
||||
mac.update(ByteBuffer.allocate(Integer.BYTES).putInt(bytes.length).array());
|
||||
mac.update(bytes);
|
||||
}
|
||||
|
||||
private static byte[] randomFingerprintKey() {
|
||||
byte[] key = new byte[32];
|
||||
new SecureRandom().nextBytes(key);
|
||||
return key;
|
||||
}
|
||||
|
||||
public enum MigrationState {
|
||||
PENDING,
|
||||
IN_PROGRESS,
|
||||
SUCCEEDED,
|
||||
FAILED
|
||||
}
|
||||
|
||||
public enum ConnectivityState {
|
||||
UNKNOWN,
|
||||
AVAILABLE,
|
||||
UNAVAILABLE
|
||||
}
|
||||
|
||||
public record ConnectivityTarget(String domain, String fingerprint) {
|
||||
public ConnectivityTarget {
|
||||
Objects.requireNonNull(domain, "Домен проверки обязателен");
|
||||
Objects.requireNonNull(fingerprint, "Fingerprint проверки обязателен");
|
||||
}
|
||||
}
|
||||
|
||||
public record TenantStatus(String domain,
|
||||
String fingerprint,
|
||||
MigrationState migrationState,
|
||||
ConnectivityState connectivityState,
|
||||
Instant connectivityCheckedAt,
|
||||
boolean h2Placeholder,
|
||||
Instant changedAt) {
|
||||
public TenantStatus {
|
||||
Objects.requireNonNull(domain, "Домен состояния обязателен");
|
||||
Objects.requireNonNull(fingerprint, "Fingerprint состояния обязателен");
|
||||
Objects.requireNonNull(migrationState, "Состояние миграции обязательно");
|
||||
Objects.requireNonNull(connectivityState, "Состояние соединения обязательно");
|
||||
Objects.requireNonNull(changedAt, "Время изменения состояния обязательно");
|
||||
}
|
||||
|
||||
private static TenantStatus pending(String domain,
|
||||
String fingerprint,
|
||||
boolean h2Placeholder,
|
||||
Instant changedAt) {
|
||||
return new TenantStatus(domain, fingerprint, MigrationState.PENDING,
|
||||
ConnectivityState.UNKNOWN, null, h2Placeholder, changedAt);
|
||||
}
|
||||
|
||||
private TenantStatus withMigration(MigrationState newMigrationState,
|
||||
ConnectivityState newConnectivityState,
|
||||
Instant now) {
|
||||
return new TenantStatus(domain, fingerprint, newMigrationState,
|
||||
newConnectivityState, null, h2Placeholder, now);
|
||||
}
|
||||
|
||||
private TenantStatus withConnectivity(ConnectivityState newConnectivityState,
|
||||
Instant checkedAt,
|
||||
Instant now) {
|
||||
return new TenantStatus(domain, fingerprint, migrationState,
|
||||
newConnectivityState, checkedAt, h2Placeholder, now);
|
||||
}
|
||||
}
|
||||
|
||||
public record AggregateReadiness(boolean ready,
|
||||
String status,
|
||||
int desiredTenants,
|
||||
int readyTenants,
|
||||
int unavailableTenants,
|
||||
int pendingMigrations,
|
||||
int failedMigrations,
|
||||
int unavailableConnections,
|
||||
int staleConnections,
|
||||
int unsupportedTenants,
|
||||
boolean configurationFailure,
|
||||
Instant assessedAt) {
|
||||
public AggregateReadiness {
|
||||
Objects.requireNonNull(status, "Текст готовности обязателен");
|
||||
Objects.requireNonNull(assessedAt, "Время оценки готовности обязательно");
|
||||
}
|
||||
}
|
||||
|
||||
private record NormalizedConfig(String name,
|
||||
String domain,
|
||||
String url,
|
||||
String username,
|
||||
String password,
|
||||
boolean h2Placeholder) {
|
||||
}
|
||||
|
||||
private record RegistryState(Map<String, TenantStatus> tenants, boolean configurationFailure) {
|
||||
private RegistryState {
|
||||
tenants = Collections.unmodifiableMap(new LinkedHashMap<>(tenants));
|
||||
}
|
||||
|
||||
private static RegistryState empty() {
|
||||
return new RegistryState(Map.of(), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,12 @@
|
||||
package com.magistr.app.service;
|
||||
|
||||
import com.magistr.app.config.tenant.KubernetesTenantSecretUpdater;
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantConfigPersistenceException;
|
||||
import com.magistr.app.config.tenant.TenantConfigStore;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import com.magistr.app.config.tenant.TenantSecretCompensationResult;
|
||||
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -31,18 +35,21 @@ public class TenantLifecycleService {
|
||||
);
|
||||
|
||||
private final TenantRoutingDataSource routingDataSource;
|
||||
private final KubernetesTenantSecretUpdater tenantSecretUpdater;
|
||||
private final TenantConfigStore tenantConfigStore;
|
||||
private final TenantDatabaseMigrationService migrationService;
|
||||
private final RetiredTenantPoolService retiredPoolService;
|
||||
private final TenantReadinessRegistry readinessRegistry;
|
||||
|
||||
public TenantLifecycleService(TenantRoutingDataSource routingDataSource,
|
||||
KubernetesTenantSecretUpdater tenantSecretUpdater,
|
||||
TenantConfigStore tenantConfigStore,
|
||||
TenantDatabaseMigrationService migrationService,
|
||||
RetiredTenantPoolService retiredPoolService) {
|
||||
RetiredTenantPoolService retiredPoolService,
|
||||
TenantReadinessRegistry readinessRegistry) {
|
||||
this.routingDataSource = routingDataSource;
|
||||
this.tenantSecretUpdater = tenantSecretUpdater;
|
||||
this.tenantConfigStore = tenantConfigStore;
|
||||
this.migrationService = migrationService;
|
||||
this.retiredPoolService = retiredPoolService;
|
||||
this.readinessRegistry = readinessRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -76,16 +83,17 @@ public class TenantLifecycleService {
|
||||
candidate = prepareCandidate(config);
|
||||
verifyConnection(candidate);
|
||||
migrateCandidate(candidate);
|
||||
persistDesiredWithCompensation(desiredConfigs, oldConfigs);
|
||||
TenantSecretUpdateReceipt receipt = persistUpsert(config);
|
||||
|
||||
TenantRoutingDataSource.TenantState previous;
|
||||
try {
|
||||
previous = routingDataSource.swapTenant(config, candidate);
|
||||
} catch (RuntimeException activationFailure) {
|
||||
throw activationFailureAfterCompensation(oldConfigs, activationFailure);
|
||||
throw activationFailureAfterCompensation(receipt, activationFailure);
|
||||
}
|
||||
|
||||
activated = true;
|
||||
publishSuccessfulMutation(receipt, sortedConfigs(desiredConfigs), config);
|
||||
retireDataSource(previous == null ? null : previous.dataSource());
|
||||
log.info("Lifecycle тенанта '{}' успешно применён", config.getDomain());
|
||||
return copyConfig(config);
|
||||
@@ -109,7 +117,7 @@ public class TenantLifecycleService {
|
||||
|
||||
Map<String, TenantConfig> desiredConfigs = new LinkedHashMap<>(oldConfigs);
|
||||
desiredConfigs.remove(domain);
|
||||
persistDesiredWithCompensation(desiredConfigs, oldConfigs);
|
||||
TenantSecretUpdateReceipt receipt = persistRemove(domain);
|
||||
|
||||
TenantRoutingDataSource.TenantState removed;
|
||||
try {
|
||||
@@ -118,9 +126,10 @@ public class TenantLifecycleService {
|
||||
throw new IllegalStateException("Локальное состояние тенанта отсутствует");
|
||||
}
|
||||
} catch (RuntimeException removalFailure) {
|
||||
throw removalFailureAfterCompensation(oldConfigs, removalFailure);
|
||||
throw removalFailureAfterCompensation(receipt, removalFailure);
|
||||
}
|
||||
|
||||
publishSuccessfulMutation(receipt, sortedConfigs(desiredConfigs), null);
|
||||
retireDataSource(removed.dataSource());
|
||||
log.info("Lifecycle тенанта '{}' удалён", domain);
|
||||
return copyConfig(existing);
|
||||
@@ -142,6 +151,12 @@ public class TenantLifecycleService {
|
||||
.map(TenantConfig::getDomain)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
readinessRegistry.clearConfigurationFailure();
|
||||
readinessRegistry.replaceDesired(configs);
|
||||
if (desiredDomains.size() != configs.size()) {
|
||||
throw new IllegalArgumentException("Конфигурация содержит повторяющиеся домены тенантов");
|
||||
}
|
||||
|
||||
for (TenantConfig config : configs) {
|
||||
if (!routingDataSource.hasTenant(config.getDomain())) {
|
||||
log.info("Активируем нового тенанта '{}' из persisted-конфигурации", config.getDomain());
|
||||
@@ -162,13 +177,19 @@ public class TenantLifecycleService {
|
||||
private void activatePersistedConfig(TenantConfig config) {
|
||||
HikariDataSource candidate = null;
|
||||
boolean activated = false;
|
||||
readinessRegistry.markMigrationStarted(config);
|
||||
try {
|
||||
candidate = prepareCandidate(config);
|
||||
verifyConnection(candidate);
|
||||
migrateCandidate(candidate);
|
||||
TenantRoutingDataSource.TenantState previous = routingDataSource.swapTenant(config, candidate);
|
||||
activated = true;
|
||||
readinessRegistry.markMigrationSucceeded(config);
|
||||
readinessRegistry.markConnectivity(config.getDomain(), true);
|
||||
retireDataSource(previous == null ? null : previous.dataSource());
|
||||
} catch (RuntimeException activationFailure) {
|
||||
readinessRegistry.markMigrationFailed(config);
|
||||
throw activationFailure;
|
||||
} finally {
|
||||
if (!activated) {
|
||||
closeCandidateDataSource(candidate);
|
||||
@@ -208,63 +229,117 @@ public class TenantLifecycleService {
|
||||
}
|
||||
}
|
||||
|
||||
private void persistDesired(Map<String, TenantConfig> desiredConfigs) {
|
||||
boolean persisted;
|
||||
private TenantSecretUpdateReceipt persistUpsert(TenantConfig config) {
|
||||
try {
|
||||
persisted = tenantSecretUpdater.updateTenantsConfig(sortedConfigs(desiredConfigs));
|
||||
return requireReceipt(tenantConfigStore.upsertTenant(copyConfig(config)));
|
||||
} catch (TenantConfigPersistenceException persistenceFailure) {
|
||||
logTechnicalFailure("Персистенция tenant Secret завершилась ошибкой", persistenceFailure);
|
||||
throw new TenantLifecycleException(persistenceFailure.getMessage(), persistenceFailure);
|
||||
} catch (RuntimeException persistenceFailure) {
|
||||
logTechnicalFailure("Персистенция tenant Secret завершилась ошибкой", persistenceFailure);
|
||||
throw new TenantLifecycleException("Не удалось сохранить конфигурацию тенантов", persistenceFailure);
|
||||
}
|
||||
if (!persisted) {
|
||||
throw new TenantLifecycleException("Не удалось сохранить конфигурацию тенантов");
|
||||
}
|
||||
}
|
||||
|
||||
private void persistDesiredWithCompensation(Map<String, TenantConfig> desiredConfigs,
|
||||
Map<String, TenantConfig> oldConfigs) {
|
||||
private TenantSecretUpdateReceipt persistRemove(String domain) {
|
||||
try {
|
||||
persistDesired(desiredConfigs);
|
||||
} catch (TenantLifecycleException persistenceFailure) {
|
||||
boolean restored = restorePersistedSnapshot(oldConfigs);
|
||||
String message = restored
|
||||
? "Не удалось сохранить конфигурацию тенантов; прежняя конфигурация восстановлена"
|
||||
: "Не удалось сохранить конфигурацию тенантов; требуется проверить tenant Secret";
|
||||
throw new TenantLifecycleException(message, persistenceFailure);
|
||||
return requireReceipt(tenantConfigStore.removeTenant(domain));
|
||||
} catch (TenantConfigPersistenceException persistenceFailure) {
|
||||
logTechnicalFailure("Персистенция tenant Secret завершилась ошибкой", persistenceFailure);
|
||||
throw new TenantLifecycleException(persistenceFailure.getMessage(), persistenceFailure);
|
||||
} catch (RuntimeException persistenceFailure) {
|
||||
logTechnicalFailure("Персистенция tenant Secret завершилась ошибкой", persistenceFailure);
|
||||
throw new TenantLifecycleException("Не удалось сохранить конфигурацию тенантов", persistenceFailure);
|
||||
}
|
||||
}
|
||||
|
||||
private TenantLifecycleException activationFailureAfterCompensation(Map<String, TenantConfig> oldConfigs,
|
||||
RuntimeException activationFailure) {
|
||||
boolean restored = restorePersistedSnapshot(oldConfigs);
|
||||
private TenantSecretUpdateReceipt requireReceipt(TenantSecretUpdateReceipt receipt) {
|
||||
if (receipt == null) {
|
||||
throw new TenantConfigPersistenceException("Хранилище конфигурации тенантов не подтвердило операцию");
|
||||
}
|
||||
return receipt;
|
||||
}
|
||||
|
||||
private TenantLifecycleException activationFailureAfterCompensation(TenantSecretUpdateReceipt receipt,
|
||||
RuntimeException activationFailure) {
|
||||
TenantSecretCompensationResult compensation = compensate(receipt);
|
||||
logTechnicalFailure("Активация подготовленного tenant pool завершилась ошибкой", activationFailure);
|
||||
String message = restored
|
||||
? "Не удалось активировать подключение тенанта; сохранённая конфигурация восстановлена"
|
||||
: "Не удалось активировать подключение тенанта; требуется проверить tenant Secret";
|
||||
String message = compensationMessage(
|
||||
"Не удалось активировать подключение тенанта",
|
||||
compensation
|
||||
);
|
||||
return new TenantLifecycleException(message, activationFailure);
|
||||
}
|
||||
|
||||
private TenantLifecycleException removalFailureAfterCompensation(Map<String, TenantConfig> oldConfigs,
|
||||
RuntimeException removalFailure) {
|
||||
boolean restored = restorePersistedSnapshot(oldConfigs);
|
||||
private TenantLifecycleException removalFailureAfterCompensation(TenantSecretUpdateReceipt receipt,
|
||||
RuntimeException removalFailure) {
|
||||
TenantSecretCompensationResult compensation = compensate(receipt);
|
||||
logTechnicalFailure("Локальное удаление тенанта завершилось ошибкой", removalFailure);
|
||||
String message = restored
|
||||
? "Не удалось отключить тенанта; сохранённая конфигурация восстановлена"
|
||||
: "Не удалось отключить тенанта; требуется проверить tenant Secret";
|
||||
String message = compensationMessage("Не удалось отключить тенанта", compensation);
|
||||
return new TenantLifecycleException(message, removalFailure);
|
||||
}
|
||||
|
||||
private boolean restorePersistedSnapshot(Map<String, TenantConfig> oldConfigs) {
|
||||
private TenantSecretCompensationResult compensate(TenantSecretUpdateReceipt receipt) {
|
||||
try {
|
||||
boolean restored = tenantSecretUpdater.updateTenantsConfig(sortedConfigs(oldConfigs));
|
||||
if (!restored) {
|
||||
log.error("Не удалось компенсирующе восстановить tenant Secret");
|
||||
TenantSecretCompensationResult result = tenantConfigStore.compensate(receipt);
|
||||
if (result == null) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
log.error("Хранилище конфигурации не вернуло результат компенсации tenant Secret");
|
||||
return null;
|
||||
}
|
||||
return restored;
|
||||
if (result == TenantSecretCompensationResult.RESTORED && receipt.persisted()) {
|
||||
readinessRegistry.clearConfigurationFailure();
|
||||
readinessRegistry.replaceDesired(receipt.previousTenants());
|
||||
}
|
||||
if (result == TenantSecretCompensationResult.NOT_REQUIRED && receipt.persisted()) {
|
||||
readinessRegistry.replaceDesired(receipt.committedTenants());
|
||||
}
|
||||
if (result == TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE
|
||||
|| (result == TenantSecretCompensationResult.NOT_REQUIRED && receipt.persisted())) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
}
|
||||
return result;
|
||||
} catch (RuntimeException compensationFailure) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
logTechnicalFailure("Компенсирующее восстановление tenant Secret завершилось ошибкой",
|
||||
compensationFailure);
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String compensationMessage(String prefix, TenantSecretCompensationResult result) {
|
||||
if (result == TenantSecretCompensationResult.RESTORED) {
|
||||
return prefix + "; сохранённая конфигурация восстановлена";
|
||||
}
|
||||
if (result == TenantSecretCompensationResult.NOT_REQUIRED) {
|
||||
return prefix + "; tenant Secret не требовал восстановления";
|
||||
}
|
||||
if (result == TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE) {
|
||||
return prefix + "; tenant Secret уже изменён другим экземпляром, автоматический откат пропущен";
|
||||
}
|
||||
return prefix + "; требуется проверить tenant Secret";
|
||||
}
|
||||
|
||||
private void publishSuccessfulMutation(TenantSecretUpdateReceipt receipt,
|
||||
List<TenantConfig> localDesired,
|
||||
TenantConfig migratedConfig) {
|
||||
try {
|
||||
List<TenantConfig> canonicalDesired = receipt.persisted()
|
||||
? receipt.committedTenants()
|
||||
: localDesired.stream()
|
||||
.filter(config -> !isH2Placeholder(config))
|
||||
.toList();
|
||||
if (receipt.persisted()) {
|
||||
readinessRegistry.clearConfigurationFailure();
|
||||
}
|
||||
readinessRegistry.replaceDesired(canonicalDesired);
|
||||
if (migratedConfig != null) {
|
||||
readinessRegistry.markMigrationSucceeded(migratedConfig);
|
||||
readinessRegistry.markConnectivity(migratedConfig.getDomain(), true);
|
||||
}
|
||||
} catch (RuntimeException healthFailure) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
logTechnicalFailure("Не удалось обновить состояние readiness после tenant lifecycle", healthFailure);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,6 +352,13 @@ public class TenantLifecycleService {
|
||||
return sorted;
|
||||
}
|
||||
|
||||
private boolean isH2Placeholder(TenantConfig config) {
|
||||
return config != null
|
||||
&& "H2 Placeholder".equals(config.getName())
|
||||
&& config.getUrl() != null
|
||||
&& config.getUrl().trim().startsWith("jdbc:h2:mem:placeholder");
|
||||
}
|
||||
|
||||
private TenantConfig validateAndNormalize(TenantConfig config) {
|
||||
if (config == null) {
|
||||
throw new IllegalArgumentException("Конфигурация тенанта обязательна");
|
||||
|
||||
@@ -13,6 +13,19 @@ spring.jpa.open-in-view=false
|
||||
|
||||
# Мультитенантность
|
||||
app.tenants.config-path=${TENANTS_CONFIG_PATH:tenants.json}
|
||||
app.tenants.config-required=${TENANTS_CONFIG_REQUIRED:false}
|
||||
|
||||
# Служебные liveness/readiness probes для Kubernetes
|
||||
management.endpoints.web.exposure.include=health
|
||||
management.endpoint.health.probes.enabled=true
|
||||
management.endpoint.health.group.liveness.include=livenessState
|
||||
management.endpoint.health.group.readiness.include=readinessState,tenantReadiness
|
||||
management.endpoint.health.show-details=never
|
||||
management.endpoint.health.show-components=never
|
||||
management.health.db.enabled=false
|
||||
|
||||
# Фоновая проверка БД не должна задерживать watcher tenant Secret
|
||||
spring.task.scheduling.pool.size=2
|
||||
|
||||
# JWT авторизация
|
||||
app.jwt.secret=${JWT_SECRET:}
|
||||
|
||||
@@ -39,7 +39,10 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
@Testcontainers
|
||||
@SpringBootTest(
|
||||
classes = RefreshTokenServiceConcurrencyIntegrationTest.TestApplication.class,
|
||||
properties = "app.jwt.refresh-ttl=7d"
|
||||
properties = {
|
||||
"app.jwt.refresh-ttl=7d",
|
||||
"management.endpoint.health.validate-group-membership=false"
|
||||
}
|
||||
)
|
||||
class RefreshTokenServiceConcurrencyIntegrationTest {
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import okhttp3.mockwebserver.Dispatcher;
|
||||
import okhttp3.mockwebserver.MockResponse;
|
||||
import okhttp3.mockwebserver.MockWebServer;
|
||||
import okhttp3.mockwebserver.RecordedRequest;
|
||||
@@ -13,50 +16,440 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class KubernetesTenantSecretUpdaterTest {
|
||||
|
||||
private static final TypeReference<List<TenantConfig>> TENANT_LIST_TYPE = new TypeReference<>() {
|
||||
};
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@TempDir
|
||||
Path tempDirectory;
|
||||
|
||||
@Test
|
||||
void updatesSecretThroughTrustedCaWithoutSendingPlainTenantJson() throws Exception {
|
||||
void readsCurrentSecretAndUsesConditionalPutWithoutPlainCredentials() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
HeldCertificate serverCertificate = serverCertificate(ca, "localhost");
|
||||
TenantConfig tenant = tenant("Тестовый университет", "test", "пароль-" + UUID.randomUUID());
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate)) {
|
||||
server.enqueue(new MockResponse().setResponseCode(200));
|
||||
TestFiles files = testFiles(ca);
|
||||
String rawPassword = UUID.randomUUID().toString();
|
||||
TenantConfig tenant = new TenantConfig("Университет", "test", "jdbc:test", "user", rawPassword);
|
||||
KubernetesTenantSecretUpdater updater = updater(files, server);
|
||||
server.enqueue(secretResponse("17", List.of()));
|
||||
server.enqueue(secretResponse("18", List.of(tenant)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server);
|
||||
|
||||
boolean updated = updater.updateTenantsConfig(List.of(tenant));
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(tenant);
|
||||
|
||||
assertThat(updated).isTrue();
|
||||
RecordedRequest request = server.takeRequest(2, TimeUnit.SECONDS);
|
||||
assertThat(request).isNotNull();
|
||||
assertThat(request.getPath()).isEqualTo("/api/v1/namespaces/magistr/secrets/tenants-secret");
|
||||
String body = request.getBody().readUtf8();
|
||||
assertThat(body).doesNotContain(rawPassword);
|
||||
assertThat(receipt.persisted()).isTrue();
|
||||
assertThat(receipt.changed()).isTrue();
|
||||
assertThat(receipt.committedResourceVersion()).isEqualTo("18");
|
||||
assertThat(domains(receipt.previousTenants())).isEmpty();
|
||||
assertThat(domains(receipt.committedTenants())).containsExactly("test");
|
||||
|
||||
JsonNode patch = objectMapper.readTree(body);
|
||||
String decoded = new String(
|
||||
Base64.getDecoder().decode(patch.path("data").path("tenants.json").asText()),
|
||||
StandardCharsets.UTF_8
|
||||
);
|
||||
assertThat(decoded).contains("\"domain\" : \"test\"");
|
||||
RecordedRequest get = server.takeRequest(2, TimeUnit.SECONDS);
|
||||
RecordedRequest put = server.takeRequest(2, TimeUnit.SECONDS);
|
||||
assertThat(get).isNotNull();
|
||||
assertThat(get.getMethod()).isEqualTo("GET");
|
||||
assertThat(get.getPath()).isEqualTo("/api/v1/namespaces/magistr/secrets/tenants-secret");
|
||||
assertThat(get.getHeader("Authorization")).isEqualTo("Bearer test-service-account-token");
|
||||
|
||||
assertThat(put).isNotNull();
|
||||
assertThat(put.getMethod()).isEqualTo("PUT");
|
||||
assertThat(put.getHeader("Content-Type")).startsWith("application/json");
|
||||
String body = put.getBody().readUtf8();
|
||||
assertThat(body).doesNotContain(tenant.getPassword());
|
||||
JsonNode update = objectMapper.readTree(body);
|
||||
assertThat(update.path("metadata").path("resourceVersion").asText()).isEqualTo("17");
|
||||
assertThat(update.path("data").path("other.key").asText()).isEqualTo("cHJlc2VydmU=");
|
||||
assertThat(domains(decodeTenants(update))).containsExactly("test");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void skipsPutWhenUpsertIsIdempotent() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig tenant = tenant("Университет", "same", "секрет");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("5", List.of(tenant)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(tenant);
|
||||
|
||||
assertThat(receipt.persisted()).isTrue();
|
||||
assertThat(receipt.changed()).isFalse();
|
||||
assertThat(receipt.committedResourceVersion()).isEqualTo("5");
|
||||
assertThat(server.getRequestCount()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void removeTenantKeepsOtherCurrentTenants() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("7", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("8", List.of(beta)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.removeTenant("alpha");
|
||||
|
||||
assertThat(receipt.changed()).isTrue();
|
||||
assertThat(domains(receipt.previousTenants())).containsExactly("alpha", "beta");
|
||||
assertThat(domains(receipt.committedTenants())).containsExactly("beta");
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
RecordedRequest put = server.takeRequest(2, TimeUnit.SECONDS);
|
||||
assertThat(domains(decodeTenants(objectMapper.readTree(put.getBody().readUtf8()))))
|
||||
.containsExactly("beta");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void retriesConflictAgainstFreshResourceVersion() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("1", List.of()));
|
||||
server.enqueue(new MockResponse().setResponseCode(409));
|
||||
server.enqueue(secretResponse("2", List.of(alpha)));
|
||||
server.enqueue(secretResponse("3", List.of(alpha, beta)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
|
||||
assertThat(receipt.committedResourceVersion()).isEqualTo("3");
|
||||
assertThat(domains(receipt.previousTenants())).containsExactly("alpha");
|
||||
assertThat(domains(receipt.committedTenants())).containsExactly("alpha", "beta");
|
||||
assertThat(server.getRequestCount()).isEqualTo(4);
|
||||
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
RecordedRequest finalPut = server.takeRequest(2, TimeUnit.SECONDS);
|
||||
JsonNode body = objectMapper.readTree(finalPut.getBody().readUtf8());
|
||||
assertThat(body.path("metadata").path("resourceVersion").asText()).isEqualTo("2");
|
||||
assertThat(domains(decodeTenants(body))).containsExactly("alpha", "beta");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopsAfterBoundedNumberOfConflicts() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
for (int attempt = 1; attempt <= 3; attempt++) {
|
||||
server.enqueue(secretResponse(String.valueOf(attempt), List.of()));
|
||||
server.enqueue(new MockResponse().setResponseCode(409));
|
||||
}
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Тест", "test", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("конкурентных");
|
||||
assertThat(server.getRequestCount()).isEqualTo(6);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcilesRetryablePutResponseWithoutCreatingUnsafeCompensationReceipt() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("20", List.of(alpha)));
|
||||
server.enqueue(new MockResponse().setResponseCode(503));
|
||||
server.enqueue(secretResponse("21", List.of(alpha, beta)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
|
||||
assertThat(receipt.changed()).isFalse();
|
||||
assertThat(receipt.committedResourceVersion()).isEqualTo("21");
|
||||
assertThat(domains(receipt.committedTenants())).containsExactly("alpha", "beta");
|
||||
assertThat(updater.compensate(receipt))
|
||||
.isEqualTo(TenantSecretCompensationResult.NOT_REQUIRED);
|
||||
assertThat(server.getRequestCount()).isEqualTo(3);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcilesRequestTimeoutWithoutCreatingUnsafeCompensationReceipt() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("24", List.of(alpha)));
|
||||
server.enqueue(new MockResponse().setResponseCode(408));
|
||||
server.enqueue(secretResponse("25", List.of(alpha, beta)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
|
||||
assertThat(receipt.persisted()).isTrue();
|
||||
assertThat(receipt.changed()).isFalse();
|
||||
assertThat(receipt.committedResourceVersion()).isEqualTo("25");
|
||||
assertThat(domains(receipt.committedTenants())).containsExactly("alpha", "beta");
|
||||
assertThat(server.getRequestCount()).isEqualTo(3);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsSuccessfulMutationResponseWithDifferentTenantContent() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
TenantConfig gamma = tenant("Гамма", "gamma", "секрет-c");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("26", List.of(alpha)));
|
||||
server.enqueue(secretResponse("27", List.of(alpha, gamma)));
|
||||
server.enqueue(secretResponse("27", List.of(alpha, gamma)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
assertThatThrownBy(() -> updater.upsertTenant(beta))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("не подтвердил сохранённое содержимое");
|
||||
assertThat(server.getRequestCount()).isEqualTo(3);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesDomainsBeforeMergeAndRejectsCaseInsensitiveDuplicates() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig upperAlpha = tenant("Альфа", " ALPHA ", "секрет-a");
|
||||
TenantConfig lowerAlpha = tenant("Другая Альфа", "alpha", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("22", List.of(upperAlpha, lowerAlpha)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Бета", "beta", "секрет-c")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("повторяющиеся домены");
|
||||
assertThat(server.getRequestCount()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void failsClosedWhenTenantsKeyIsMissing() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
ObjectNode secret = objectMapper.createObjectNode();
|
||||
secret.put("apiVersion", "v1");
|
||||
secret.put("kind", "Secret");
|
||||
secret.putObject("metadata").put("resourceVersion", "23");
|
||||
secret.putObject("data");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(new MockResponse()
|
||||
.setResponseCode(200)
|
||||
.setHeader("Content-Type", "application/json")
|
||||
.setBody(objectMapper.writeValueAsString(secret)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Бета", "beta", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("data.tenants.json");
|
||||
assertThat(server.getRequestCount()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalKubernetesErrorIsNotRetried() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(new MockResponse().setResponseCode(403));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Бета", "beta", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("HTTP 403");
|
||||
assertThat(server.getRequestCount()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoConcurrentUpdatersPreserveBothMutations() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
CasDispatcher dispatcher = new CasDispatcher(List.of(), true);
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.setDispatcher(dispatcher);
|
||||
TestFiles files = testFiles(ca);
|
||||
KubernetesTenantSecretUpdater first = updater(files, server, 4, Duration.ZERO);
|
||||
KubernetesTenantSecretUpdater second = updater(files, server, 4, Duration.ZERO);
|
||||
ExecutorService executor = Executors.newFixedThreadPool(2);
|
||||
try {
|
||||
Future<TenantSecretUpdateReceipt> firstResult =
|
||||
executor.submit(() -> first.upsertTenant(alpha));
|
||||
Future<TenantSecretUpdateReceipt> secondResult =
|
||||
executor.submit(() -> second.upsertTenant(beta));
|
||||
|
||||
assertThat(firstResult.get(10, TimeUnit.SECONDS).changed()).isTrue();
|
||||
assertThat(secondResult.get(10, TimeUnit.SECONDS).changed()).isTrue();
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
assertThat(domains(dispatcher.currentTenants())).containsExactly("alpha", "beta");
|
||||
assertThat(dispatcher.conflictCount()).isGreaterThanOrEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void compensationRestoresOnlyVersionCommittedByOperation() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("30", List.of(alpha)));
|
||||
server.enqueue(secretResponse("31", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("31", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("32", List.of(alpha)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
TenantSecretCompensationResult result = updater.compensate(receipt);
|
||||
|
||||
assertThat(result).isEqualTo(TenantSecretCompensationResult.RESTORED);
|
||||
assertThat(server.getRequestCount()).isEqualTo(4);
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
server.takeRequest(2, TimeUnit.SECONDS);
|
||||
RecordedRequest compensationPut = server.takeRequest(2, TimeUnit.SECONDS);
|
||||
JsonNode body = objectMapper.readTree(compensationPut.getBody().readUtf8());
|
||||
assertThat(body.path("metadata").path("resourceVersion").asText()).isEqualTo("31");
|
||||
assertThat(domains(decodeTenants(body))).containsExactly("alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void compensationDoesNotOverwriteNewerConcurrentChange() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
TenantConfig gamma = tenant("Гамма", "gamma", "секрет-c");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("40", List.of(alpha)));
|
||||
server.enqueue(secretResponse("41", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("42", List.of(alpha, beta, gamma)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
TenantSecretCompensationResult result = updater.compensate(receipt);
|
||||
|
||||
assertThat(result).isEqualTo(TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE);
|
||||
assertThat(server.getRequestCount()).isEqualTo(3);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void compensationDoesNotRetryAfterPutLosesResourceVersionRace() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("50", List.of(alpha)));
|
||||
server.enqueue(secretResponse("51", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("51", List.of(alpha, beta)));
|
||||
server.enqueue(new MockResponse().setResponseCode(409));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
TenantSecretCompensationResult result = updater.compensate(receipt);
|
||||
|
||||
assertThat(result).isEqualTo(TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE);
|
||||
assertThat(server.getRequestCount()).isEqualTo(4);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void compensationDoesNotTrustSuccessfulResponseWithDifferentTenantContent() throws Exception {
|
||||
HeldCertificate ca = certificateAuthority("Доверенный тестовый CA");
|
||||
TenantConfig alpha = tenant("Альфа", "alpha", "секрет-a");
|
||||
TenantConfig beta = tenant("Бета", "beta", "секрет-b");
|
||||
TenantConfig gamma = tenant("Гамма", "gamma", "секрет-c");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate(ca, "localhost"))) {
|
||||
server.enqueue(secretResponse("60", List.of(alpha)));
|
||||
server.enqueue(secretResponse("61", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("61", List.of(alpha, beta)));
|
||||
server.enqueue(secretResponse("62", List.of(alpha, gamma)));
|
||||
server.enqueue(secretResponse("62", List.of(alpha, gamma)));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 3, Duration.ZERO);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(beta);
|
||||
TenantSecretCompensationResult result = updater.compensate(receipt);
|
||||
|
||||
assertThat(result).isEqualTo(TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE);
|
||||
assertThat(server.getRequestCount()).isEqualTo(5);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void localModeIsNoopAndDoesNotRequireKubernetesCa() {
|
||||
Path missingToken = tempDirectory.resolve("missing-token");
|
||||
KubernetesTenantSecretUpdater updater = new KubernetesTenantSecretUpdater(
|
||||
missingToken,
|
||||
tempDirectory.resolve("missing-namespace"),
|
||||
tempDirectory.resolve("missing-ca"),
|
||||
"https://localhost:1",
|
||||
"tenants-secret",
|
||||
objectMapper
|
||||
);
|
||||
|
||||
TenantSecretUpdateReceipt receipt = updater.upsertTenant(tenant("Локальный", "local", "секрет"));
|
||||
|
||||
assertThat(receipt.persisted()).isFalse();
|
||||
assertThat(receipt.changed()).isFalse();
|
||||
assertThat(updater.compensate(receipt))
|
||||
.isEqualTo(TenantSecretCompensationResult.NOT_REQUIRED);
|
||||
}
|
||||
|
||||
@Test
|
||||
void kubernetesModeFailsClosedWhenServiceAccountTokenIsMissing() {
|
||||
KubernetesTenantSecretUpdater updater = new KubernetesTenantSecretUpdater(
|
||||
tempDirectory.resolve("missing-kubernetes-token"),
|
||||
tempDirectory.resolve("missing-kubernetes-namespace"),
|
||||
tempDirectory.resolve("missing-kubernetes-ca"),
|
||||
"https://localhost:1",
|
||||
"tenants-secret",
|
||||
objectMapper,
|
||||
1,
|
||||
Duration.ZERO,
|
||||
true
|
||||
);
|
||||
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Тест", "test", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("ServiceAccount");
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsServerSignedByDifferentCa() throws Exception {
|
||||
HeldCertificate trustedCa = certificateAuthority("Доверенный тестовый CA");
|
||||
@@ -64,10 +457,11 @@ class KubernetesTenantSecretUpdaterTest {
|
||||
HeldCertificate serverCertificate = serverCertificate(untrustedCa, "localhost");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate)) {
|
||||
server.enqueue(new MockResponse().setResponseCode(200));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(trustedCa), server);
|
||||
server.enqueue(secretResponse("1", List.of()));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(trustedCa), server, 1, Duration.ZERO);
|
||||
|
||||
assertThat(updater.updateTenantsConfig(List.of())).isFalse();
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Тест", "test", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,10 +471,11 @@ class KubernetesTenantSecretUpdaterTest {
|
||||
HeldCertificate serverCertificate = serverCertificate(ca, "not-localhost.invalid");
|
||||
|
||||
try (MockWebServer server = httpsServer(serverCertificate)) {
|
||||
server.enqueue(new MockResponse().setResponseCode(200));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server);
|
||||
server.enqueue(secretResponse("1", List.of()));
|
||||
KubernetesTenantSecretUpdater updater = updater(testFiles(ca), server, 1, Duration.ZERO);
|
||||
|
||||
assertThat(updater.updateTenantsConfig(List.of())).isFalse();
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Тест", "test", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,21 +489,33 @@ class KubernetesTenantSecretUpdaterTest {
|
||||
files.caPath(),
|
||||
"https://localhost:1",
|
||||
"tenants-secret",
|
||||
objectMapper
|
||||
objectMapper,
|
||||
1,
|
||||
Duration.ZERO
|
||||
);
|
||||
|
||||
assertThat(updater.updateTenantsConfig(List.of())).isFalse();
|
||||
assertThatThrownBy(() -> updater.upsertTenant(tenant("Тест", "test", "секрет")))
|
||||
.isInstanceOf(TenantConfigPersistenceException.class)
|
||||
.hasMessageContaining("защищённое соединение");
|
||||
}
|
||||
|
||||
private KubernetesTenantSecretUpdater updater(TestFiles files, MockWebServer server) {
|
||||
String apiBase = server.url("/").toString();
|
||||
return updater(files, server, 3, Duration.ofMillis(5));
|
||||
}
|
||||
|
||||
private KubernetesTenantSecretUpdater updater(TestFiles files,
|
||||
MockWebServer server,
|
||||
int maxAttempts,
|
||||
Duration retryDelay) {
|
||||
return new KubernetesTenantSecretUpdater(
|
||||
files.tokenPath(),
|
||||
files.namespacePath(),
|
||||
files.caPath(),
|
||||
apiBase,
|
||||
server.url("/").toString(),
|
||||
"tenants-secret",
|
||||
objectMapper
|
||||
objectMapper,
|
||||
maxAttempts,
|
||||
retryDelay
|
||||
);
|
||||
}
|
||||
|
||||
@@ -122,6 +529,58 @@ class KubernetesTenantSecretUpdaterTest {
|
||||
return new TestFiles(tokenPath, namespacePath, caPath);
|
||||
}
|
||||
|
||||
private TenantConfig tenant(String name, String domain, String password) {
|
||||
return new TenantConfig(
|
||||
name,
|
||||
domain,
|
||||
"jdbc:postgresql://db/" + domain,
|
||||
"user-" + domain,
|
||||
password
|
||||
);
|
||||
}
|
||||
|
||||
private MockResponse secretResponse(String resourceVersion, List<TenantConfig> tenants) {
|
||||
return new MockResponse()
|
||||
.setResponseCode(200)
|
||||
.setHeader("Content-Type", "application/json")
|
||||
.setBody(secretJson(resourceVersion, tenants));
|
||||
}
|
||||
|
||||
private String secretJson(String resourceVersion, List<TenantConfig> tenants) {
|
||||
try {
|
||||
ObjectNode secret = objectMapper.createObjectNode();
|
||||
secret.put("apiVersion", "v1");
|
||||
secret.put("kind", "Secret");
|
||||
ObjectNode metadata = secret.putObject("metadata");
|
||||
metadata.put("name", "tenants-secret");
|
||||
metadata.put("namespace", "magistr");
|
||||
metadata.put("resourceVersion", resourceVersion);
|
||||
ObjectNode data = secret.putObject("data");
|
||||
data.put("tenants.json", encodeTenants(tenants));
|
||||
data.put("other.key", "cHJlc2VydmU=");
|
||||
return objectMapper.writeValueAsString(secret);
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private String encodeTenants(List<TenantConfig> tenants) throws Exception {
|
||||
return Base64.getEncoder().encodeToString(
|
||||
objectMapper.writeValueAsBytes(tenants)
|
||||
);
|
||||
}
|
||||
|
||||
private List<TenantConfig> decodeTenants(JsonNode secret) throws Exception {
|
||||
byte[] decoded = Base64.getDecoder().decode(
|
||||
secret.path("data").path("tenants.json").asText()
|
||||
);
|
||||
return objectMapper.readValue(decoded, TENANT_LIST_TYPE);
|
||||
}
|
||||
|
||||
private List<String> domains(List<TenantConfig> tenants) {
|
||||
return tenants.stream().map(TenantConfig::getDomain).sorted().toList();
|
||||
}
|
||||
|
||||
private HeldCertificate certificateAuthority(String commonName) {
|
||||
return new HeldCertificate.Builder()
|
||||
.certificateAuthority(1)
|
||||
@@ -149,6 +608,68 @@ class KubernetesTenantSecretUpdaterTest {
|
||||
return server;
|
||||
}
|
||||
|
||||
private final class CasDispatcher extends Dispatcher {
|
||||
private final Object monitor = new Object();
|
||||
private final CountDownLatch firstReads;
|
||||
private final AtomicInteger initialReadCount = new AtomicInteger();
|
||||
private final AtomicInteger conflicts = new AtomicInteger();
|
||||
private List<TenantConfig> currentTenants;
|
||||
private int resourceVersion = 1;
|
||||
|
||||
private CasDispatcher(List<TenantConfig> initialTenants, boolean synchronizeFirstReads) {
|
||||
this.currentTenants = new ArrayList<>(initialTenants);
|
||||
this.firstReads = synchronizeFirstReads ? new CountDownLatch(2) : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
|
||||
if ("GET".equals(request.getMethod())) {
|
||||
MockResponse snapshot;
|
||||
synchronized (monitor) {
|
||||
snapshot = secretResponse(String.valueOf(resourceVersion), currentTenants);
|
||||
}
|
||||
if (firstReads != null && initialReadCount.incrementAndGet() <= 2) {
|
||||
firstReads.countDown();
|
||||
if (!firstReads.await(5, TimeUnit.SECONDS)) {
|
||||
return new MockResponse().setResponseCode(500);
|
||||
}
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
if ("PUT".equals(request.getMethod())) {
|
||||
try {
|
||||
JsonNode body = objectMapper.readTree(request.getBody().readUtf8());
|
||||
int expectedVersion = body.path("metadata").path("resourceVersion").asInt(-1);
|
||||
List<TenantConfig> requestedTenants = decodeTenants(body);
|
||||
synchronized (monitor) {
|
||||
if (expectedVersion != resourceVersion) {
|
||||
conflicts.incrementAndGet();
|
||||
return new MockResponse().setResponseCode(409);
|
||||
}
|
||||
currentTenants = new ArrayList<>(requestedTenants);
|
||||
resourceVersion++;
|
||||
return secretResponse(String.valueOf(resourceVersion), currentTenants);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return new MockResponse().setResponseCode(400);
|
||||
}
|
||||
}
|
||||
|
||||
return new MockResponse().setResponseCode(405);
|
||||
}
|
||||
|
||||
private List<TenantConfig> currentTenants() {
|
||||
synchronized (monitor) {
|
||||
return new ArrayList<>(currentTenants);
|
||||
}
|
||||
}
|
||||
|
||||
private int conflictCount() {
|
||||
return conflicts.get();
|
||||
}
|
||||
}
|
||||
|
||||
private record TestFiles(Path tokenPath, Path namespacePath, Path caPath) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.magistr.app.service.RetiredTenantPoolService;
|
||||
import com.magistr.app.service.TenantDatabaseMigrationService;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
@@ -103,11 +104,15 @@ class TenantConfigWatcherTest {
|
||||
TenantRoutingDataSource routingDataSource = mock(TenantRoutingDataSource.class);
|
||||
TenantLifecycleService lifecycleService = new TenantLifecycleService(
|
||||
routingDataSource,
|
||||
mock(KubernetesTenantSecretUpdater.class),
|
||||
mock(TenantConfigStore.class),
|
||||
mock(TenantDatabaseMigrationService.class),
|
||||
mock(RetiredTenantPoolService.class)
|
||||
mock(RetiredTenantPoolService.class),
|
||||
mock(TenantReadinessRegistry.class)
|
||||
);
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(
|
||||
lifecycleService,
|
||||
mock(TenantReadinessRegistry.class)
|
||||
);
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(lifecycleService);
|
||||
ReflectionTestUtils.setField(watcher, "tenantsConfigPath", configPath.toString());
|
||||
CountDownLatch apiMutationReachedRefreshWindow = new CountDownLatch(1);
|
||||
CountDownLatch allowApiRefresh = new CountDownLatch(1);
|
||||
@@ -148,7 +153,10 @@ class TenantConfigWatcherTest {
|
||||
invocation.<Runnable>getArgument(0).run();
|
||||
return null;
|
||||
}).when(lifecycleService).executeSerialized(any(Runnable.class));
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(lifecycleService);
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(
|
||||
lifecycleService,
|
||||
mock(TenantReadinessRegistry.class)
|
||||
);
|
||||
ReflectionTestUtils.setField(watcher, "tenantsConfigPath", configPath.toString());
|
||||
return watcher;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.magistr.app.service.TenantDatabaseMigrationService;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class TenantDataSourceConfigTest {
|
||||
|
||||
private static final Duration MAX_CONNECTIVITY_AGE = Duration.ofSeconds(30);
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
private final List<TenantRoutingDataSource> createdDataSources = new ArrayList<>();
|
||||
|
||||
@AfterEach
|
||||
void closeDataSources() {
|
||||
createdDataSources.forEach(routingDataSource -> routingDataSource
|
||||
.getTenantDataSource("default")
|
||||
.filter(HikariDataSource.class::isInstance)
|
||||
.map(HikariDataSource.class::cast)
|
||||
.ifPresent(HikariDataSource::close));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Обязательный отсутствующий tenants.json переводит readiness в состояние DOWN")
|
||||
void requiredMissingConfigMarksConfigurationFailure() {
|
||||
TenantReadinessRegistry registry = new TenantReadinessRegistry();
|
||||
|
||||
createDataSource(tempDir.resolve("missing-tenants.json"), true, registry);
|
||||
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = registry.aggregate(MAX_CONNECTIVITY_AGE);
|
||||
assertThat(readiness.configurationFailure()).isTrue();
|
||||
assertThat(readiness.ready()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Невалидный JSON переводит readiness в состояние DOWN")
|
||||
void invalidJsonMarksConfigurationFailure() throws IOException {
|
||||
Path configPath = tempDir.resolve("tenants.json");
|
||||
Files.writeString(configPath, "{невалидный-json");
|
||||
TenantReadinessRegistry registry = new TenantReadinessRegistry();
|
||||
|
||||
createDataSource(configPath, false, registry);
|
||||
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = registry.aggregate(MAX_CONNECTIVITY_AGE);
|
||||
assertThat(readiness.configurationFailure()).isTrue();
|
||||
assertThat(readiness.ready()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Необязательный отсутствующий tenants.json не создаёт ошибку конфигурации")
|
||||
void optionalMissingConfigDoesNotMarkConfigurationFailure() {
|
||||
TenantReadinessRegistry registry = new TenantReadinessRegistry();
|
||||
|
||||
createDataSource(tempDir.resolve("missing-tenants.json"), false, registry);
|
||||
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = registry.aggregate(MAX_CONNECTIVITY_AGE);
|
||||
assertThat(readiness.configurationFailure()).isFalse();
|
||||
}
|
||||
|
||||
private void createDataSource(Path configPath,
|
||||
boolean configRequired,
|
||||
TenantReadinessRegistry registry) {
|
||||
TenantDataSourceConfig config = new TenantDataSourceConfig();
|
||||
ReflectionTestUtils.setField(config, "tenantsConfigPath", configPath.toString());
|
||||
ReflectionTestUtils.setField(config, "tenantsConfigRequired", configRequired);
|
||||
ReflectionTestUtils.setField(config, "defaultDbUrl", "");
|
||||
ReflectionTestUtils.setField(config, "defaultDbUsername", "");
|
||||
ReflectionTestUtils.setField(config, "defaultDbPassword", "");
|
||||
|
||||
TenantRoutingDataSource routingDataSource = (TenantRoutingDataSource) config.dataSource(
|
||||
mock(TenantDatabaseMigrationService.class),
|
||||
registry
|
||||
);
|
||||
createdDataSources.add(routingDataSource);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import com.magistr.app.config.auth.AuthorizationInterceptor;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.util.AntPathMatcher;
|
||||
import org.springframework.web.servlet.handler.MappedInterceptor;
|
||||
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class TenantWebMvcConfigTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("Actuator probes исключены из проверки домена тенанта")
|
||||
void actuatorProbesAreExcludedFromTenantInterceptor() {
|
||||
TenantRoutingDataSource routingDataSource = mock(TenantRoutingDataSource.class);
|
||||
AuthorizationInterceptor authorizationInterceptor = mock(AuthorizationInterceptor.class);
|
||||
TenantWebMvcConfig config = new TenantWebMvcConfig(routingDataSource, authorizationInterceptor);
|
||||
ExposedInterceptorRegistry registry = new ExposedInterceptorRegistry();
|
||||
|
||||
config.addInterceptors(registry);
|
||||
|
||||
MappedInterceptor tenantInterceptor = (MappedInterceptor) registry.interceptors().get(0);
|
||||
AntPathMatcher pathMatcher = new AntPathMatcher();
|
||||
|
||||
assertThat(tenantInterceptor.matches("/actuator/health/liveness", pathMatcher)).isFalse();
|
||||
assertThat(tenantInterceptor.matches("/actuator/health/readiness", pathMatcher)).isFalse();
|
||||
assertThat(tenantInterceptor.matches("/api/schedule", pathMatcher)).isTrue();
|
||||
}
|
||||
|
||||
private static final class ExposedInterceptorRegistry extends InterceptorRegistry {
|
||||
|
||||
private List<Object> interceptors() {
|
||||
return getInterceptors();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringBootConfiguration;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
|
||||
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
@SpringBootTest(
|
||||
classes = HealthProbeEndpointIntegrationTest.ProbeApplication.class,
|
||||
webEnvironment = SpringBootTest.WebEnvironment.MOCK,
|
||||
properties = {
|
||||
"management.endpoints.web.exposure.include=health",
|
||||
"management.endpoint.health.probes.enabled=true",
|
||||
"management.endpoint.health.group.liveness.include=livenessState",
|
||||
"management.endpoint.health.group.readiness.include=readinessState,tenantReadiness",
|
||||
"management.endpoint.health.show-details=never",
|
||||
"management.endpoint.health.show-components=never"
|
||||
}
|
||||
)
|
||||
@AutoConfigureMockMvc
|
||||
class HealthProbeEndpointIntegrationTest {
|
||||
|
||||
@Autowired
|
||||
private MockMvc mockMvc;
|
||||
|
||||
@Autowired
|
||||
private TenantReadinessRegistry readinessRegistry;
|
||||
|
||||
@Test
|
||||
void отказTenantБдСнимаетReadinessНоНеLiveness() throws Exception {
|
||||
TenantConfig tenant = new TenantConfig(
|
||||
"Тестовый университет",
|
||||
"test",
|
||||
"jdbc:postgresql://database/test",
|
||||
"user",
|
||||
"secret"
|
||||
);
|
||||
readinessRegistry.replaceDesired(List.of(tenant));
|
||||
readinessRegistry.markMigrationFailed(tenant);
|
||||
|
||||
mockMvc.perform(get("/actuator/health/readiness").header("Host", "10.20.30.40"))
|
||||
.andExpect(status().isServiceUnavailable())
|
||||
.andExpect(jsonPath("$.status").value("DOWN"))
|
||||
.andExpect(jsonPath("$.components").doesNotExist());
|
||||
|
||||
mockMvc.perform(get("/actuator/health/liveness").header("Host", "10.20.30.40"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.status").value("UP"))
|
||||
.andExpect(jsonPath("$.components").doesNotExist());
|
||||
|
||||
readinessRegistry.markMigrationSucceeded(tenant);
|
||||
readinessRegistry.markConnectivity(tenant.getDomain(), true);
|
||||
|
||||
mockMvc.perform(get("/actuator/health/readiness").header("Host", "10.20.30.40"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.status").value("UP"));
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
@EnableAutoConfiguration(exclude = {
|
||||
DataSourceAutoConfiguration.class,
|
||||
HibernateJpaAutoConfiguration.class
|
||||
})
|
||||
static class ProbeApplication {
|
||||
|
||||
@Bean
|
||||
TenantReadinessRegistry tenantReadinessRegistry() {
|
||||
return new TenantReadinessRegistry();
|
||||
}
|
||||
|
||||
@Bean
|
||||
TenantDatabaseHealthMonitor tenantDatabaseHealthMonitor(TenantReadinessRegistry registry) {
|
||||
return new TenantDatabaseHealthMonitor(
|
||||
registry,
|
||||
new TenantRoutingDataSource(),
|
||||
1,
|
||||
Duration.ofSeconds(1),
|
||||
Duration.ofSeconds(30),
|
||||
Clock.systemUTC()
|
||||
);
|
||||
}
|
||||
|
||||
@Bean("tenantReadiness")
|
||||
TenantReadinessHealthIndicator tenantReadiness(
|
||||
TenantReadinessRegistry registry,
|
||||
TenantDatabaseHealthMonitor monitor) {
|
||||
return new TenantReadinessHealthIndicator(registry, monitor);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class TenantDatabaseHealthMonitorTest {
|
||||
|
||||
private static final byte[] KEY = "0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.UTF_8);
|
||||
private static final Instant FAST_COMPLETED_AT = Instant.parse("2026-07-15T10:00:01Z");
|
||||
private static final Instant TIMEOUT_RECORDED_AT = Instant.parse("2026-07-15T10:00:02Z");
|
||||
private static final Instant SLOW_COMPLETED_AT = Instant.parse("2026-07-15T10:00:03Z");
|
||||
|
||||
@Test
|
||||
void проверяетTenantПараллельноНоНеПревышаетЛимит() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
List<TenantConfig> tenants = new ArrayList<>();
|
||||
for (int index = 0; index < 6; index++) {
|
||||
tenants.add(config("tenant-" + index));
|
||||
}
|
||||
registry.replaceDesired(tenants);
|
||||
tenants.forEach(registry::markMigrationSucceeded);
|
||||
|
||||
ControlledRoutingDataSource routing = new ControlledRoutingDataSource(60, true);
|
||||
TenantDatabaseHealthMonitor monitor = monitor(registry, routing, 2, Duration.ofSeconds(1));
|
||||
try {
|
||||
TenantDatabaseHealthMonitor.RefreshSummary summary = monitor.refreshNow();
|
||||
|
||||
assertEquals(6, summary.requested());
|
||||
assertEquals(6, summary.available());
|
||||
assertEquals(0, summary.unavailable());
|
||||
assertTrue(routing.maxActive.get() >= 2);
|
||||
assertTrue(routing.maxActive.get() <= 2);
|
||||
assertTrue(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
} finally {
|
||||
monitor.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void превышениеТаймаутаОтмечаетСоединениеНедоступным() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig tenant = config("slow");
|
||||
registry.replaceDesired(List.of(tenant));
|
||||
registry.markMigrationSucceeded(tenant);
|
||||
|
||||
ControlledRoutingDataSource routing = new ControlledRoutingDataSource(300, true);
|
||||
TenantDatabaseHealthMonitor monitor = monitor(registry, routing, 1, Duration.ofMillis(25));
|
||||
try {
|
||||
TenantDatabaseHealthMonitor.RefreshSummary summary = monitor.refreshNow();
|
||||
|
||||
assertEquals(1, summary.requested());
|
||||
assertEquals(0, summary.available());
|
||||
assertEquals(1, summary.unavailable());
|
||||
assertFalse(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
} finally {
|
||||
monitor.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void таймаутПервогоTenantНеОтменяетУжеЗавершившийсяВторой() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig slow = config("slow");
|
||||
TenantConfig fast = config("fast");
|
||||
registry.replaceDesired(List.of(slow, fast));
|
||||
registry.markMigrationSucceeded(slow);
|
||||
registry.markMigrationSucceeded(fast);
|
||||
|
||||
CompletionAwareClock clock = new CompletionAwareClock();
|
||||
SlowFirstRoutingDataSource routing = new SlowFirstRoutingDataSource(clock);
|
||||
TenantDatabaseHealthMonitor monitor = new TenantDatabaseHealthMonitor(
|
||||
registry,
|
||||
routing,
|
||||
2,
|
||||
Duration.ofMillis(200),
|
||||
Duration.ofSeconds(30),
|
||||
clock);
|
||||
try {
|
||||
TenantDatabaseHealthMonitor.RefreshSummary summary = monitor.refreshNow();
|
||||
|
||||
assertEquals(2, summary.requested());
|
||||
assertEquals(1, summary.available());
|
||||
assertEquals(1, summary.unavailable());
|
||||
|
||||
TenantReadinessRegistry.TenantStatus slowStatus = registry.snapshot().get("slow");
|
||||
TenantReadinessRegistry.TenantStatus fastStatus = registry.snapshot().get("fast");
|
||||
assertEquals(TenantReadinessRegistry.ConnectivityState.UNAVAILABLE,
|
||||
slowStatus.connectivityState());
|
||||
assertEquals(TIMEOUT_RECORDED_AT, slowStatus.connectivityCheckedAt());
|
||||
assertEquals(TenantReadinessRegistry.ConnectivityState.AVAILABLE,
|
||||
fastStatus.connectivityState());
|
||||
assertEquals(FAST_COMPLETED_AT, fastStatus.connectivityCheckedAt());
|
||||
} finally {
|
||||
monitor.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void неПроверяетTenantДоУспехаМиграцииИХ2() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig ready = config("ready");
|
||||
TenantConfig pending = config("pending");
|
||||
TenantConfig h2 = new TenantConfig("h2", "h2", "jdbc:h2:mem:test", "sa", "");
|
||||
registry.replaceDesired(List.of(ready, pending, h2));
|
||||
registry.markMigrationSucceeded(ready);
|
||||
registry.markMigrationStarted(pending);
|
||||
registry.markMigrationSucceeded(h2);
|
||||
|
||||
ControlledRoutingDataSource routing = new ControlledRoutingDataSource(0, true);
|
||||
TenantDatabaseHealthMonitor monitor = monitor(registry, routing, 3, Duration.ofSeconds(1));
|
||||
try {
|
||||
TenantDatabaseHealthMonitor.RefreshSummary summary = monitor.refreshNow();
|
||||
|
||||
assertEquals(1, summary.requested());
|
||||
assertEquals(1, routing.total.get());
|
||||
assertFalse(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
} finally {
|
||||
monitor.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void сообщаетНастроенныйMaxStaleness() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantDatabaseHealthMonitor monitor = new TenantDatabaseHealthMonitor(
|
||||
registry,
|
||||
new ControlledRoutingDataSource(0, true),
|
||||
1,
|
||||
Duration.ofSeconds(1),
|
||||
Duration.ofSeconds(17),
|
||||
Clock.systemUTC());
|
||||
try {
|
||||
assertEquals(Duration.ofSeconds(17), monitor.maxStaleness());
|
||||
assertEquals(1, monitor.parallelism());
|
||||
} finally {
|
||||
monitor.close();
|
||||
}
|
||||
}
|
||||
|
||||
private TenantReadinessRegistry registry() {
|
||||
return new TenantReadinessRegistry(Clock.systemUTC(), KEY);
|
||||
}
|
||||
|
||||
private TenantDatabaseHealthMonitor monitor(TenantReadinessRegistry registry,
|
||||
TenantRoutingDataSource routing,
|
||||
int parallelism,
|
||||
Duration timeout) {
|
||||
return new TenantDatabaseHealthMonitor(
|
||||
registry,
|
||||
routing,
|
||||
parallelism,
|
||||
timeout,
|
||||
Duration.ofSeconds(30),
|
||||
Clock.systemUTC());
|
||||
}
|
||||
|
||||
private TenantConfig config(String domain) {
|
||||
return new TenantConfig(domain, domain, "jdbc:postgresql://db/" + domain, "user", "secret");
|
||||
}
|
||||
|
||||
private static final class ControlledRoutingDataSource extends TenantRoutingDataSource {
|
||||
private final long delayMillis;
|
||||
private final boolean result;
|
||||
private final AtomicInteger active = new AtomicInteger();
|
||||
private final AtomicInteger maxActive = new AtomicInteger();
|
||||
private final AtomicInteger total = new AtomicInteger();
|
||||
|
||||
private ControlledRoutingDataSource(long delayMillis, boolean result) {
|
||||
this.delayMillis = delayMillis;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean testConnection(String domain) {
|
||||
total.incrementAndGet();
|
||||
int current = active.incrementAndGet();
|
||||
maxActive.accumulateAndGet(current, Math::max);
|
||||
try {
|
||||
if (delayMillis > 0) {
|
||||
Thread.sleep(delayMillis);
|
||||
}
|
||||
return result;
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
} finally {
|
||||
active.decrementAndGet();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class SlowFirstRoutingDataSource extends TenantRoutingDataSource {
|
||||
private final CompletionAwareClock clock;
|
||||
private final CountDownLatch fastCompleted = new CountDownLatch(1);
|
||||
|
||||
private SlowFirstRoutingDataSource(CompletionAwareClock clock) {
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean testConnection(String domain) {
|
||||
clock.bind(domain);
|
||||
if ("fast".equals(domain)) {
|
||||
fastCompleted.countDown();
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
if (!fastCompleted.await(5, TimeUnit.SECONDS)) {
|
||||
return false;
|
||||
}
|
||||
Thread.sleep(1_000);
|
||||
return true;
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class CompletionAwareClock extends Clock {
|
||||
private final ThreadLocal<String> currentDomain = new ThreadLocal<>();
|
||||
|
||||
private void bind(String domain) {
|
||||
currentDomain.set(domain);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ZoneId getZone() {
|
||||
return ZoneOffset.UTC;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Clock withZone(ZoneId zone) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Instant instant() {
|
||||
return switch (String.valueOf(currentDomain.get())) {
|
||||
case "fast" -> FAST_COMPLETED_AT;
|
||||
case "slow" -> SLOW_COMPLETED_AT;
|
||||
default -> TIMEOUT_RECORDED_AT;
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.actuate.health.Status;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class TenantReadinessHealthIndicatorTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-07-15T10:00:00Z");
|
||||
|
||||
@Test
|
||||
void зарегистрированПодИменемTenantReadiness() {
|
||||
Component annotation = TenantReadinessHealthIndicator.class.getAnnotation(Component.class);
|
||||
|
||||
assertEquals("tenantReadiness", annotation.value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void возвращаетТолькоАгрегированныеДанные() {
|
||||
Clock clock = Clock.fixed(NOW, ZoneOffset.UTC);
|
||||
TenantReadinessRegistry registry = new TenantReadinessRegistry(
|
||||
clock,
|
||||
"0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.UTF_8));
|
||||
TenantConfig tenant = new TenantConfig(
|
||||
"Secret university",
|
||||
"alpha",
|
||||
"jdbc:postgresql://private-host/private-db",
|
||||
"private-user",
|
||||
"private-password");
|
||||
registry.replaceDesired(List.of(tenant));
|
||||
registry.markMigrationSucceeded(tenant);
|
||||
registry.markConnectivity("alpha", true);
|
||||
|
||||
TenantDatabaseHealthMonitor monitor = new TenantDatabaseHealthMonitor(
|
||||
registry,
|
||||
new TenantRoutingDataSource(),
|
||||
1,
|
||||
Duration.ofSeconds(1),
|
||||
Duration.ofSeconds(30),
|
||||
clock);
|
||||
try {
|
||||
TenantReadinessHealthIndicator indicator = new TenantReadinessHealthIndicator(registry, monitor);
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = indicator.readiness();
|
||||
|
||||
assertEquals(Status.UP, indicator.health().getStatus());
|
||||
assertTrue(readiness.ready());
|
||||
assertEquals("ГОТОВ", readiness.status());
|
||||
assertEquals(1, readiness.desiredTenants());
|
||||
assertEquals(1, readiness.readyTenants());
|
||||
assertEquals(0, readiness.unavailableTenants());
|
||||
assertFalse(readiness.configurationFailure());
|
||||
} finally {
|
||||
monitor.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package com.magistr.app.config.tenant.health;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class TenantReadinessRegistryTest {
|
||||
|
||||
private static final Instant NOW = Instant.parse("2026-07-15T10:00:00Z");
|
||||
private static final Clock CLOCK = Clock.fixed(NOW, ZoneOffset.UTC);
|
||||
private static final byte[] KEY = "0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
@Test
|
||||
void пустойСписокИH2НеДолжныБытьГотовы() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
|
||||
assertFalse(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
|
||||
TenantConfig h2 = config("default", "jdbc:h2:mem:placeholder", "sa", "секрет");
|
||||
registry.replaceDesired(List.of(h2));
|
||||
registry.markMigrationSucceeded(h2);
|
||||
|
||||
assertFalse(registry.markConnectivity("default", true));
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = registry.aggregate(Duration.ofSeconds(30));
|
||||
assertFalse(readiness.ready());
|
||||
assertEquals(1, readiness.unsupportedTenants());
|
||||
}
|
||||
|
||||
@Test
|
||||
void готовностьТребуетУспешнуюМиграциюИСвежееСоединение() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig tenant = config("alpha", "jdbc:postgresql://db/alpha", "user", "secret");
|
||||
|
||||
registry.replaceDesired(List.of(tenant));
|
||||
registry.markMigrationStarted(tenant);
|
||||
assertFalse(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
|
||||
registry.markMigrationSucceeded(tenant);
|
||||
assertFalse(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
|
||||
assertTrue(registry.markConnectivity("alpha", true));
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = registry.aggregate(Duration.ofSeconds(30));
|
||||
assertTrue(readiness.ready());
|
||||
assertEquals("ГОТОВ", readiness.status());
|
||||
assertEquals(1, readiness.readyTenants());
|
||||
}
|
||||
|
||||
@Test
|
||||
void устаревшаяПроверкаИFingerprintСтаройКонфигурацииИгнорируются() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig first = config("alpha", "jdbc:postgresql://db/alpha", "user", "first-secret");
|
||||
registry.replaceDesired(List.of(first));
|
||||
registry.markMigrationSucceeded(first);
|
||||
TenantReadinessRegistry.ConnectivityTarget oldTarget = registry.connectivityTargets().get(0);
|
||||
|
||||
TenantConfig changed = config("alpha", "jdbc:postgresql://db/alpha", "user", "second-secret");
|
||||
registry.replaceDesired(List.of(changed));
|
||||
registry.markMigrationSucceeded(changed);
|
||||
TenantReadinessRegistry.ConnectivityTarget currentTarget = registry.connectivityTargets().get(0);
|
||||
|
||||
assertNotEquals(oldTarget.fingerprint(), currentTarget.fingerprint());
|
||||
assertFalse(registry.markConnectivity("alpha", oldTarget.fingerprint(), true, NOW));
|
||||
assertTrue(registry.markConnectivity("alpha", currentTarget.fingerprint(), true, NOW));
|
||||
assertFalse(registry.markConnectivity(
|
||||
"alpha", currentTarget.fingerprint(), false, NOW.minusSeconds(1)));
|
||||
assertTrue(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
}
|
||||
|
||||
@Test
|
||||
void эквивалентнаяНормализованнаяКонфигурацияНеСбрасываетГотовность() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig raw = new TenantConfig(
|
||||
" Университет ",
|
||||
" ALPHA ",
|
||||
" jdbc:postgresql://db/alpha ",
|
||||
"user",
|
||||
"secret"
|
||||
);
|
||||
TenantConfig normalized = new TenantConfig(
|
||||
"Университет",
|
||||
"alpha",
|
||||
"jdbc:postgresql://db/alpha",
|
||||
"user",
|
||||
"secret"
|
||||
);
|
||||
|
||||
registry.replaceDesired(List.of(raw));
|
||||
registry.markMigrationSucceeded(raw);
|
||||
registry.markConnectivity("alpha", true);
|
||||
String before = registry.snapshot().get("alpha").fingerprint();
|
||||
|
||||
registry.replaceDesired(List.of(normalized));
|
||||
|
||||
assertEquals(before, registry.snapshot().get("alpha").fingerprint());
|
||||
assertTrue(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fingerprintНеРаскрываетРеквизиты() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig tenant = config(
|
||||
"secret-domain",
|
||||
"jdbc:postgresql://private-host/private-db",
|
||||
"private-user",
|
||||
"private-password");
|
||||
|
||||
registry.replaceDesired(List.of(tenant));
|
||||
String fingerprint = registry.snapshot().get("secret-domain").fingerprint();
|
||||
|
||||
assertFalse(fingerprint.contains("private-host"));
|
||||
assertFalse(fingerprint.contains("private-user"));
|
||||
assertFalse(fingerprint.contains("private-password"));
|
||||
assertFalse(fingerprint.contains("secret-domain"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void явнаяОшибкаКонфигурацииСохраняетсяДоЯвнойОчистки() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig tenant = config("alpha", "jdbc:postgresql://db/alpha", "user", "secret");
|
||||
|
||||
registry.markConfigurationFailure();
|
||||
registry.replaceDesired(List.of(tenant));
|
||||
registry.markMigrationSucceeded(tenant);
|
||||
registry.markConnectivity("alpha", true);
|
||||
|
||||
assertTrue(registry.aggregate(Duration.ofSeconds(30)).configurationFailure());
|
||||
assertFalse(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
|
||||
registry.clearConfigurationFailure();
|
||||
assertTrue(registry.aggregate(Duration.ofSeconds(30)).ready());
|
||||
}
|
||||
|
||||
@Test
|
||||
void просроченнаяПроверкаДелаетСервисНеготовым() {
|
||||
TenantReadinessRegistry registry = registry();
|
||||
TenantConfig tenant = config("alpha", "jdbc:postgresql://db/alpha", "user", "secret");
|
||||
registry.replaceDesired(List.of(tenant));
|
||||
registry.markMigrationSucceeded(tenant);
|
||||
TenantReadinessRegistry.ConnectivityTarget target = registry.connectivityTargets().get(0);
|
||||
|
||||
registry.markConnectivity("alpha", target.fingerprint(), true, NOW.minusSeconds(31));
|
||||
|
||||
TenantReadinessRegistry.AggregateReadiness readiness = registry.aggregate(Duration.ofSeconds(30));
|
||||
assertFalse(readiness.ready());
|
||||
assertEquals(1, readiness.staleConnections());
|
||||
}
|
||||
|
||||
private TenantReadinessRegistry registry() {
|
||||
return new TenantReadinessRegistry(CLOCK, KEY);
|
||||
}
|
||||
|
||||
private TenantConfig config(String domain, String url, String username, String password) {
|
||||
return new TenantConfig(domain, domain, url, username, password);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package com.magistr.app.controller;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfigWatcher;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import com.magistr.app.service.TenantLifecycleException;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
import static org.springframework.test.web.servlet.setup.MockMvcBuilders.standaloneSetup;
|
||||
|
||||
class DatabaseControllerTest {
|
||||
|
||||
private TenantLifecycleService lifecycleService;
|
||||
private TenantConfigWatcher tenantConfigWatcher;
|
||||
private MockMvc mockMvc;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
TenantRoutingDataSource routingDataSource = mock(TenantRoutingDataSource.class);
|
||||
lifecycleService = mock(TenantLifecycleService.class);
|
||||
tenantConfigWatcher = mock(TenantConfigWatcher.class);
|
||||
mockMvc = standaloneSetup(new DatabaseController(
|
||||
routingDataSource,
|
||||
lifecycleService,
|
||||
tenantConfigWatcher
|
||||
)).build();
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsServiceUnavailableInsteadOfFalseSuccessWhenLifecycleFails() throws Exception {
|
||||
when(lifecycleService.executeSerialized(
|
||||
org.mockito.ArgumentMatchers.<Supplier<ResponseEntity<Map<String, Object>>>>any()
|
||||
)).thenThrow(new TenantLifecycleException(
|
||||
"Не удалось выполнить миграции базы данных тенанта",
|
||||
new IllegalStateException("тестовые технические детали JDBC")
|
||||
));
|
||||
|
||||
mockMvc.perform(post("/api/database/tenants")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"name": "Тестовый университет",
|
||||
"domain": "test",
|
||||
"url": "jdbc:postgresql://db/test",
|
||||
"username": "user",
|
||||
"password": "password"
|
||||
}
|
||||
"""))
|
||||
.andExpect(status().isServiceUnavailable())
|
||||
.andExpect(jsonPath("$.success").value(false))
|
||||
.andExpect(jsonPath("$.message")
|
||||
.value("Не удалось выполнить миграции базы данных тенанта"))
|
||||
.andExpect(jsonPath("$.message")
|
||||
.value(org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("JDBC"))));
|
||||
|
||||
verify(tenantConfigWatcher, never()).refreshHash();
|
||||
verify(lifecycleService).executeSerialized(any(Supplier.class));
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,10 @@ import static org.mockito.Mockito.verify;
|
||||
@Testcontainers
|
||||
@SpringBootTest(
|
||||
classes = AcademicCalendarGridServiceIntegrationTest.TestApplication.class,
|
||||
properties = "spring.jpa.open-in-view=false"
|
||||
properties = {
|
||||
"spring.jpa.open-in-view=false",
|
||||
"management.endpoint.health.validate-group-membership=false"
|
||||
}
|
||||
)
|
||||
class AcademicCalendarGridServiceIntegrationTest {
|
||||
|
||||
|
||||
@@ -32,7 +32,8 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
classes = PostgreSqlAdvisoryLockIntegrationTest.TestApplication.class,
|
||||
properties = {
|
||||
"spring.flyway.enabled=false",
|
||||
"spring.jpa.hibernate.ddl-auto=none"
|
||||
"spring.jpa.hibernate.ddl-auto=none",
|
||||
"management.endpoint.health.validate-group-membership=false"
|
||||
}
|
||||
)
|
||||
class PostgreSqlAdvisoryLockIntegrationTest {
|
||||
|
||||
@@ -45,7 +45,10 @@ import static org.mockito.Mockito.verify;
|
||||
@Testcontainers
|
||||
@SpringBootTest(
|
||||
classes = ScheduleRuleServiceConcurrencyIntegrationTest.TestApplication.class,
|
||||
properties = "spring.jpa.open-in-view=false"
|
||||
properties = {
|
||||
"spring.jpa.open-in-view=false",
|
||||
"management.endpoint.health.validate-group-membership=false"
|
||||
}
|
||||
)
|
||||
class ScheduleRuleServiceConcurrencyIntegrationTest {
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package com.magistr.app.service;
|
||||
|
||||
import com.magistr.app.config.tenant.KubernetesTenantSecretUpdater;
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantConfigPersistenceException;
|
||||
import com.magistr.app.config.tenant.TenantConfigStore;
|
||||
import com.magistr.app.config.tenant.TenantContext;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -18,6 +21,7 @@ import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -42,7 +46,8 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
|
||||
private TenantRoutingDataSource routingDataSource;
|
||||
private TenantDatabaseMigrationService migrationService;
|
||||
private KubernetesTenantSecretUpdater tenantSecretUpdater;
|
||||
private TenantConfigStore tenantConfigStore;
|
||||
private TenantReadinessRegistry readinessRegistry;
|
||||
private TenantLifecycleService lifecycleService;
|
||||
private RetiredTenantPoolService retiredPoolService;
|
||||
private HikariDataSource oldPool;
|
||||
@@ -56,13 +61,15 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
|
||||
routingDataSource = new TenantRoutingDataSource();
|
||||
migrationService = new TenantDatabaseMigrationService();
|
||||
tenantSecretUpdater = mock(KubernetesTenantSecretUpdater.class);
|
||||
tenantConfigStore = mock(TenantConfigStore.class);
|
||||
readinessRegistry = new TenantReadinessRegistry();
|
||||
retiredPoolService = new RetiredTenantPoolService(Duration.ofSeconds(2), Duration.ofMillis(25));
|
||||
lifecycleService = new TenantLifecycleService(
|
||||
routingDataSource,
|
||||
tenantSecretUpdater,
|
||||
tenantConfigStore,
|
||||
migrationService,
|
||||
retiredPoolService
|
||||
retiredPoolService,
|
||||
readinessRegistry
|
||||
);
|
||||
|
||||
oldConfig = config("Старый tenant", jdbcUrl(OLD_DATABASE), POSTGRES.getPassword());
|
||||
@@ -77,6 +84,9 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
|
||||
oldPool = routingDataSource.prepareTenantDataSource(oldConfig);
|
||||
assertThat(routingDataSource.swapTenant(oldConfig, oldPool)).isNull();
|
||||
readinessRegistry.replaceDesired(List.of(oldConfig));
|
||||
readinessRegistry.markMigrationSucceeded(oldConfig);
|
||||
readinessRegistry.markConnectivity(DOMAIN, true);
|
||||
assertOldState();
|
||||
}
|
||||
|
||||
@@ -105,10 +115,10 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
assertThatThrownBy(() -> lifecycleService.addOrUpdateTenant(wrongCredentials))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось подключиться к базе данных тенанта");
|
||||
verify(tenantSecretUpdater, never()).updateTenantsConfig(any());
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
assertOldState();
|
||||
|
||||
reset(tenantSecretUpdater);
|
||||
reset(tenantConfigStore);
|
||||
TenantConfig badChecksum = config(
|
||||
"Повреждённый Flyway",
|
||||
jdbcUrl(BAD_CHECKSUM_DATABASE),
|
||||
@@ -117,11 +127,13 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
assertThatThrownBy(() -> lifecycleService.addOrUpdateTenant(badChecksum))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
|
||||
verify(tenantSecretUpdater, never()).updateTenantsConfig(any());
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
assertOldState();
|
||||
|
||||
reset(tenantSecretUpdater);
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(false, true);
|
||||
reset(tenantConfigStore);
|
||||
when(tenantConfigStore.upsertTenant(any())).thenThrow(
|
||||
new TenantConfigPersistenceException("Не удалось сохранить конфигурацию тенантов")
|
||||
);
|
||||
TenantConfig validCandidate = config(
|
||||
"Новый tenant",
|
||||
jdbcUrl(VALID_DATABASE),
|
||||
@@ -129,12 +141,19 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
);
|
||||
assertThatThrownBy(() -> lifecycleService.addOrUpdateTenant(validCandidate))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов; прежняя конфигурация восстановлена");
|
||||
verify(tenantSecretUpdater, org.mockito.Mockito.times(2)).updateTenantsConfig(any());
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов");
|
||||
verify(tenantConfigStore).upsertTenant(any());
|
||||
verify(tenantConfigStore, never()).compensate(any());
|
||||
assertOldState();
|
||||
|
||||
reset(tenantSecretUpdater);
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(true);
|
||||
reset(tenantConfigStore);
|
||||
when(tenantConfigStore.upsertTenant(any())).thenReturn(new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
true,
|
||||
"2",
|
||||
List.of(oldConfig),
|
||||
List.of(validCandidate)
|
||||
));
|
||||
TenantConfig saved;
|
||||
try (Connection inFlightOldConnection = oldPool.getConnection()) {
|
||||
saved = lifecycleService.addOrUpdateTenant(validCandidate);
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package com.magistr.app.service;
|
||||
|
||||
import com.magistr.app.config.tenant.KubernetesTenantSecretUpdater;
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantConfigPersistenceException;
|
||||
import com.magistr.app.config.tenant.TenantConfigStore;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import com.magistr.app.config.tenant.TenantSecretCompensationResult;
|
||||
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -16,6 +20,8 @@ import org.mockito.quality.Strictness;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -35,12 +41,14 @@ class TenantLifecycleServiceTest {
|
||||
@Mock
|
||||
private TenantRoutingDataSource routingDataSource;
|
||||
@Mock
|
||||
private KubernetesTenantSecretUpdater tenantSecretUpdater;
|
||||
private TenantConfigStore tenantConfigStore;
|
||||
@Mock
|
||||
private TenantDatabaseMigrationService migrationService;
|
||||
@Mock
|
||||
private RetiredTenantPoolService retiredPoolService;
|
||||
@Mock
|
||||
private TenantReadinessRegistry readinessRegistry;
|
||||
@Mock
|
||||
private HikariDataSource candidate;
|
||||
@Mock
|
||||
private HikariDataSource oldDataSource;
|
||||
@@ -56,9 +64,10 @@ class TenantLifecycleServiceTest {
|
||||
void setUp() throws Exception {
|
||||
service = new TenantLifecycleService(
|
||||
routingDataSource,
|
||||
tenantSecretUpdater,
|
||||
tenantConfigStore,
|
||||
migrationService,
|
||||
retiredPoolService
|
||||
retiredPoolService,
|
||||
readinessRegistry
|
||||
);
|
||||
oldAlpha = config("Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "old-password");
|
||||
beta = config("Beta", "beta", "jdbc:beta", "beta-user", "beta-password");
|
||||
@@ -70,7 +79,11 @@ class TenantLifecycleServiceTest {
|
||||
when(routingDataSource.prepareTenantDataSource(any(TenantConfig.class))).thenReturn(candidate);
|
||||
when(candidate.getConnection()).thenReturn(candidateConnection);
|
||||
when(candidateConnection.isValid(5)).thenReturn(true);
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(true);
|
||||
when(tenantConfigStore.upsertTenant(any(TenantConfig.class)))
|
||||
.thenAnswer(invocation -> upsertReceipt(invocation.getArgument(0)));
|
||||
when(tenantConfigStore.removeTenant(any(String.class))).thenAnswer(invocation ->
|
||||
removeReceipt(invocation.getArgument(0)));
|
||||
when(tenantConfigStore.compensate(any())).thenReturn(TenantSecretCompensationResult.RESTORED);
|
||||
when(routingDataSource.swapTenant(any(TenantConfig.class), any()))
|
||||
.thenReturn(new TenantRoutingDataSource.TenantState("alpha", oldAlpha, oldDataSource));
|
||||
}
|
||||
@@ -85,13 +98,14 @@ class TenantLifecycleServiceTest {
|
||||
|
||||
assertConfig(saved, "Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
ArgumentCaptor<TenantConfig> preparedConfig = ArgumentCaptor.forClass(TenantConfig.class);
|
||||
ArgumentCaptor<List<TenantConfig>> persistedConfigs = listCaptor();
|
||||
ArgumentCaptor<TenantConfig> persistedConfig = ArgumentCaptor.forClass(TenantConfig.class);
|
||||
InOrder order = inOrder(
|
||||
routingDataSource,
|
||||
candidate,
|
||||
candidateConnection,
|
||||
migrationService,
|
||||
tenantSecretUpdater,
|
||||
tenantConfigStore,
|
||||
readinessRegistry,
|
||||
retiredPoolService
|
||||
);
|
||||
order.verify(routingDataSource).snapshotTenantConfigs();
|
||||
@@ -100,19 +114,63 @@ class TenantLifecycleServiceTest {
|
||||
order.verify(candidateConnection).isValid(5);
|
||||
order.verify(candidateConnection).close();
|
||||
order.verify(migrationService).migrate(candidate);
|
||||
order.verify(tenantSecretUpdater).updateTenantsConfig(persistedConfigs.capture());
|
||||
order.verify(tenantConfigStore).upsertTenant(persistedConfig.capture());
|
||||
order.verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
|
||||
order.verify(readinessRegistry).replaceDesired(any());
|
||||
order.verify(readinessRegistry).markMigrationSucceeded(any(TenantConfig.class));
|
||||
order.verify(readinessRegistry).markConnectivity("alpha", true);
|
||||
order.verify(retiredPoolService).retire(oldDataSource);
|
||||
|
||||
assertConfig(preparedConfig.getValue(),
|
||||
"Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
assertThat(persistedConfigs.getValue()).extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "beta");
|
||||
assertConfig(persistedConfigs.getValue().get(0),
|
||||
assertConfig(persistedConfig.getValue(),
|
||||
"Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
verify(candidate, never()).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void addsNewTenantWithoutRetiringExistingPool() {
|
||||
Map<String, TenantConfig> createSnapshot = new LinkedHashMap<>();
|
||||
createSnapshot.put("beta", beta);
|
||||
when(routingDataSource.snapshotTenantConfigs()).thenReturn(createSnapshot);
|
||||
when(routingDataSource.swapTenant(any(TenantConfig.class), any())).thenReturn(null);
|
||||
TenantConfig gamma = config(
|
||||
"Новый Gamma", " GAMMA ", " jdbc:gamma ", "gamma-user", "gamma-password"
|
||||
);
|
||||
|
||||
TenantConfig saved = service.addOrUpdateTenant(gamma);
|
||||
|
||||
assertConfig(saved, "Новый Gamma", "gamma", "jdbc:gamma", "gamma-user", "gamma-password");
|
||||
ArgumentCaptor<TenantConfig> persistedConfig = ArgumentCaptor.forClass(TenantConfig.class);
|
||||
verify(tenantConfigStore).upsertTenant(persistedConfig.capture());
|
||||
assertThat(persistedConfig.getValue().getDomain()).isEqualTo("gamma");
|
||||
verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
|
||||
verify(retiredPoolService, never()).retire(any());
|
||||
verify(candidate, never()).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotBlindlyCompensateWhenCreatePersistenceFails() {
|
||||
Map<String, TenantConfig> createSnapshot = new LinkedHashMap<>();
|
||||
createSnapshot.put("beta", beta);
|
||||
when(routingDataSource.snapshotTenantConfigs()).thenReturn(createSnapshot);
|
||||
when(tenantConfigStore.upsertTenant(any()))
|
||||
.thenThrow(new TenantConfigPersistenceException("Не удалось сохранить конфигурацию тенантов"));
|
||||
TenantConfig gamma = config(
|
||||
"Новый Gamma", "gamma", "jdbc:gamma", "gamma-user", "gamma-password"
|
||||
);
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(gamma))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов");
|
||||
|
||||
verify(tenantConfigStore).upsertTenant(any());
|
||||
verify(tenantConfigStore, never()).compensate(any());
|
||||
verify(candidate).close();
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void closesCandidateAndKeepsOldPoolWhenCredentialsAreWrong() throws Exception {
|
||||
when(candidate.getConnection()).thenThrow(new SQLException("тестовая ошибка аутентификации"));
|
||||
@@ -123,11 +181,27 @@ class TenantLifecycleServiceTest {
|
||||
|
||||
verify(candidate).close();
|
||||
verify(migrationService, never()).migrate(any());
|
||||
verify(tenantSecretUpdater, never()).updateTenantsConfig(any());
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void closesCandidateAndDoesNotPersistWhenConnectionIsInvalid() throws Exception {
|
||||
when(candidateConnection.isValid(5)).thenReturn(false);
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(newAlpha()))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("База данных тенанта не подтвердила готовность подключения");
|
||||
|
||||
verify(candidateConnection).close();
|
||||
verify(candidate).close();
|
||||
verify(migrationService, never()).migrate(any());
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void closesCandidateAndKeepsOldStateWhenFlywayFails() {
|
||||
org.mockito.Mockito.doThrow(new IllegalStateException("тестовая ошибка Flyway"))
|
||||
@@ -138,134 +212,141 @@ class TenantLifecycleServiceTest {
|
||||
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
|
||||
|
||||
verify(candidate).close();
|
||||
verify(tenantSecretUpdater, never()).updateTenantsConfig(any());
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void closesCandidateAndRestoresOldSnapshotWhenPersistenceReturnsFalse() {
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(false, true);
|
||||
void persistenceConflictDoesNotStartBlindCompensation() {
|
||||
when(tenantConfigStore.upsertTenant(any())).thenThrow(new TenantConfigPersistenceException(
|
||||
"Конфигурация тенантов одновременно изменяется другим экземпляром; повторите запрос"
|
||||
));
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(newAlpha()))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов; прежняя конфигурация восстановлена");
|
||||
.hasMessage("Конфигурация тенантов одновременно изменяется другим экземпляром; повторите запрос");
|
||||
|
||||
ArgumentCaptor<List<TenantConfig>> snapshots = listCaptor();
|
||||
verify(tenantSecretUpdater, org.mockito.Mockito.times(2)).updateTenantsConfig(snapshots.capture());
|
||||
assertThat(snapshots.getAllValues().get(1)).extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "beta");
|
||||
assertConfig(snapshots.getAllValues().get(1).get(0),
|
||||
"Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "old-password");
|
||||
verify(tenantConfigStore, never()).compensate(any());
|
||||
verify(candidate).close();
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void restoresOldSnapshotWhenFirstPersistenceThrows() {
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any()))
|
||||
.thenThrow(new IllegalStateException("неопределённый исход PATCH"))
|
||||
.thenReturn(true);
|
||||
void unknownPersistenceFailureUsesSafeMessageAndDoesNotCompensate() {
|
||||
when(tenantConfigStore.upsertTenant(any()))
|
||||
.thenThrow(new IllegalStateException("неопределённый технический результат PUT"));
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(newAlpha()))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов; прежняя конфигурация восстановлена");
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов");
|
||||
|
||||
verify(tenantSecretUpdater, org.mockito.Mockito.times(2)).updateTenantsConfig(any());
|
||||
verify(tenantConfigStore, never()).compensate(any());
|
||||
verify(candidate).close();
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportsDesynchronizationWhenPersistenceAndCompensationFail() {
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(false, false);
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(newAlpha()))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось сохранить конфигурацию тенантов; требуется проверить tenant Secret");
|
||||
|
||||
verify(tenantSecretUpdater, org.mockito.Mockito.times(2)).updateTenantsConfig(any());
|
||||
verify(candidate).close();
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void compensatesPersistedSnapshotWhenActivationFails() {
|
||||
void compensatesOnlyCommittedReceiptWhenActivationFails() {
|
||||
TenantSecretUpdateReceipt receipt = upsertReceipt(newAlpha());
|
||||
when(tenantConfigStore.upsertTenant(any())).thenReturn(receipt);
|
||||
when(routingDataSource.swapTenant(any(), any()))
|
||||
.thenThrow(new IllegalStateException("тестовый отказ активации"));
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(true, true);
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(newAlpha()))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось активировать подключение тенанта; сохранённая конфигурация восстановлена");
|
||||
|
||||
ArgumentCaptor<List<TenantConfig>> snapshots = listCaptor();
|
||||
verify(tenantSecretUpdater, org.mockito.Mockito.times(2)).updateTenantsConfig(snapshots.capture());
|
||||
assertThat(snapshots.getAllValues().get(0)).extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "beta");
|
||||
assertConfig(snapshots.getAllValues().get(0).get(0),
|
||||
"Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
assertThat(snapshots.getAllValues().get(1)).extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "beta");
|
||||
assertConfig(snapshots.getAllValues().get(1).get(0),
|
||||
"Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "old-password");
|
||||
verify(tenantConfigStore).compensate(receipt);
|
||||
verify(candidate).close();
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportsSecretDesynchronizationWhenActivationAndCompensationFail() {
|
||||
void concurrentChangeSkipsUnsafeCompensationAndMarksReadinessFailed() {
|
||||
TenantSecretUpdateReceipt receipt = upsertReceipt(newAlpha());
|
||||
when(tenantConfigStore.upsertTenant(any())).thenReturn(receipt);
|
||||
when(tenantConfigStore.compensate(receipt))
|
||||
.thenReturn(TenantSecretCompensationResult.SKIPPED_CONCURRENT_CHANGE);
|
||||
when(routingDataSource.swapTenant(any(), any()))
|
||||
.thenThrow(new IllegalStateException("тестовый отказ активации"));
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(true, false);
|
||||
|
||||
assertThatThrownBy(() -> service.addOrUpdateTenant(newAlpha()))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось активировать подключение тенанта; требуется проверить tenant Secret");
|
||||
.hasMessage("Не удалось активировать подключение тенанта; tenant Secret уже изменён "
|
||||
+ "другим экземпляром, автоматический откат пропущен");
|
||||
|
||||
verify(readinessRegistry).markConfigurationFailure();
|
||||
verify(candidate).close();
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void removalPersistsDesiredStateBeforeAtomicRemovalAndClosesRemovedPool() {
|
||||
void removalPersistsOperationBeforeAtomicRemovalAndClosesRemovedPool() {
|
||||
when(routingDataSource.removeTenantAtomically("alpha"))
|
||||
.thenReturn(new TenantRoutingDataSource.TenantState("alpha", oldAlpha, oldDataSource));
|
||||
|
||||
TenantConfig removed = service.removeTenant(" ALPHA ");
|
||||
|
||||
assertConfig(removed, "Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "old-password");
|
||||
ArgumentCaptor<List<TenantConfig>> desired = listCaptor();
|
||||
InOrder order = inOrder(routingDataSource, tenantSecretUpdater, retiredPoolService);
|
||||
InOrder order = inOrder(routingDataSource, tenantConfigStore, readinessRegistry, retiredPoolService);
|
||||
order.verify(routingDataSource).snapshotTenantConfigs();
|
||||
order.verify(tenantSecretUpdater).updateTenantsConfig(desired.capture());
|
||||
order.verify(tenantConfigStore).removeTenant("alpha");
|
||||
order.verify(routingDataSource).removeTenantAtomically("alpha");
|
||||
order.verify(readinessRegistry).replaceDesired(any());
|
||||
order.verify(retiredPoolService).retire(oldDataSource);
|
||||
assertThat(desired.getValue()).extracting(TenantConfig::getDomain).containsExactly("beta");
|
||||
}
|
||||
|
||||
@Test
|
||||
void removalFailureRestoresOldPersistedSnapshotAndKeepsPoolOpen() {
|
||||
void removalFailureCompensatesCommittedReceiptAndKeepsPoolOpen() {
|
||||
TenantSecretUpdateReceipt receipt = removeReceipt("alpha");
|
||||
when(tenantConfigStore.removeTenant("alpha")).thenReturn(receipt);
|
||||
when(routingDataSource.removeTenantAtomically("alpha"))
|
||||
.thenThrow(new IllegalStateException("тестовый отказ удаления"));
|
||||
when(tenantSecretUpdater.updateTenantsConfig(any())).thenReturn(true, true);
|
||||
|
||||
assertThatThrownBy(() -> service.removeTenant("alpha"))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось отключить тенанта; сохранённая конфигурация восстановлена");
|
||||
|
||||
ArgumentCaptor<List<TenantConfig>> snapshots = listCaptor();
|
||||
verify(tenantSecretUpdater, org.mockito.Mockito.times(2)).updateTenantsConfig(snapshots.capture());
|
||||
assertThat(snapshots.getAllValues().get(0)).extracting(TenantConfig::getDomain)
|
||||
.containsExactly("beta");
|
||||
assertThat(snapshots.getAllValues().get(1)).extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "beta");
|
||||
verify(tenantConfigStore).compensate(receipt);
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void persistedTenantBecomesReadyOnlyAfterMigrationAndSwap() {
|
||||
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "user", "password");
|
||||
when(routingDataSource.hasTenant("gamma")).thenReturn(false);
|
||||
when(routingDataSource.swapTenant(any(TenantConfig.class), any())).thenReturn(null);
|
||||
|
||||
service.synchronizeFromPersistedConfig(List.of(gamma));
|
||||
|
||||
InOrder order = inOrder(readinessRegistry, migrationService, routingDataSource);
|
||||
order.verify(readinessRegistry).clearConfigurationFailure();
|
||||
order.verify(readinessRegistry).replaceDesired(any());
|
||||
order.verify(readinessRegistry).markMigrationStarted(any(TenantConfig.class));
|
||||
order.verify(migrationService).migrate(candidate);
|
||||
order.verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
|
||||
order.verify(readinessRegistry).markMigrationSucceeded(any(TenantConfig.class));
|
||||
order.verify(readinessRegistry).markConnectivity("gamma", true);
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedPersistedMigrationLeavesTenantNotReadyForWatcherRetry() {
|
||||
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "user", "password");
|
||||
when(routingDataSource.hasTenant("gamma")).thenReturn(false);
|
||||
org.mockito.Mockito.doThrow(new IllegalStateException("тестовая ошибка Flyway"))
|
||||
.when(migrationService).migrate(candidate);
|
||||
|
||||
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(gamma)))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
|
||||
|
||||
verify(readinessRegistry).markMigrationStarted(any(TenantConfig.class));
|
||||
verify(readinessRegistry).markMigrationFailed(any(TenantConfig.class));
|
||||
verify(readinessRegistry, never()).markMigrationSucceeded(any(TenantConfig.class));
|
||||
verify(candidate).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsDnsLabelBoundariesAndRejectsTooLongOrDottedDomain() {
|
||||
String maxLengthDomain = "a".repeat(63);
|
||||
@@ -295,6 +376,36 @@ class TenantLifecycleServiceTest {
|
||||
return config("Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt upsertReceipt(TenantConfig tenant) {
|
||||
Map<String, TenantConfig> committed = new LinkedHashMap<>(oldSnapshot);
|
||||
committed.put(tenant.getDomain(), tenant);
|
||||
return new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
true,
|
||||
"2",
|
||||
sortedConfigs(oldSnapshot),
|
||||
sortedConfigs(committed)
|
||||
);
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt removeReceipt(String domain) {
|
||||
Map<String, TenantConfig> committed = new LinkedHashMap<>(oldSnapshot);
|
||||
committed.remove(domain);
|
||||
return new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
true,
|
||||
"2",
|
||||
sortedConfigs(oldSnapshot),
|
||||
sortedConfigs(committed)
|
||||
);
|
||||
}
|
||||
|
||||
private List<TenantConfig> sortedConfigs(Map<String, TenantConfig> configs) {
|
||||
List<TenantConfig> result = new ArrayList<>(configs.values());
|
||||
result.sort(Comparator.comparing(TenantConfig::getDomain));
|
||||
return result;
|
||||
}
|
||||
|
||||
private TenantConfig config(String name,
|
||||
String domain,
|
||||
String url,
|
||||
@@ -316,8 +427,4 @@ class TenantLifecycleServiceTest {
|
||||
assertThat(actual.getPassword()).isEqualTo(password);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
private ArgumentCaptor<List<TenantConfig>> listCaptor() {
|
||||
return ArgumentCaptor.forClass((Class) List.class);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user