Задача Егора 2ч.
This commit is contained in:
@@ -3,6 +3,7 @@ package com.magistr.app.config.tenant;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.magistr.app.service.TenantLifecycleException;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -10,24 +11,35 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.LongSupplier;
|
||||
|
||||
/**
|
||||
* Периодически перечитывает tenants.json (mounted Secret).
|
||||
* Если Secret был обновлён через K8s API, этот компонент
|
||||
* подхватит изменения и синхронизирует in-memory datasource'ы.
|
||||
*
|
||||
* Также отвечает за инициализацию БД (init.sql) для новых тенантов.
|
||||
* Периодически применяет конфигурацию тенантов из смонтированного tenants.json.
|
||||
* Все чтения и lifecycle-изменения сериализуются одним monitor'ом сервиса.
|
||||
*/
|
||||
@Component
|
||||
public class TenantConfigWatcher {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TenantConfigWatcher.class);
|
||||
private static final String MISSING_FAILURE_KEY = "missing";
|
||||
private static final String READ_FAILURE_KEY = "read";
|
||||
|
||||
static final long RETRY_BASE_DELAY_MILLIS = 30_000L;
|
||||
static final long RETRY_MAX_DELAY_MILLIS = 300_000L;
|
||||
|
||||
private final TenantLifecycleService tenantLifecycleService;
|
||||
private final TenantReadinessRegistry readinessRegistry;
|
||||
@@ -39,9 +51,23 @@ public class TenantConfigWatcher {
|
||||
@Value("${app.tenants.config-required:false}")
|
||||
private boolean tenantsConfigRequired;
|
||||
|
||||
// Хеш последнего прочитанного конфига — чтобы не перезагружать зря
|
||||
/** Хеш означает только полностью применённое содержимое файла. */
|
||||
private String lastConfigHash = "";
|
||||
private ConfigSnapshot lastAppliedSnapshot;
|
||||
|
||||
private String failedRevisionKey = "";
|
||||
private int consecutiveFailures;
|
||||
private long retryNotBeforeMillis;
|
||||
|
||||
/**
|
||||
* Fence хранит семантические снимки, а не байтовые представления JSON.
|
||||
* Поэтому форматирование файла не может ошибочно снять или установить fence.
|
||||
*/
|
||||
private ConfigSnapshot expectedCommittedSnapshot;
|
||||
private final Set<ConfigSnapshot> deferredSnapshots = new LinkedHashSet<>();
|
||||
|
||||
private boolean configurationUnavailable;
|
||||
private LongSupplier currentTimeMillis = System::currentTimeMillis;
|
||||
|
||||
public TenantConfigWatcher(TenantLifecycleService tenantLifecycleService,
|
||||
TenantReadinessRegistry readinessRegistry) {
|
||||
@@ -49,73 +75,135 @@ public class TenantConfigWatcher {
|
||||
this.readinessRegistry = readinessRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Каждые 30 секунд проверяет, изменился ли tenants.json.
|
||||
*/
|
||||
@Scheduled(fixedDelay = 30_000, initialDelay = 30_000)
|
||||
public void watchForChanges() {
|
||||
tenantLifecycleService.executeSerialized(this::watchForChangesSerialized);
|
||||
}
|
||||
|
||||
private void watchForChangesSerialized() {
|
||||
long now = currentTimeMillis.getAsLong();
|
||||
Path path = Path.of(tenantsConfigPath);
|
||||
if (!Files.exists(path)) {
|
||||
if (tenantsConfigRequired) {
|
||||
handleRequiredMissingFile(now);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (isRetryDeferred(READ_FAILURE_KEY, now)) {
|
||||
logDeferredRetry(now);
|
||||
return;
|
||||
}
|
||||
|
||||
String failureKey = READ_FAILURE_KEY;
|
||||
try {
|
||||
File file = new File(tenantsConfigPath);
|
||||
if (!file.exists()) {
|
||||
if (tenantsConfigRequired) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
if (!configurationUnavailable) {
|
||||
log.error("Обязательный файл конфигурации тенантов недоступен");
|
||||
}
|
||||
configurationUnavailable = true;
|
||||
}
|
||||
String content = Files.readString(path, StandardCharsets.UTF_8);
|
||||
String hash = configHash(content);
|
||||
failureKey = hash;
|
||||
Projection projection = parseProjection(content, hash);
|
||||
|
||||
// Даже неизменившееся содержимое сначала разбирается: fence никогда не
|
||||
// принимает решение по сырым байтам до проверки структуры JSON.
|
||||
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
|
||||
clearFenceWhenExpectedSnapshotIsAlreadyApplied(projection.snapshot());
|
||||
return;
|
||||
}
|
||||
if (isRetryDeferred(hash, now)) {
|
||||
logDeferredRetry(now);
|
||||
return;
|
||||
}
|
||||
if (!hash.equals(failedRevisionKey)) {
|
||||
resetRetryState();
|
||||
}
|
||||
if (shouldDefer(projection.snapshot())) {
|
||||
return;
|
||||
}
|
||||
|
||||
String content = new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
|
||||
String hash = configHash(content);
|
||||
|
||||
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
|
||||
return; // Ничего не изменилось
|
||||
}
|
||||
|
||||
log.info("Обнаружено изменение tenants.json (хеш: {} -> {}), перечитываем конфиг", lastConfigHash, hash);
|
||||
List<TenantConfig> newTenants = objectMapper.readValue(content, new TypeReference<>() {});
|
||||
syncTenants(newTenants);
|
||||
lastConfigHash = hash;
|
||||
configurationUnavailable = false;
|
||||
|
||||
} catch (Exception e) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
log.error("Ошибка при проверке конфига тенантов: errorType={}", e.getClass().getSimpleName());
|
||||
log.debug("Технические детали синхронизации конфига тенантов", e);
|
||||
log.info("Обнаружено изменение tenants.json, выполняется синхронизация конфигурации тенантов");
|
||||
syncTenants(projection.tenants());
|
||||
markApplied(projection);
|
||||
} catch (Exception failure) {
|
||||
recordFailure(failureKey, failure);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Обновляет хеш конфига после ручного обновления Secret с этого же пода.
|
||||
* Перед API-мутацией приводит runtime к актуальной безопасной проекции файла.
|
||||
* Ошибка чтения или синхронизации не скрывается: API-операция не должна начинаться
|
||||
* с неизвестного baseline.
|
||||
*/
|
||||
public void refreshHash() {
|
||||
tenantLifecycleService.executeSerialized(this::refreshHashSerialized);
|
||||
public void prepareForApiMutation() {
|
||||
tenantLifecycleService.executeSerialized(this::prepareForApiMutationSerialized);
|
||||
}
|
||||
|
||||
private void refreshHashSerialized() {
|
||||
try {
|
||||
File file = new File(tenantsConfigPath);
|
||||
if (file.exists()) {
|
||||
String content = new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
|
||||
lastConfigHash = configHash(content);
|
||||
} else if (tenantsConfigRequired) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
private void prepareForApiMutationSerialized() {
|
||||
Path path = Path.of(tenantsConfigPath);
|
||||
if (!Files.exists(path)) {
|
||||
if (!tenantsConfigRequired) {
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
log.warn("Не удалось обновить хеш конфига тенантов: errorType={}",
|
||||
e.getClass().getSimpleName());
|
||||
log.debug("Технические детали обновления хеша тенантов", e);
|
||||
TenantLifecycleException failure = new TenantLifecycleException(
|
||||
"Обязательный файл конфигурации тенантов недоступен"
|
||||
);
|
||||
recordFailure(MISSING_FAILURE_KEY, failure);
|
||||
throw failure;
|
||||
}
|
||||
|
||||
String failureKey = READ_FAILURE_KEY;
|
||||
try {
|
||||
String content = Files.readString(path, StandardCharsets.UTF_8);
|
||||
String hash = configHash(content);
|
||||
failureKey = hash;
|
||||
Projection projection = parseProjection(content, hash);
|
||||
|
||||
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
|
||||
clearFenceWhenExpectedSnapshotIsAlreadyApplied(projection.snapshot());
|
||||
return;
|
||||
}
|
||||
if (shouldDefer(projection.snapshot())) {
|
||||
return;
|
||||
}
|
||||
|
||||
syncTenants(projection.tenants());
|
||||
markApplied(projection);
|
||||
} catch (Exception failure) {
|
||||
recordFailure(failureKey, failure);
|
||||
if (failure instanceof TenantLifecycleException lifecycleFailure) {
|
||||
throw lifecycleFailure;
|
||||
}
|
||||
throw new TenantLifecycleException(
|
||||
"Не удалось подготовить актуальную конфигурацию тенантов перед изменением",
|
||||
failure
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Устанавливает snapshot fence только после подтверждённой записи в общее
|
||||
* хранилище. Локальные и no-op операции не создают ложных ожиданий от mount.
|
||||
*/
|
||||
public void registerSuccessfulMutation(TenantSecretUpdateReceipt receipt) {
|
||||
Objects.requireNonNull(receipt, "Квитанция изменения tenant Secret не задана");
|
||||
tenantLifecycleService.executeSerialized(() -> registerSuccessfulMutationSerialized(receipt));
|
||||
}
|
||||
|
||||
private void registerSuccessfulMutationSerialized(TenantSecretUpdateReceipt receipt) {
|
||||
if (!receipt.persisted() || !receipt.changed()) {
|
||||
return;
|
||||
}
|
||||
|
||||
ConfigSnapshot previous = snapshotOf(receipt.previousTenants());
|
||||
ConfigSnapshot committed = snapshotOf(receipt.committedTenants());
|
||||
|
||||
if (lastAppliedSnapshot != null) {
|
||||
deferredSnapshots.add(lastAppliedSnapshot);
|
||||
}
|
||||
if (expectedCommittedSnapshot != null) {
|
||||
deferredSnapshots.add(expectedCommittedSnapshot);
|
||||
}
|
||||
deferredSnapshots.add(previous);
|
||||
deferredSnapshots.remove(committed);
|
||||
expectedCommittedSnapshot = committed;
|
||||
}
|
||||
|
||||
static String configHash(String content) {
|
||||
@@ -127,11 +215,158 @@ public class TenantConfigWatcher {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Синхронизирует in-memory тенантов с конфигом из файла.
|
||||
*/
|
||||
private void syncTenants(List<TenantConfig> newTenants) {
|
||||
tenantLifecycleService.synchronizeFromPersistedConfig(newTenants);
|
||||
private Projection parseProjection(String content, String hash) throws IOException {
|
||||
List<TenantConfig> tenants = objectMapper.readValue(content, new TypeReference<>() { });
|
||||
if (tenants == null) {
|
||||
throw new IllegalArgumentException(
|
||||
"Корневое значение конфигурации тенантов должно быть массивом"
|
||||
);
|
||||
}
|
||||
if (tenantsConfigRequired && tenants.isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Обязательная конфигурация тенантов не может быть пустой"
|
||||
);
|
||||
}
|
||||
return new Projection(hash, List.copyOf(tenants), snapshotOf(tenants));
|
||||
}
|
||||
|
||||
private ConfigSnapshot snapshotOf(List<TenantConfig> tenants) {
|
||||
Objects.requireNonNull(tenants, "Список конфигураций тенантов не задан");
|
||||
List<NormalizedTenantConfig> normalized = new ArrayList<>(tenants.size());
|
||||
for (TenantConfig tenant : tenants) {
|
||||
Objects.requireNonNull(tenant, "Конфигурация тенанта не задана");
|
||||
String domain = normalizeDomain(tenant.getDomain());
|
||||
String name = tenant.getName() == null || tenant.getName().isBlank()
|
||||
? domain
|
||||
: tenant.getName().trim();
|
||||
String url = tenant.getUrl() == null ? null : tenant.getUrl().trim();
|
||||
normalized.add(new NormalizedTenantConfig(
|
||||
name,
|
||||
domain,
|
||||
url,
|
||||
tenant.getUsername(),
|
||||
tenant.getPassword()
|
||||
));
|
||||
}
|
||||
normalized.sort(NormalizedTenantConfig.ORDER);
|
||||
return new ConfigSnapshot(normalized);
|
||||
}
|
||||
|
||||
private String normalizeDomain(String domain) {
|
||||
return domain == null ? null : domain.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private void syncTenants(List<TenantConfig> tenants) {
|
||||
tenantLifecycleService.synchronizeFromPersistedConfig(tenants);
|
||||
}
|
||||
|
||||
private boolean shouldDefer(ConfigSnapshot snapshot) {
|
||||
return expectedCommittedSnapshot != null && deferredSnapshots.contains(snapshot);
|
||||
}
|
||||
|
||||
private void clearFenceWhenExpectedSnapshotIsAlreadyApplied(ConfigSnapshot snapshot) {
|
||||
if (snapshot.equals(expectedCommittedSnapshot)) {
|
||||
clearSnapshotFence();
|
||||
}
|
||||
}
|
||||
|
||||
private void markApplied(Projection projection) {
|
||||
lastConfigHash = projection.hash();
|
||||
lastAppliedSnapshot = projection.snapshot();
|
||||
configurationUnavailable = false;
|
||||
resetRetryState();
|
||||
clearSnapshotFence();
|
||||
}
|
||||
|
||||
private void handleRequiredMissingFile(long now) {
|
||||
if (isRetryDeferred(MISSING_FAILURE_KEY, now)) {
|
||||
logDeferredRetry(now);
|
||||
return;
|
||||
}
|
||||
recordFailure(
|
||||
MISSING_FAILURE_KEY,
|
||||
new TenantLifecycleException("Обязательный файл конфигурации тенантов недоступен")
|
||||
);
|
||||
}
|
||||
|
||||
private boolean isRetryDeferred(String failureKey, long now) {
|
||||
return failureKey.equals(failedRevisionKey) && now < retryNotBeforeMillis;
|
||||
}
|
||||
|
||||
private void logDeferredRetry(long now) {
|
||||
log.debug("Повторная синхронизация tenants.json отложена ещё на {} мс",
|
||||
Math.max(0L, retryNotBeforeMillis - now));
|
||||
}
|
||||
|
||||
private void recordFailure(String failureKey, Exception failure) {
|
||||
readinessRegistry.markConfigurationFailure();
|
||||
configurationUnavailable = true;
|
||||
long retryDelay = registerFailure(failureKey);
|
||||
log.error("Ошибка синхронизации конфигурации тенантов: попытка={}, "
|
||||
+ "следующий повтор не ранее чем через {} мс, errorType={}",
|
||||
consecutiveFailures,
|
||||
retryDelay,
|
||||
failure.getClass().getSimpleName());
|
||||
log.debug("Технические детали синхронизации конфигурации тенантов", failure);
|
||||
}
|
||||
|
||||
private long registerFailure(String failureKey) {
|
||||
if (failureKey.equals(failedRevisionKey) && consecutiveFailures > 0) {
|
||||
consecutiveFailures = Math.min(consecutiveFailures + 1, 31);
|
||||
} else {
|
||||
failedRevisionKey = failureKey;
|
||||
consecutiveFailures = 1;
|
||||
}
|
||||
long delay = retryDelayMillis(consecutiveFailures);
|
||||
retryNotBeforeMillis = safeAdd(currentTimeMillis.getAsLong(), delay);
|
||||
return delay;
|
||||
}
|
||||
|
||||
static long retryDelayMillis(int failureCount) {
|
||||
long delay = RETRY_BASE_DELAY_MILLIS;
|
||||
for (int attempt = 1; attempt < failureCount && delay < RETRY_MAX_DELAY_MILLIS; attempt++) {
|
||||
delay = Math.min(delay * 2, RETRY_MAX_DELAY_MILLIS);
|
||||
}
|
||||
return delay;
|
||||
}
|
||||
|
||||
private long safeAdd(long value, long increment) {
|
||||
return value > Long.MAX_VALUE - increment ? Long.MAX_VALUE : value + increment;
|
||||
}
|
||||
|
||||
private void resetRetryState() {
|
||||
failedRevisionKey = "";
|
||||
consecutiveFailures = 0;
|
||||
retryNotBeforeMillis = 0L;
|
||||
}
|
||||
|
||||
private void clearSnapshotFence() {
|
||||
expectedCommittedSnapshot = null;
|
||||
deferredSnapshots.clear();
|
||||
}
|
||||
|
||||
private record Projection(String hash, List<TenantConfig> tenants, ConfigSnapshot snapshot) {
|
||||
}
|
||||
|
||||
private record ConfigSnapshot(List<NormalizedTenantConfig> tenants) {
|
||||
private ConfigSnapshot {
|
||||
tenants = List.copyOf(tenants);
|
||||
}
|
||||
}
|
||||
|
||||
private record NormalizedTenantConfig(
|
||||
String name,
|
||||
String domain,
|
||||
String url,
|
||||
String username,
|
||||
String password
|
||||
) {
|
||||
private static final Comparator<String> NULL_SAFE_TEXT = Comparator.nullsFirst(String::compareTo);
|
||||
private static final Comparator<NormalizedTenantConfig> ORDER =
|
||||
Comparator.comparing(NormalizedTenantConfig::domain, NULL_SAFE_TEXT)
|
||||
.thenComparing(NormalizedTenantConfig::name, NULL_SAFE_TEXT)
|
||||
.thenComparing(NormalizedTenantConfig::url, NULL_SAFE_TEXT)
|
||||
.thenComparing(NormalizedTenantConfig::username, NULL_SAFE_TEXT)
|
||||
.thenComparing(NormalizedTenantConfig::password, NULL_SAFE_TEXT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.magistr.app.config.tenant.TenantContext;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import com.magistr.app.model.Role;
|
||||
import com.magistr.app.service.TenantLifecycleException;
|
||||
import com.magistr.app.service.TenantLifecycleMutationResult;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -90,8 +91,11 @@ public class DatabaseController {
|
||||
public ResponseEntity<Map<String, Object>> addTenant(@RequestBody TenantConfig config) {
|
||||
try {
|
||||
return tenantLifecycleService.executeSerialized(() -> {
|
||||
TenantConfig saved = tenantLifecycleService.addOrUpdateTenant(config);
|
||||
tenantConfigWatcher.refreshHash();
|
||||
tenantConfigWatcher.prepareForApiMutation();
|
||||
TenantLifecycleMutationResult mutation =
|
||||
tenantLifecycleService.addOrUpdateTenantWithReceipt(config);
|
||||
tenantConfigWatcher.registerSuccessfulMutation(mutation.receipt());
|
||||
TenantConfig saved = mutation.tenant();
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("success", true);
|
||||
result.put("message", "Тенант '" + saved.getDomain() + "' добавлен");
|
||||
@@ -111,8 +115,11 @@ public class DatabaseController {
|
||||
public ResponseEntity<Map<String, Object>> removeTenant(@PathVariable("domain") String domain) {
|
||||
try {
|
||||
return tenantLifecycleService.executeSerialized(() -> {
|
||||
TenantConfig removed = tenantLifecycleService.removeTenant(domain);
|
||||
tenantConfigWatcher.refreshHash();
|
||||
tenantConfigWatcher.prepareForApiMutation();
|
||||
TenantLifecycleMutationResult mutation =
|
||||
tenantLifecycleService.removeTenantWithReceipt(domain);
|
||||
tenantConfigWatcher.registerSuccessfulMutation(mutation.receipt());
|
||||
TenantConfig removed = mutation.tenant();
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("success", true);
|
||||
result.put("message", "Тенант '" + removed.getDomain() + "' удалён");
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.magistr.app.service;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Результат успешной lifecycle-мутации тенанта вместе с квитанцией персистенции.
|
||||
*
|
||||
* <p>Конфигурация копируется при создании и чтении, чтобы вызывающий код не мог изменить
|
||||
* уже активированный runtime-снимок через изменяемый DTO.</p>
|
||||
*/
|
||||
public final class TenantLifecycleMutationResult {
|
||||
|
||||
private final TenantConfig tenant;
|
||||
private final TenantSecretUpdateReceipt receipt;
|
||||
|
||||
public TenantLifecycleMutationResult(TenantConfig tenant, TenantSecretUpdateReceipt receipt) {
|
||||
this.tenant = copy(Objects.requireNonNull(tenant, "Конфигурация тенанта не задана"));
|
||||
this.receipt = Objects.requireNonNull(receipt, "Квитанция изменения tenant Secret не задана");
|
||||
}
|
||||
|
||||
public TenantConfig tenant() {
|
||||
return copy(tenant);
|
||||
}
|
||||
|
||||
public TenantSecretUpdateReceipt receipt() {
|
||||
return receipt;
|
||||
}
|
||||
|
||||
private static TenantConfig copy(TenantConfig source) {
|
||||
return new TenantConfig(
|
||||
source.getName(),
|
||||
source.getDomain(),
|
||||
source.getUrl(),
|
||||
source.getUsername(),
|
||||
source.getPassword()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -21,10 +21,9 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Set;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class TenantLifecycleService {
|
||||
@@ -72,6 +71,13 @@ public class TenantLifecycleService {
|
||||
* Добавляет или обновляет tenant без разрыва действующего подключения.
|
||||
*/
|
||||
public synchronized TenantConfig addOrUpdateTenant(TenantConfig requestedConfig) {
|
||||
return addOrUpdateTenantWithReceipt(requestedConfig).tenant();
|
||||
}
|
||||
|
||||
/**
|
||||
* Добавляет или обновляет tenant и возвращает подтверждённый результат персистенции.
|
||||
*/
|
||||
public synchronized TenantLifecycleMutationResult addOrUpdateTenantWithReceipt(TenantConfig requestedConfig) {
|
||||
TenantConfig config = validateAndNormalize(requestedConfig);
|
||||
Map<String, TenantConfig> oldConfigs = routingDataSource.snapshotTenantConfigs();
|
||||
Map<String, TenantConfig> desiredConfigs = new LinkedHashMap<>(oldConfigs);
|
||||
@@ -96,7 +102,7 @@ public class TenantLifecycleService {
|
||||
publishSuccessfulMutation(receipt, sortedConfigs(desiredConfigs), config);
|
||||
retireDataSource(previous == null ? null : previous.dataSource());
|
||||
log.info("Lifecycle тенанта '{}' успешно применён", config.getDomain());
|
||||
return copyConfig(config);
|
||||
return new TenantLifecycleMutationResult(config, receipt);
|
||||
} finally {
|
||||
if (!activated) {
|
||||
closeCandidateDataSource(candidate);
|
||||
@@ -108,6 +114,13 @@ public class TenantLifecycleService {
|
||||
* Сначала сохраняет desired-конфигурацию, затем локально отключает tenant.
|
||||
*/
|
||||
public synchronized TenantConfig removeTenant(String requestedDomain) {
|
||||
return removeTenantWithReceipt(requestedDomain).tenant();
|
||||
}
|
||||
|
||||
/**
|
||||
* Удаляет tenant и возвращает подтверждённый результат персистенции.
|
||||
*/
|
||||
public synchronized TenantLifecycleMutationResult removeTenantWithReceipt(String requestedDomain) {
|
||||
String domain = validateAndNormalizeDomain(requestedDomain);
|
||||
Map<String, TenantConfig> oldConfigs = routingDataSource.snapshotTenantConfigs();
|
||||
TenantConfig existing = oldConfigs.get(domain);
|
||||
@@ -132,13 +145,14 @@ public class TenantLifecycleService {
|
||||
publishSuccessfulMutation(receipt, sortedConfigs(desiredConfigs), null);
|
||||
retireDataSource(removed.dataSource());
|
||||
log.info("Lifecycle тенанта '{}' удалён", domain);
|
||||
return copyConfig(existing);
|
||||
return new TenantLifecycleMutationResult(existing, receipt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Синхронизирует уже персистированный список из mounted Secret. Метод не пишет
|
||||
* Secret повторно и использует тот же monitor, что ручные API add/update/remove.
|
||||
* Обновление параметров существующего domain остаётся отдельной задачей watcher №13.
|
||||
* Для нового или изменённого домена подключение сначала полностью подготавливается,
|
||||
* проверяется и мигрируется, после чего атомарно заменяет прежний runtime-снимок.
|
||||
*/
|
||||
public synchronized void synchronizeFromPersistedConfig(List<TenantConfig> requestedConfigs) {
|
||||
if (requestedConfigs == null) {
|
||||
@@ -146,47 +160,55 @@ public class TenantLifecycleService {
|
||||
}
|
||||
List<TenantConfig> configs = requestedConfigs.stream()
|
||||
.map(this::validateAndNormalize)
|
||||
.sorted(Comparator.comparing(TenantConfig::getDomain))
|
||||
.toList();
|
||||
Set<String> desiredDomains = configs.stream()
|
||||
.map(TenantConfig::getDomain)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
readinessRegistry.clearConfigurationFailure();
|
||||
readinessRegistry.replaceDesired(configs);
|
||||
if (desiredDomains.size() != configs.size()) {
|
||||
throw new IllegalArgumentException("Конфигурация содержит повторяющиеся домены тенантов");
|
||||
}
|
||||
|
||||
Map<String, TenantConfig> desiredConfigs = new LinkedHashMap<>();
|
||||
for (TenantConfig config : configs) {
|
||||
if (!routingDataSource.hasTenant(config.getDomain())) {
|
||||
log.info("Активируем нового тенанта '{}' из persisted-конфигурации", config.getDomain());
|
||||
activatePersistedConfig(config);
|
||||
if (desiredConfigs.put(config.getDomain(), config) != null) {
|
||||
throw new IllegalArgumentException("Конфигурация содержит повторяющиеся домены тенантов");
|
||||
}
|
||||
}
|
||||
|
||||
for (String existingDomain : new ArrayList<>(routingDataSource.snapshotTenantConfigs().keySet())) {
|
||||
if (!desiredDomains.contains(existingDomain)) {
|
||||
Map<String, TenantConfig> currentConfigs = routingDataSource.snapshotTenantConfigs();
|
||||
readinessRegistry.replaceDesired(configs);
|
||||
|
||||
for (TenantConfig config : configs) {
|
||||
TenantConfig currentConfig = currentConfigs.get(config.getDomain());
|
||||
if (currentConfig == null) {
|
||||
log.info("Активируем нового тенанта '{}' из persisted-конфигурации", config.getDomain());
|
||||
activatePersistedConfig(config);
|
||||
} else if (!sameConfig(currentConfig, config)) {
|
||||
log.info("Заменяем подключение тенанта '{}' из persisted-конфигурации", config.getDomain());
|
||||
activatePersistedConfig(config);
|
||||
} else {
|
||||
recoverPersistedReadinessIfNeeded(config);
|
||||
}
|
||||
}
|
||||
|
||||
for (String existingDomain : new ArrayList<>(currentConfigs.keySet())) {
|
||||
if (!desiredConfigs.containsKey(existingDomain)) {
|
||||
log.info("Отключаем тенанта '{}' — его нет в persisted-конфигурации", existingDomain);
|
||||
TenantRoutingDataSource.TenantState removed =
|
||||
routingDataSource.removeTenantAtomically(existingDomain);
|
||||
retireDataSource(removed == null ? null : removed.dataSource());
|
||||
}
|
||||
}
|
||||
|
||||
readinessRegistry.clearConfigurationFailure();
|
||||
}
|
||||
|
||||
private void activatePersistedConfig(TenantConfig config) {
|
||||
HikariDataSource candidate = null;
|
||||
boolean activated = false;
|
||||
TenantRoutingDataSource.TenantState previous;
|
||||
readinessRegistry.markMigrationStarted(config);
|
||||
try {
|
||||
candidate = prepareCandidate(config);
|
||||
verifyConnection(candidate);
|
||||
migrateCandidate(candidate);
|
||||
TenantRoutingDataSource.TenantState previous = routingDataSource.swapTenant(config, candidate);
|
||||
previous = routingDataSource.swapTenant(config, candidate);
|
||||
activated = true;
|
||||
readinessRegistry.markMigrationSucceeded(config);
|
||||
readinessRegistry.markConnectivity(config.getDomain(), true);
|
||||
retireDataSource(previous == null ? null : previous.dataSource());
|
||||
} catch (RuntimeException activationFailure) {
|
||||
readinessRegistry.markMigrationFailed(config);
|
||||
throw activationFailure;
|
||||
@@ -195,6 +217,29 @@ public class TenantLifecycleService {
|
||||
closeCandidateDataSource(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
// После swap новый pool уже является рабочим. Ошибка публикации readiness не должна
|
||||
// ошибочно помечать успешно мигрированный pool как FAILED: следующий retry восстановит
|
||||
// состояние по активному подключению без повторной замены DataSource.
|
||||
retireDataSource(previous == null ? null : previous.dataSource());
|
||||
readinessRegistry.markMigrationSucceeded(config);
|
||||
readinessRegistry.markConnectivity(config.getDomain(), true);
|
||||
}
|
||||
|
||||
private void recoverPersistedReadinessIfNeeded(TenantConfig config) {
|
||||
TenantReadinessRegistry.TenantStatus status =
|
||||
readinessRegistry.snapshot().get(config.getDomain());
|
||||
if (status == null
|
||||
|| status.migrationState() == TenantReadinessRegistry.MigrationState.SUCCEEDED) {
|
||||
return;
|
||||
}
|
||||
if (!routingDataSource.testConnection(config.getDomain())) {
|
||||
throw new TenantLifecycleException(
|
||||
"Активное подключение тенанта не прошло проверку после предыдущей синхронизации"
|
||||
);
|
||||
}
|
||||
readinessRegistry.markMigrationSucceeded(config);
|
||||
readinessRegistry.markConnectivity(config.getDomain(), true);
|
||||
}
|
||||
|
||||
private HikariDataSource prepareCandidate(TenantConfig config) {
|
||||
@@ -398,9 +443,24 @@ public class TenantLifecycleService {
|
||||
);
|
||||
}
|
||||
|
||||
private boolean sameConfig(TenantConfig first, TenantConfig second) {
|
||||
TenantConfig normalizedFirst = validateAndNormalize(first);
|
||||
TenantConfig normalizedSecond = validateAndNormalize(second);
|
||||
return Objects.equals(normalizedFirst.getName(), normalizedSecond.getName())
|
||||
&& Objects.equals(normalizedFirst.getDomain(), normalizedSecond.getDomain())
|
||||
&& Objects.equals(normalizedFirst.getUrl(), normalizedSecond.getUrl())
|
||||
&& Objects.equals(normalizedFirst.getUsername(), normalizedSecond.getUsername())
|
||||
&& Objects.equals(normalizedFirst.getPassword(), normalizedSecond.getPassword());
|
||||
}
|
||||
|
||||
private void retireDataSource(DataSource dataSource) {
|
||||
if (dataSource != null) {
|
||||
retiredPoolService.retire(dataSource);
|
||||
try {
|
||||
retiredPoolService.retire(dataSource);
|
||||
} catch (RuntimeException retirementFailure) {
|
||||
logTechnicalFailure("Не удалось передать прежний tenant pool на отложенное закрытие",
|
||||
retirementFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -511,7 +511,10 @@ class KubernetesTenantSecretUpdaterTest {
|
||||
files.tokenPath(),
|
||||
files.namespacePath(),
|
||||
files.caPath(),
|
||||
server.url("/").toString(),
|
||||
// MockWebServer на Windows может вернуть canonical loopback hostname
|
||||
// kubernetes.docker.internal. Фиксируем имя из SAN тестового сертификата,
|
||||
// сохраняя реальную проверку hostname в клиенте.
|
||||
"https://localhost:" + server.getPort(),
|
||||
"tenants-secret",
|
||||
objectMapper,
|
||||
maxAttempts,
|
||||
|
||||
@@ -1,33 +1,35 @@
|
||||
package com.magistr.app.config.tenant;
|
||||
|
||||
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
|
||||
import com.magistr.app.service.RetiredTenantPoolService;
|
||||
import com.magistr.app.service.TenantDatabaseMigrationService;
|
||||
import com.magistr.app.service.TenantLifecycleException;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.boot.test.system.CapturedOutput;
|
||||
import org.springframework.boot.test.system.OutputCaptureExtension;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.LongSupplier;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doNothing;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.never;
|
||||
|
||||
@ExtendWith(OutputCaptureExtension.class)
|
||||
class TenantConfigWatcherTest {
|
||||
|
||||
@TempDir
|
||||
@@ -48,29 +50,62 @@ class TenantConfigWatcherTest {
|
||||
.isNotEqualTo(TenantConfigWatcher.configHash("tenant-b"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void prepareForApiMutationSynchronizesUnappliedBaselineOnce() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String content = json("alpha");
|
||||
Files.writeString(configPath, content);
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
|
||||
watcher.prepareForApiMutation();
|
||||
watcher.prepareForApiMutation();
|
||||
|
||||
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash(content));
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastAppliedSnapshot")).isNotNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void prepareForApiMutationPropagatesReadFailure() throws Exception {
|
||||
Path configPath = Files.createDirectory(tempDirectory.resolve("tenants.json"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
|
||||
TenantConfigWatcher watcher = watcher(
|
||||
lifecycleService,
|
||||
readinessRegistry,
|
||||
configPath,
|
||||
new AtomicLong(1_000L)
|
||||
);
|
||||
|
||||
assertThatThrownBy(watcher::prepareForApiMutation)
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось подготовить актуальную конфигурацию тенантов перед изменением");
|
||||
|
||||
verify(readinessRegistry).markConfigurationFailure();
|
||||
verify(lifecycleService, never()).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "configurationUnavailable")).isEqualTo(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void advancesHashOnlyAfterSuccessfulLifecycleSynchronization() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String content = """
|
||||
[{
|
||||
"name": "Тестовый tenant",
|
||||
"domain": "alpha",
|
||||
"url": "jdbc:postgresql://db/alpha",
|
||||
"username": "alpha",
|
||||
"password": "тестовый-пароль"
|
||||
}]
|
||||
""";
|
||||
String content = json("alpha");
|
||||
Files.writeString(configPath, content);
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
doThrow(new IllegalStateException("тестовая ошибка синхронизации"))
|
||||
.doNothing()
|
||||
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
AtomicLong now = new AtomicLong(1_000L);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, now);
|
||||
|
||||
watcher.watchForChanges();
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash")).isEqualTo("");
|
||||
|
||||
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
|
||||
watcher.watchForChanges();
|
||||
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash(content));
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
|
||||
@@ -80,112 +115,441 @@ class TenantConfigWatcherTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void successfulHashPreventsRepeatedSynchronizationOfSameContent() throws Exception {
|
||||
void retriesSameRevisionOnlyAfterBackoffAndResetsStateAfterSuccess() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String content = "[]";
|
||||
Files.writeString(configPath, content);
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
doNothing().when(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
doThrow(new IllegalStateException("первая тестовая ошибка"))
|
||||
.doThrow(new IllegalStateException("вторая тестовая ошибка"))
|
||||
.doNothing()
|
||||
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
AtomicLong now = new AtomicLong(10_000L);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, now);
|
||||
|
||||
watcher.watchForChanges();
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
|
||||
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
|
||||
watcher.watchForChanges();
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
|
||||
|
||||
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS * 2 - 1);
|
||||
watcher.watchForChanges();
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
|
||||
|
||||
now.incrementAndGet();
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(lifecycleService, times(3)).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(0);
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "retryNotBeforeMillis")).isEqualTo(0L);
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash(content));
|
||||
}
|
||||
|
||||
@Test
|
||||
void watcherCannotReadBetweenSerializedApiMutationAndRefreshHash() throws Exception {
|
||||
void retryDelayGrowsExponentiallyAndIsCapped() {
|
||||
assertThat(TenantConfigWatcher.retryDelayMillis(1)).isEqualTo(30_000L);
|
||||
assertThat(TenantConfigWatcher.retryDelayMillis(2)).isEqualTo(60_000L);
|
||||
assertThat(TenantConfigWatcher.retryDelayMillis(3)).isEqualTo(120_000L);
|
||||
assertThat(TenantConfigWatcher.retryDelayMillis(4)).isEqualTo(240_000L);
|
||||
assertThat(TenantConfigWatcher.retryDelayMillis(5)).isEqualTo(300_000L);
|
||||
assertThat(TenantConfigWatcher.retryDelayMillis(31)).isEqualTo(300_000L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void newRevisionBypassesBackoffImmediately() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String content = "[]";
|
||||
Files.writeString(configPath, content);
|
||||
TenantRoutingDataSource routingDataSource = mock(TenantRoutingDataSource.class);
|
||||
TenantLifecycleService lifecycleService = new TenantLifecycleService(
|
||||
routingDataSource,
|
||||
mock(TenantConfigStore.class),
|
||||
mock(TenantDatabaseMigrationService.class),
|
||||
mock(RetiredTenantPoolService.class),
|
||||
mock(TenantReadinessRegistry.class)
|
||||
);
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(
|
||||
Files.writeString(configPath, "[]");
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
doThrow(new IllegalStateException("тестовая ошибка"))
|
||||
.doNothing()
|
||||
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, new AtomicLong(1_000L));
|
||||
|
||||
watcher.watchForChanges();
|
||||
Files.writeString(configPath, "[ ]");
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash("[ ]"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void restoredLastAppliedRevisionIsReconciledAfterFailedChange() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String firstRevision = "[]";
|
||||
String failedRevision = "[ ]";
|
||||
Files.writeString(configPath, firstRevision);
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
doNothing()
|
||||
.doThrow(new IllegalStateException("тестовая ошибка новой ревизии"))
|
||||
.doNothing()
|
||||
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, new AtomicLong(1_000L));
|
||||
|
||||
watcher.watchForChanges();
|
||||
Files.writeString(configPath, failedRevision);
|
||||
watcher.watchForChanges();
|
||||
Files.writeString(configPath, firstRevision);
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(lifecycleService, times(3)).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash(firstRevision));
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "configurationUnavailable")).isEqualTo(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void semanticFenceDefersStaleProjectionWithDifferentFormatting() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String baseline = json("alpha");
|
||||
Files.writeString(configPath, baseline);
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
watcher.prepareForApiMutation();
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha"),
|
||||
tenants("alpha", "beta")
|
||||
));
|
||||
|
||||
Files.writeString(configPath, prettyJson("alpha"));
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash(baseline));
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNotNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void prepareForNextMutationDoesNotApplyKnownStaleProjection() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
Files.writeString(configPath, json("alpha"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
watcher.prepareForApiMutation();
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha"),
|
||||
tenants("alpha", "beta")
|
||||
));
|
||||
|
||||
watcher.prepareForApiMutation();
|
||||
|
||||
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonPersistedReceiptDoesNotArmFence() {
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(
|
||||
lifecycleService,
|
||||
mock(TenantReadinessRegistry.class)
|
||||
tempDirectory.resolve("tenants.json")
|
||||
);
|
||||
ReflectionTestUtils.setField(watcher, "tenantsConfigPath", configPath.toString());
|
||||
CountDownLatch apiMutationReachedRefreshWindow = new CountDownLatch(1);
|
||||
CountDownLatch allowApiRefresh = new CountDownLatch(1);
|
||||
AtomicReference<Thread> watcherThread = new AtomicReference<>();
|
||||
ExecutorService executor = Executors.newFixedThreadPool(2);
|
||||
|
||||
try {
|
||||
Future<?> api = executor.submit(() -> lifecycleService.executeSerialized(() -> {
|
||||
apiMutationReachedRefreshWindow.countDown();
|
||||
await(allowApiRefresh, "Ожидание разрешения refreshHash прервано");
|
||||
watcher.refreshHash();
|
||||
}));
|
||||
assertThat(apiMutationReachedRefreshWindow.await(5, TimeUnit.SECONDS)).isTrue();
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
false,
|
||||
true,
|
||||
tenants("alpha"),
|
||||
tenants("alpha", "beta")
|
||||
));
|
||||
|
||||
Future<?> concurrentWatcher = executor.submit(() -> {
|
||||
watcherThread.set(Thread.currentThread());
|
||||
watcher.watchForChanges();
|
||||
});
|
||||
awaitBlocked(watcherThread);
|
||||
assertThat(concurrentWatcher).isNotDone();
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
|
||||
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
|
||||
}
|
||||
|
||||
allowApiRefresh.countDown();
|
||||
api.get(5, TimeUnit.SECONDS);
|
||||
concurrentWatcher.get(5, TimeUnit.SECONDS);
|
||||
@Test
|
||||
void persistedNoOpDoesNotSuppressMergedProjection() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
Files.writeString(configPath, json("alpha"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
watcher.prepareForApiMutation();
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
false,
|
||||
tenants("alpha"),
|
||||
tenants("alpha")
|
||||
));
|
||||
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
|
||||
.isEqualTo(TenantConfigWatcher.configHash(content));
|
||||
verify(routingDataSource, never()).snapshotTenantConfigs();
|
||||
} finally {
|
||||
allowApiRefresh.countDown();
|
||||
executor.shutdownNow();
|
||||
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
Files.writeString(configPath, json("alpha", "gamma"));
|
||||
watcher.watchForChanges();
|
||||
|
||||
ArgumentCaptor<List<TenantConfig>> captor = tenantListCaptor();
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(captor.capture());
|
||||
assertThat(captor.getAllValues().get(1))
|
||||
.extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "gamma");
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoRapidMutationsDeferIntermediateProjectionUntilLatestCommit() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
Files.writeString(configPath, json("alpha"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
watcher.prepareForApiMutation();
|
||||
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha"),
|
||||
tenants("alpha", "beta")
|
||||
));
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha", "beta"),
|
||||
tenants("alpha", "beta", "gamma")
|
||||
));
|
||||
|
||||
Files.writeString(configPath, json("alpha", "beta"));
|
||||
watcher.watchForChanges();
|
||||
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
|
||||
Files.writeString(configPath, json("alpha", "beta", "gamma"));
|
||||
watcher.watchForChanges();
|
||||
|
||||
ArgumentCaptor<List<TenantConfig>> captor = tenantListCaptor();
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(captor.capture());
|
||||
assertThat(captor.getAllValues().get(1))
|
||||
.extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "beta", "gamma");
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
|
||||
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void cyclicRapidMutationsClearFenceWhenLatestCommitIsAlreadyApplied() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
Files.writeString(configPath, json("alpha"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
watcher.prepareForApiMutation();
|
||||
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha"),
|
||||
tenants("alpha", "beta")
|
||||
));
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha", "beta"),
|
||||
tenants("alpha")
|
||||
));
|
||||
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
|
||||
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownConcurrentSnapshotSupersedesExpectedAndIsApplied() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
Files.writeString(configPath, json("alpha"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
|
||||
watcher.prepareForApiMutation();
|
||||
watcher.registerSuccessfulMutation(receipt(
|
||||
true,
|
||||
true,
|
||||
tenants("alpha"),
|
||||
tenants("alpha", "beta")
|
||||
));
|
||||
|
||||
Files.writeString(configPath, json("alpha", "gamma"));
|
||||
watcher.watchForChanges();
|
||||
|
||||
ArgumentCaptor<List<TenantConfig>> captor = tenantListCaptor();
|
||||
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(captor.capture());
|
||||
assertThat(captor.getAllValues().get(1))
|
||||
.extracting(TenantConfig::getDomain)
|
||||
.containsExactly("alpha", "gamma");
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
|
||||
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void requiredMissingFileUsesBackoff() {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
|
||||
AtomicLong now = new AtomicLong(1_000L);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, readinessRegistry, configPath, now);
|
||||
ReflectionTestUtils.setField(watcher, "tenantsConfigRequired", true);
|
||||
|
||||
watcher.watchForChanges();
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(readinessRegistry).markConfigurationFailure();
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(1);
|
||||
|
||||
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(readinessRegistry, times(2)).markConfigurationFailure();
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void readFailureUsesBackoff() throws Exception {
|
||||
Path configPath = Files.createDirectory(tempDirectory.resolve("tenants.json"));
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
|
||||
AtomicLong now = new AtomicLong(1_000L);
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, readinessRegistry, configPath, now);
|
||||
|
||||
watcher.watchForChanges();
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(readinessRegistry).markConfigurationFailure();
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(1);
|
||||
|
||||
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(readinessRegistry, times(2)).markConfigurationFailure();
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void requiredEmptyConfigMarksFailureAndKeepsHashUnapplied() throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
Files.writeString(configPath, "[]");
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
|
||||
TenantConfigWatcher watcher = watcher(
|
||||
lifecycleService,
|
||||
readinessRegistry,
|
||||
configPath,
|
||||
new AtomicLong(1_000L)
|
||||
);
|
||||
ReflectionTestUtils.setField(watcher, "tenantsConfigRequired", true);
|
||||
|
||||
watcher.watchForChanges();
|
||||
|
||||
verify(readinessRegistry).markConfigurationFailure();
|
||||
verify(lifecycleService, never()).synchronizeFromPersistedConfig(anyList());
|
||||
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash")).isEqualTo("");
|
||||
}
|
||||
|
||||
@Test
|
||||
void failureLogsRetryMetadataWithoutSecretsOrFingerprints(CapturedOutput output) throws Exception {
|
||||
Path configPath = tempDirectory.resolve("tenants.json");
|
||||
String content = """
|
||||
[{"domain":"alpha","url":"jdbc:postgresql://secret-host/db",
|
||||
"username":"secret-user","password":"secret-password"}]
|
||||
""";
|
||||
Files.writeString(configPath, content);
|
||||
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
|
||||
doThrow(new IllegalStateException("секретные технические детали"))
|
||||
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
|
||||
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, new AtomicLong(1_000L));
|
||||
|
||||
watcher.watchForChanges();
|
||||
|
||||
assertThat(output)
|
||||
.contains("Ошибка синхронизации конфигурации тенантов")
|
||||
.contains("попытка=1")
|
||||
.contains("через 30000 мс")
|
||||
.doesNotContain("secret-host")
|
||||
.doesNotContain("secret-user")
|
||||
.doesNotContain("secret-password")
|
||||
.doesNotContain(TenantConfigWatcher.configHash(content))
|
||||
.doesNotContain("revision=")
|
||||
.doesNotContain("fingerprint")
|
||||
.doesNotContain("хеш:");
|
||||
}
|
||||
|
||||
private TenantConfigWatcher watcher(TenantLifecycleService lifecycleService, Path configPath) {
|
||||
return watcher(lifecycleService, configPath, new AtomicLong(1_000L));
|
||||
}
|
||||
|
||||
private TenantConfigWatcher watcher(TenantLifecycleService lifecycleService,
|
||||
Path configPath,
|
||||
AtomicLong now) {
|
||||
return watcher(lifecycleService, mock(TenantReadinessRegistry.class), configPath, now);
|
||||
}
|
||||
|
||||
private TenantConfigWatcher watcher(TenantLifecycleService lifecycleService,
|
||||
TenantReadinessRegistry readinessRegistry,
|
||||
Path configPath,
|
||||
AtomicLong now) {
|
||||
doAnswer(invocation -> {
|
||||
invocation.<Runnable>getArgument(0).run();
|
||||
return null;
|
||||
}).when(lifecycleService).executeSerialized(any(Runnable.class));
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(
|
||||
lifecycleService,
|
||||
mock(TenantReadinessRegistry.class)
|
||||
);
|
||||
TenantConfigWatcher watcher = new TenantConfigWatcher(lifecycleService, readinessRegistry);
|
||||
ReflectionTestUtils.setField(watcher, "tenantsConfigPath", configPath.toString());
|
||||
ReflectionTestUtils.setField(watcher, "currentTimeMillis", (LongSupplier) now::get);
|
||||
return watcher;
|
||||
}
|
||||
|
||||
private void await(CountDownLatch latch, String message) {
|
||||
try {
|
||||
if (!latch.await(5, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException(message);
|
||||
}
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException(message, exception);
|
||||
}
|
||||
private TenantSecretUpdateReceipt receipt(boolean persisted,
|
||||
boolean changed,
|
||||
List<TenantConfig> previous,
|
||||
List<TenantConfig> committed) {
|
||||
return new TenantSecretUpdateReceipt(
|
||||
persisted,
|
||||
changed,
|
||||
persisted ? "resource-version" : null,
|
||||
previous,
|
||||
committed
|
||||
);
|
||||
}
|
||||
|
||||
private void awaitBlocked(AtomicReference<Thread> threadReference) {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
|
||||
while (System.nanoTime() < deadline) {
|
||||
Thread thread = threadReference.get();
|
||||
if (thread != null && thread.getState() == Thread.State.BLOCKED) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(10);
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("Ожидание блокировки watcher прервано", exception);
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("Watcher не заблокировался на общем lifecycle coordinator");
|
||||
private List<TenantConfig> tenants(String... domains) {
|
||||
return java.util.Arrays.stream(domains).map(this::tenant).toList();
|
||||
}
|
||||
|
||||
private TenantConfig tenant(String domain) {
|
||||
return new TenantConfig(
|
||||
domain,
|
||||
domain,
|
||||
"jdbc:postgresql://db/" + domain,
|
||||
domain,
|
||||
"password-" + domain
|
||||
);
|
||||
}
|
||||
|
||||
private String json(String... domains) {
|
||||
return "[" + java.util.Arrays.stream(domains)
|
||||
.map(domain -> "{\"name\":\"" + domain
|
||||
+ "\",\"domain\":\"" + domain
|
||||
+ "\",\"url\":\"jdbc:postgresql://db/" + domain
|
||||
+ "\",\"username\":\"" + domain
|
||||
+ "\",\"password\":\"password-" + domain + "\"}")
|
||||
.collect(java.util.stream.Collectors.joining(",")) + "]";
|
||||
}
|
||||
|
||||
private String prettyJson(String domain) {
|
||||
return """
|
||||
[
|
||||
{
|
||||
"password": "password-%1$s",
|
||||
"username": "%1$s",
|
||||
"url": "jdbc:postgresql://db/%1$s",
|
||||
"domain": "%1$s",
|
||||
"name": "%1$s"
|
||||
}
|
||||
]
|
||||
""".formatted(domain);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
private ArgumentCaptor<List<TenantConfig>> tenantListCaptor() {
|
||||
return (ArgumentCaptor) ArgumentCaptor.forClass(List.class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,32 @@
|
||||
package com.magistr.app.controller;
|
||||
|
||||
import com.magistr.app.config.tenant.TenantConfigWatcher;
|
||||
import com.magistr.app.config.tenant.TenantConfig;
|
||||
import com.magistr.app.config.tenant.TenantRoutingDataSource;
|
||||
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
|
||||
import com.magistr.app.service.TenantLifecycleException;
|
||||
import com.magistr.app.service.TenantLifecycleMutationResult;
|
||||
import com.magistr.app.service.TenantLifecycleService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.InOrder;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
@@ -68,7 +77,128 @@ class DatabaseControllerTest {
|
||||
.andExpect(jsonPath("$.message")
|
||||
.value(org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("JDBC"))));
|
||||
|
||||
verify(tenantConfigWatcher, never()).refreshHash();
|
||||
verify(tenantConfigWatcher, never()).prepareForApiMutation();
|
||||
verify(tenantConfigWatcher, never()).registerSuccessfulMutation(any());
|
||||
verify(lifecycleService).executeSerialized(any(Supplier.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void successfulAddPreparesProjectionAndRegistersPersistenceReceipt() throws Exception {
|
||||
executeSerializedActionsImmediately();
|
||||
TenantConfig saved = new TenantConfig(
|
||||
"Тестовый университет",
|
||||
"test",
|
||||
"jdbc:postgresql://db/test",
|
||||
"user",
|
||||
"password"
|
||||
);
|
||||
TenantSecretUpdateReceipt receipt = receipt(saved);
|
||||
when(lifecycleService.addOrUpdateTenantWithReceipt(any(TenantConfig.class)))
|
||||
.thenReturn(new TenantLifecycleMutationResult(saved, receipt));
|
||||
|
||||
mockMvc.perform(post("/api/database/tenants")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"name": "Тестовый университет",
|
||||
"domain": "test",
|
||||
"url": "jdbc:postgresql://db/test",
|
||||
"username": "user",
|
||||
"password": "password"
|
||||
}
|
||||
"""))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true));
|
||||
|
||||
InOrder order = inOrder(lifecycleService, tenantConfigWatcher);
|
||||
order.verify(lifecycleService).executeSerialized(any(Supplier.class));
|
||||
order.verify(tenantConfigWatcher).prepareForApiMutation();
|
||||
order.verify(lifecycleService).addOrUpdateTenantWithReceipt(any(TenantConfig.class));
|
||||
order.verify(tenantConfigWatcher).registerSuccessfulMutation(receipt);
|
||||
}
|
||||
|
||||
@Test
|
||||
void successfulRemovalPreparesProjectionAndRegistersPersistenceReceipt() throws Exception {
|
||||
executeSerializedActionsImmediately();
|
||||
TenantConfig removed = new TenantConfig(
|
||||
"Тестовый университет",
|
||||
"test",
|
||||
"jdbc:postgresql://db/test",
|
||||
"user",
|
||||
"password"
|
||||
);
|
||||
TenantSecretUpdateReceipt receipt = receipt();
|
||||
when(lifecycleService.removeTenantWithReceipt("test"))
|
||||
.thenReturn(new TenantLifecycleMutationResult(removed, receipt));
|
||||
|
||||
mockMvc.perform(delete("/api/database/tenants/test"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.success").value(true));
|
||||
|
||||
InOrder order = inOrder(lifecycleService, tenantConfigWatcher);
|
||||
order.verify(lifecycleService).executeSerialized(any(Supplier.class));
|
||||
order.verify(tenantConfigWatcher).prepareForApiMutation();
|
||||
order.verify(lifecycleService).removeTenantWithReceipt("test");
|
||||
order.verify(tenantConfigWatcher).registerSuccessfulMutation(receipt);
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectionReadFailurePreventsAddAndReturnsServiceUnavailable() throws Exception {
|
||||
executeSerializedActionsImmediately();
|
||||
doThrow(new TenantLifecycleException(
|
||||
"Не удалось прочитать конфигурацию тенантов перед изменением",
|
||||
new IllegalStateException("тестовая ошибка чтения")
|
||||
)).when(tenantConfigWatcher).prepareForApiMutation();
|
||||
|
||||
mockMvc.perform(post("/api/database/tenants")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("""
|
||||
{
|
||||
"name": "Тестовый университет",
|
||||
"domain": "test",
|
||||
"url": "jdbc:postgresql://db/test",
|
||||
"username": "user",
|
||||
"password": "password"
|
||||
}
|
||||
"""))
|
||||
.andExpect(status().isServiceUnavailable())
|
||||
.andExpect(jsonPath("$.success").value(false))
|
||||
.andExpect(jsonPath("$.message")
|
||||
.value("Не удалось прочитать конфигурацию тенантов перед изменением"));
|
||||
|
||||
verify(lifecycleService, never()).addOrUpdateTenantWithReceipt(any());
|
||||
verify(tenantConfigWatcher, never()).registerSuccessfulMutation(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectionReadFailurePreventsRemovalAndReturnsServiceUnavailable() throws Exception {
|
||||
executeSerializedActionsImmediately();
|
||||
doThrow(new TenantLifecycleException(
|
||||
"Не удалось прочитать конфигурацию тенантов перед изменением",
|
||||
new IllegalStateException("тестовая ошибка чтения")
|
||||
)).when(tenantConfigWatcher).prepareForApiMutation();
|
||||
|
||||
mockMvc.perform(delete("/api/database/tenants/test"))
|
||||
.andExpect(status().isServiceUnavailable())
|
||||
.andExpect(jsonPath("$.success").value(false));
|
||||
|
||||
verify(lifecycleService, never()).removeTenantWithReceipt(any());
|
||||
verify(tenantConfigWatcher, never()).registerSuccessfulMutation(any());
|
||||
}
|
||||
|
||||
private void executeSerializedActionsImmediately() {
|
||||
doAnswer(invocation -> invocation.<Supplier<?>>getArgument(0).get())
|
||||
.when(lifecycleService)
|
||||
.executeSerialized(org.mockito.ArgumentMatchers.<Supplier<Object>>any());
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt receipt(TenantConfig... tenants) {
|
||||
return new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
true,
|
||||
"resource-version",
|
||||
List.of(),
|
||||
List.of(tenants)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,6 +178,52 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
assertThat(currentDatabaseThroughRouter()).isEqualTo(VALID_DATABASE);
|
||||
}
|
||||
|
||||
@Test
|
||||
void persistedConfigReplacementKeepsOldPoolOnFailureAndSwapsWithoutSecretWrite() throws Exception {
|
||||
TenantConfig wrongCredentials = config(
|
||||
"Неверные credentials из проекции",
|
||||
jdbcUrl(OLD_DATABASE),
|
||||
POSTGRES.getPassword() + "-wrong"
|
||||
);
|
||||
|
||||
assertThatThrownBy(() -> lifecycleService.synchronizeFromPersistedConfig(List.of(wrongCredentials)))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось подключиться к базе данных тенанта");
|
||||
assertOldState();
|
||||
|
||||
TenantConfig badChecksum = config(
|
||||
"Повреждённый Flyway из проекции",
|
||||
jdbcUrl(BAD_CHECKSUM_DATABASE),
|
||||
POSTGRES.getPassword()
|
||||
);
|
||||
assertThatThrownBy(() -> lifecycleService.synchronizeFromPersistedConfig(List.of(badChecksum)))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
|
||||
assertOldState();
|
||||
|
||||
TenantConfig validCandidate = config(
|
||||
"Новый tenant из проекции",
|
||||
jdbcUrl(VALID_DATABASE),
|
||||
POSTGRES.getPassword()
|
||||
);
|
||||
try (Connection inFlightOldConnection = oldPool.getConnection()) {
|
||||
lifecycleService.synchronizeFromPersistedConfig(List.of(validCandidate));
|
||||
|
||||
assertThat(oldPool.isClosed()).isFalse();
|
||||
try (Statement statement = inFlightOldConnection.createStatement();
|
||||
ResultSet result = statement.executeQuery("SELECT current_database()")) {
|
||||
assertThat(result.next()).isTrue();
|
||||
assertThat(result.getString(1)).isEqualTo(OLD_DATABASE);
|
||||
}
|
||||
}
|
||||
|
||||
awaitPoolClosed(oldPool);
|
||||
assertThat(currentDatabaseThroughRouter()).isEqualTo(VALID_DATABASE);
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
verify(tenantConfigStore, never()).removeTenant(any());
|
||||
verify(tenantConfigStore, never()).compensate(any());
|
||||
}
|
||||
|
||||
private void awaitPoolClosed(HikariDataSource pool) {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3);
|
||||
while (!pool.isClosed() && System.nanoTime() < deadline) {
|
||||
@@ -218,6 +264,7 @@ class TenantLifecyclePostgreSqlIntegrationTest {
|
||||
private void createDatabase(String databaseName) throws Exception {
|
||||
try (Connection connection = POSTGRES.createConnection("");
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)");
|
||||
statement.execute("CREATE DATABASE " + databaseName);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,9 @@ import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mock;
|
||||
@@ -20,17 +23,21 @@ import org.mockito.quality.Strictness;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -128,6 +135,25 @@ class TenantLifecycleServiceTest {
|
||||
verify(candidate, never()).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsConfirmedReceiptWithSuccessfulUpsert() {
|
||||
TenantConfig requested = newAlpha();
|
||||
TenantSecretUpdateReceipt receipt = new TenantSecretUpdateReceipt(
|
||||
true,
|
||||
false,
|
||||
"resource-version-noop",
|
||||
List.of(requested),
|
||||
List.of(requested)
|
||||
);
|
||||
when(tenantConfigStore.upsertTenant(any())).thenReturn(receipt);
|
||||
|
||||
TenantLifecycleMutationResult result = service.addOrUpdateTenantWithReceipt(requested);
|
||||
|
||||
assertThat(result.receipt()).isSameAs(receipt);
|
||||
assertConfig(result.tenant(),
|
||||
"Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
}
|
||||
|
||||
@Test
|
||||
void addsNewTenantWithoutRetiringExistingPool() {
|
||||
Map<String, TenantConfig> createSnapshot = new LinkedHashMap<>();
|
||||
@@ -296,6 +322,26 @@ class TenantLifecycleServiceTest {
|
||||
order.verify(retiredPoolService).retire(oldDataSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsLocalOnlyReceiptWithSuccessfulRemoval() {
|
||||
TenantSecretUpdateReceipt receipt = new TenantSecretUpdateReceipt(
|
||||
false,
|
||||
false,
|
||||
null,
|
||||
List.of(oldAlpha),
|
||||
List.of(beta)
|
||||
);
|
||||
when(tenantConfigStore.removeTenant("alpha")).thenReturn(receipt);
|
||||
when(routingDataSource.removeTenantAtomically("alpha"))
|
||||
.thenReturn(new TenantRoutingDataSource.TenantState("alpha", oldAlpha, oldDataSource));
|
||||
|
||||
TenantLifecycleMutationResult result = service.removeTenantWithReceipt("alpha");
|
||||
|
||||
assertThat(result.receipt()).isSameAs(receipt);
|
||||
assertConfig(result.tenant(),
|
||||
"Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "old-password");
|
||||
}
|
||||
|
||||
@Test
|
||||
void removalFailureCompensatesCommittedReceiptAndKeepsPoolOpen() {
|
||||
TenantSecretUpdateReceipt receipt = removeReceipt("alpha");
|
||||
@@ -314,22 +360,156 @@ class TenantLifecycleServiceTest {
|
||||
@Test
|
||||
void persistedTenantBecomesReadyOnlyAfterMigrationAndSwap() {
|
||||
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "user", "password");
|
||||
when(routingDataSource.hasTenant("gamma")).thenReturn(false);
|
||||
when(routingDataSource.swapTenant(any(TenantConfig.class), any())).thenReturn(null);
|
||||
|
||||
service.synchronizeFromPersistedConfig(List.of(gamma));
|
||||
|
||||
InOrder order = inOrder(readinessRegistry, migrationService, routingDataSource);
|
||||
order.verify(readinessRegistry).clearConfigurationFailure();
|
||||
order.verify(readinessRegistry).replaceDesired(any());
|
||||
order.verify(readinessRegistry).markMigrationStarted(any(TenantConfig.class));
|
||||
order.verify(migrationService).migrate(candidate);
|
||||
order.verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
|
||||
order.verify(readinessRegistry).markMigrationSucceeded(any(TenantConfig.class));
|
||||
order.verify(readinessRegistry).markConnectivity("gamma", true);
|
||||
order.verify(readinessRegistry).clearConfigurationFailure();
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("changedPersistedAlphaConfigs")
|
||||
void replacesExistingPersistedTenantWhenAnyNormalizedFieldChanges(TenantConfig changedConfig) {
|
||||
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of("alpha", oldAlpha));
|
||||
|
||||
service.synchronizeFromPersistedConfig(List.of(changedConfig));
|
||||
|
||||
ArgumentCaptor<TenantConfig> preparedConfig = ArgumentCaptor.forClass(TenantConfig.class);
|
||||
verify(routingDataSource).prepareTenantDataSource(preparedConfig.capture());
|
||||
verify(migrationService).migrate(candidate);
|
||||
verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
|
||||
verify(retiredPoolService).retire(oldDataSource);
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
verify(tenantConfigStore, never()).removeTenant(any());
|
||||
assertConfig(
|
||||
preparedConfig.getValue(),
|
||||
changedConfig.getName().trim(),
|
||||
"alpha",
|
||||
changedConfig.getUrl().trim(),
|
||||
changedConfig.getUsername(),
|
||||
changedConfig.getPassword()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizedEquivalentPersistedTenantDoesNotRecreatePool() {
|
||||
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of("alpha", oldAlpha));
|
||||
TenantConfig equivalent = config(
|
||||
" Старый Alpha ",
|
||||
" ALPHA ",
|
||||
" jdbc:old-alpha ",
|
||||
"old-user",
|
||||
"old-password"
|
||||
);
|
||||
|
||||
service.synchronizeFromPersistedConfig(List.of(equivalent));
|
||||
|
||||
verify(routingDataSource, never()).prepareTenantDataSource(any());
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(routingDataSource, never()).removeTenantAtomically(any());
|
||||
verify(migrationService, never()).migrate(any());
|
||||
verify(retiredPoolService, never()).retire(any());
|
||||
verify(readinessRegistry).clearConfigurationFailure();
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedPersistedReplacementKeepsOldPoolAndClosesCandidate() {
|
||||
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of("alpha", oldAlpha));
|
||||
org.mockito.Mockito.doThrow(new IllegalStateException("тестовая ошибка Flyway"))
|
||||
.when(migrationService).migrate(candidate);
|
||||
|
||||
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(newAlpha())))
|
||||
.isInstanceOf(TenantLifecycleException.class)
|
||||
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
|
||||
|
||||
verify(routingDataSource, never()).swapTenant(any(), any());
|
||||
verify(retiredPoolService, never()).retire(oldDataSource);
|
||||
verify(candidate).close();
|
||||
verify(readinessRegistry).markMigrationFailed(any(TenantConfig.class));
|
||||
verify(readinessRegistry, never()).clearConfigurationFailure();
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void retryRestoresReadinessAfterPostSwapPublicationFailureWithoutSecondSwap() {
|
||||
TenantConfig changed = newAlpha();
|
||||
Map<String, TenantConfig> activatedSnapshot = new LinkedHashMap<>();
|
||||
activatedSnapshot.put("alpha", changed);
|
||||
activatedSnapshot.put("beta", beta);
|
||||
when(routingDataSource.snapshotTenantConfigs())
|
||||
.thenReturn(oldSnapshot, activatedSnapshot);
|
||||
TenantReadinessRegistry.TenantStatus interruptedStatus = new TenantReadinessRegistry.TenantStatus(
|
||||
"alpha",
|
||||
"тестовый-fingerprint",
|
||||
TenantReadinessRegistry.MigrationState.IN_PROGRESS,
|
||||
TenantReadinessRegistry.ConnectivityState.UNKNOWN,
|
||||
null,
|
||||
false,
|
||||
Instant.EPOCH
|
||||
);
|
||||
when(readinessRegistry.snapshot()).thenReturn(Map.of("alpha", interruptedStatus));
|
||||
when(routingDataSource.testConnection("alpha")).thenReturn(true);
|
||||
doThrow(new IllegalStateException("тестовый отказ публикации readiness"))
|
||||
.doNothing()
|
||||
.when(readinessRegistry).markMigrationSucceeded(any(TenantConfig.class));
|
||||
|
||||
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(changed, beta)))
|
||||
.isInstanceOf(IllegalStateException.class);
|
||||
|
||||
service.synchronizeFromPersistedConfig(List.of(changed, beta));
|
||||
|
||||
verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
|
||||
verify(migrationService).migrate(candidate);
|
||||
verify(retiredPoolService).retire(oldDataSource);
|
||||
verify(readinessRegistry, never()).markMigrationFailed(any(TenantConfig.class));
|
||||
verify(routingDataSource).testConnection("alpha");
|
||||
verify(readinessRegistry).markConnectivity("alpha", true);
|
||||
verify(readinessRegistry).clearConfigurationFailure();
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesMergedPersistedSnapshotWithoutWritingSecret() {
|
||||
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "gamma-user", "gamma-password");
|
||||
TenantConfig newBeta = config("Новый Beta", "beta", "jdbc:new-beta", "new-beta-user", "new-beta-password");
|
||||
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of(
|
||||
"alpha", oldAlpha,
|
||||
"gamma", gamma
|
||||
));
|
||||
when(routingDataSource.removeTenantAtomically("gamma"))
|
||||
.thenReturn(new TenantRoutingDataSource.TenantState("gamma", gamma, oldDataSource));
|
||||
|
||||
service.synchronizeFromPersistedConfig(List.of(newAlpha(), newBeta));
|
||||
|
||||
verify(routingDataSource, times(2)).prepareTenantDataSource(any(TenantConfig.class));
|
||||
verify(routingDataSource, times(2)).swapTenant(any(TenantConfig.class), any());
|
||||
verify(routingDataSource).removeTenantAtomically("gamma");
|
||||
verify(tenantConfigStore, never()).upsertTenant(any());
|
||||
verify(tenantConfigStore, never()).removeTenant(any());
|
||||
verify(tenantConfigStore, never()).compensate(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsDuplicateNormalizedDomainsBeforeRuntimeOrReadinessMutation() {
|
||||
TenantConfig first = config("Alpha 1", "alpha", "jdbc:alpha-1", "user", "password");
|
||||
TenantConfig second = config("Alpha 2", " ALPHA ", "jdbc:alpha-2", "user", "password");
|
||||
|
||||
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(first, second)))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessage("Конфигурация содержит повторяющиеся домены тенантов");
|
||||
|
||||
verify(routingDataSource, never()).snapshotTenantConfigs();
|
||||
verify(routingDataSource, never()).prepareTenantDataSource(any());
|
||||
verify(readinessRegistry, never()).replaceDesired(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedPersistedMigrationLeavesTenantNotReadyForWatcherRetry() {
|
||||
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "user", "password");
|
||||
@@ -376,6 +556,23 @@ class TenantLifecycleServiceTest {
|
||||
return config("Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
|
||||
}
|
||||
|
||||
private static Stream<Arguments> changedPersistedAlphaConfigs() {
|
||||
return Stream.of(
|
||||
Arguments.of(new TenantConfig(
|
||||
"Новое имя", "alpha", "jdbc:old-alpha", "old-user", "old-password"
|
||||
)),
|
||||
Arguments.of(new TenantConfig(
|
||||
"Старый Alpha", "alpha", "jdbc:new-alpha", "old-user", "old-password"
|
||||
)),
|
||||
Arguments.of(new TenantConfig(
|
||||
"Старый Alpha", "alpha", "jdbc:old-alpha", "new-user", "old-password"
|
||||
)),
|
||||
Arguments.of(new TenantConfig(
|
||||
"Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "new-password"
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
private TenantSecretUpdateReceipt upsertReceipt(TenantConfig tenant) {
|
||||
Map<String, TenantConfig> committed = new LinkedHashMap<>(oldSnapshot);
|
||||
committed.put(tenant.getDomain(), tenant);
|
||||
|
||||
Reference in New Issue
Block a user