Задача Егора 2ч.

This commit is contained in:
2026-07-17 01:27:02 +03:00
parent 3f685d3e22
commit 3d798c13e3
20 changed files with 2003 additions and 275 deletions

View File

@@ -3,6 +3,7 @@ package com.magistr.app.config.tenant;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
import com.magistr.app.service.TenantLifecycleException;
import com.magistr.app.service.TenantLifecycleService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -10,24 +11,35 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HexFormat;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
import java.util.function.LongSupplier;
/**
* Периодически перечитывает tenants.json (mounted Secret).
* Если Secret был обновлён через K8s API, этот компонент
* подхватит изменения и синхронизирует in-memory datasource'ы.
*
* Также отвечает за инициализацию БД (init.sql) для новых тенантов.
* Периодически применяет конфигурацию тенантов из смонтированного tenants.json.
* Все чтения и lifecycle-изменения сериализуются одним monitor'ом сервиса.
*/
@Component
public class TenantConfigWatcher {
private static final Logger log = LoggerFactory.getLogger(TenantConfigWatcher.class);
private static final String MISSING_FAILURE_KEY = "missing";
private static final String READ_FAILURE_KEY = "read";
static final long RETRY_BASE_DELAY_MILLIS = 30_000L;
static final long RETRY_MAX_DELAY_MILLIS = 300_000L;
private final TenantLifecycleService tenantLifecycleService;
private final TenantReadinessRegistry readinessRegistry;
@@ -39,9 +51,23 @@ public class TenantConfigWatcher {
@Value("${app.tenants.config-required:false}")
private boolean tenantsConfigRequired;
// Хеш последнего прочитанного конфига — чтобы не перезагружать зря
/** Хеш означает только полностью применённое содержимое файла. */
private String lastConfigHash = "";
private ConfigSnapshot lastAppliedSnapshot;
private String failedRevisionKey = "";
private int consecutiveFailures;
private long retryNotBeforeMillis;
/**
* Fence хранит семантические снимки, а не байтовые представления JSON.
* Поэтому форматирование файла не может ошибочно снять или установить fence.
*/
private ConfigSnapshot expectedCommittedSnapshot;
private final Set<ConfigSnapshot> deferredSnapshots = new LinkedHashSet<>();
private boolean configurationUnavailable;
private LongSupplier currentTimeMillis = System::currentTimeMillis;
public TenantConfigWatcher(TenantLifecycleService tenantLifecycleService,
TenantReadinessRegistry readinessRegistry) {
@@ -49,73 +75,135 @@ public class TenantConfigWatcher {
this.readinessRegistry = readinessRegistry;
}
/**
* Каждые 30 секунд проверяет, изменился ли tenants.json.
*/
@Scheduled(fixedDelay = 30_000, initialDelay = 30_000)
public void watchForChanges() {
tenantLifecycleService.executeSerialized(this::watchForChangesSerialized);
}
private void watchForChangesSerialized() {
long now = currentTimeMillis.getAsLong();
Path path = Path.of(tenantsConfigPath);
if (!Files.exists(path)) {
if (tenantsConfigRequired) {
handleRequiredMissingFile(now);
}
return;
}
if (isRetryDeferred(READ_FAILURE_KEY, now)) {
logDeferredRetry(now);
return;
}
String failureKey = READ_FAILURE_KEY;
try {
File file = new File(tenantsConfigPath);
if (!file.exists()) {
if (tenantsConfigRequired) {
readinessRegistry.markConfigurationFailure();
if (!configurationUnavailable) {
log.error("Обязательный файл конфигурации тенантов недоступен");
}
configurationUnavailable = true;
}
String content = Files.readString(path, StandardCharsets.UTF_8);
String hash = configHash(content);
failureKey = hash;
Projection projection = parseProjection(content, hash);
// Даже неизменившееся содержимое сначала разбирается: fence никогда не
// принимает решение по сырым байтам до проверки структуры JSON.
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
clearFenceWhenExpectedSnapshotIsAlreadyApplied(projection.snapshot());
return;
}
if (isRetryDeferred(hash, now)) {
logDeferredRetry(now);
return;
}
if (!hash.equals(failedRevisionKey)) {
resetRetryState();
}
if (shouldDefer(projection.snapshot())) {
return;
}
String content = new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
String hash = configHash(content);
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
return; // Ничего не изменилось
}
log.info("Обнаружено изменение tenants.json (хеш: {} -> {}), перечитываем конфиг", lastConfigHash, hash);
List<TenantConfig> newTenants = objectMapper.readValue(content, new TypeReference<>() {});
syncTenants(newTenants);
lastConfigHash = hash;
configurationUnavailable = false;
} catch (Exception e) {
readinessRegistry.markConfigurationFailure();
configurationUnavailable = true;
log.error("Ошибка при проверке конфига тенантов: errorType={}", e.getClass().getSimpleName());
log.debug("Технические детали синхронизации конфига тенантов", e);
log.info("Обнаружено изменение tenants.json, выполняется синхронизация конфигурации тенантов");
syncTenants(projection.tenants());
markApplied(projection);
} catch (Exception failure) {
recordFailure(failureKey, failure);
}
}
/**
* Обновляет хеш конфига после ручного обновления Secret с этого же пода.
* Перед API-мутацией приводит runtime к актуальной безопасной проекции файла.
* Ошибка чтения или синхронизации не скрывается: API-операция не должна начинаться
* с неизвестного baseline.
*/
public void refreshHash() {
tenantLifecycleService.executeSerialized(this::refreshHashSerialized);
public void prepareForApiMutation() {
tenantLifecycleService.executeSerialized(this::prepareForApiMutationSerialized);
}
private void refreshHashSerialized() {
try {
File file = new File(tenantsConfigPath);
if (file.exists()) {
String content = new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
lastConfigHash = configHash(content);
} else if (tenantsConfigRequired) {
readinessRegistry.markConfigurationFailure();
configurationUnavailable = true;
private void prepareForApiMutationSerialized() {
Path path = Path.of(tenantsConfigPath);
if (!Files.exists(path)) {
if (!tenantsConfigRequired) {
return;
}
} catch (Exception e) {
readinessRegistry.markConfigurationFailure();
configurationUnavailable = true;
log.warn("Не удалось обновить хеш конфига тенантов: errorType={}",
e.getClass().getSimpleName());
log.debug("Технические детали обновления хеша тенантов", e);
TenantLifecycleException failure = new TenantLifecycleException(
"Обязательный файл конфигурации тенантов недоступен"
);
recordFailure(MISSING_FAILURE_KEY, failure);
throw failure;
}
String failureKey = READ_FAILURE_KEY;
try {
String content = Files.readString(path, StandardCharsets.UTF_8);
String hash = configHash(content);
failureKey = hash;
Projection projection = parseProjection(content, hash);
if (hash.equals(lastConfigHash) && !configurationUnavailable) {
clearFenceWhenExpectedSnapshotIsAlreadyApplied(projection.snapshot());
return;
}
if (shouldDefer(projection.snapshot())) {
return;
}
syncTenants(projection.tenants());
markApplied(projection);
} catch (Exception failure) {
recordFailure(failureKey, failure);
if (failure instanceof TenantLifecycleException lifecycleFailure) {
throw lifecycleFailure;
}
throw new TenantLifecycleException(
"Не удалось подготовить актуальную конфигурацию тенантов перед изменением",
failure
);
}
}
/**
* Устанавливает snapshot fence только после подтверждённой записи в общее
* хранилище. Локальные и no-op операции не создают ложных ожиданий от mount.
*/
public void registerSuccessfulMutation(TenantSecretUpdateReceipt receipt) {
Objects.requireNonNull(receipt, "Квитанция изменения tenant Secret не задана");
tenantLifecycleService.executeSerialized(() -> registerSuccessfulMutationSerialized(receipt));
}
private void registerSuccessfulMutationSerialized(TenantSecretUpdateReceipt receipt) {
if (!receipt.persisted() || !receipt.changed()) {
return;
}
ConfigSnapshot previous = snapshotOf(receipt.previousTenants());
ConfigSnapshot committed = snapshotOf(receipt.committedTenants());
if (lastAppliedSnapshot != null) {
deferredSnapshots.add(lastAppliedSnapshot);
}
if (expectedCommittedSnapshot != null) {
deferredSnapshots.add(expectedCommittedSnapshot);
}
deferredSnapshots.add(previous);
deferredSnapshots.remove(committed);
expectedCommittedSnapshot = committed;
}
static String configHash(String content) {
@@ -127,11 +215,158 @@ public class TenantConfigWatcher {
}
}
/**
* Синхронизирует in-memory тенантов с конфигом из файла.
*/
private void syncTenants(List<TenantConfig> newTenants) {
tenantLifecycleService.synchronizeFromPersistedConfig(newTenants);
private Projection parseProjection(String content, String hash) throws IOException {
List<TenantConfig> tenants = objectMapper.readValue(content, new TypeReference<>() { });
if (tenants == null) {
throw new IllegalArgumentException(
"Корневое значение конфигурации тенантов должно быть массивом"
);
}
if (tenantsConfigRequired && tenants.isEmpty()) {
throw new IllegalArgumentException(
"Обязательная конфигурация тенантов не может быть пустой"
);
}
return new Projection(hash, List.copyOf(tenants), snapshotOf(tenants));
}
private ConfigSnapshot snapshotOf(List<TenantConfig> tenants) {
Objects.requireNonNull(tenants, "Список конфигураций тенантов не задан");
List<NormalizedTenantConfig> normalized = new ArrayList<>(tenants.size());
for (TenantConfig tenant : tenants) {
Objects.requireNonNull(tenant, "Конфигурация тенанта не задана");
String domain = normalizeDomain(tenant.getDomain());
String name = tenant.getName() == null || tenant.getName().isBlank()
? domain
: tenant.getName().trim();
String url = tenant.getUrl() == null ? null : tenant.getUrl().trim();
normalized.add(new NormalizedTenantConfig(
name,
domain,
url,
tenant.getUsername(),
tenant.getPassword()
));
}
normalized.sort(NormalizedTenantConfig.ORDER);
return new ConfigSnapshot(normalized);
}
private String normalizeDomain(String domain) {
return domain == null ? null : domain.trim().toLowerCase(Locale.ROOT);
}
private void syncTenants(List<TenantConfig> tenants) {
tenantLifecycleService.synchronizeFromPersistedConfig(tenants);
}
private boolean shouldDefer(ConfigSnapshot snapshot) {
return expectedCommittedSnapshot != null && deferredSnapshots.contains(snapshot);
}
private void clearFenceWhenExpectedSnapshotIsAlreadyApplied(ConfigSnapshot snapshot) {
if (snapshot.equals(expectedCommittedSnapshot)) {
clearSnapshotFence();
}
}
private void markApplied(Projection projection) {
lastConfigHash = projection.hash();
lastAppliedSnapshot = projection.snapshot();
configurationUnavailable = false;
resetRetryState();
clearSnapshotFence();
}
private void handleRequiredMissingFile(long now) {
if (isRetryDeferred(MISSING_FAILURE_KEY, now)) {
logDeferredRetry(now);
return;
}
recordFailure(
MISSING_FAILURE_KEY,
new TenantLifecycleException("Обязательный файл конфигурации тенантов недоступен")
);
}
private boolean isRetryDeferred(String failureKey, long now) {
return failureKey.equals(failedRevisionKey) && now < retryNotBeforeMillis;
}
private void logDeferredRetry(long now) {
log.debug("Повторная синхронизация tenants.json отложена ещё на {} мс",
Math.max(0L, retryNotBeforeMillis - now));
}
private void recordFailure(String failureKey, Exception failure) {
readinessRegistry.markConfigurationFailure();
configurationUnavailable = true;
long retryDelay = registerFailure(failureKey);
log.error("Ошибка синхронизации конфигурации тенантов: попытка={}, "
+ "следующий повтор не ранее чем через {} мс, errorType={}",
consecutiveFailures,
retryDelay,
failure.getClass().getSimpleName());
log.debug("Технические детали синхронизации конфигурации тенантов", failure);
}
private long registerFailure(String failureKey) {
if (failureKey.equals(failedRevisionKey) && consecutiveFailures > 0) {
consecutiveFailures = Math.min(consecutiveFailures + 1, 31);
} else {
failedRevisionKey = failureKey;
consecutiveFailures = 1;
}
long delay = retryDelayMillis(consecutiveFailures);
retryNotBeforeMillis = safeAdd(currentTimeMillis.getAsLong(), delay);
return delay;
}
static long retryDelayMillis(int failureCount) {
long delay = RETRY_BASE_DELAY_MILLIS;
for (int attempt = 1; attempt < failureCount && delay < RETRY_MAX_DELAY_MILLIS; attempt++) {
delay = Math.min(delay * 2, RETRY_MAX_DELAY_MILLIS);
}
return delay;
}
private long safeAdd(long value, long increment) {
return value > Long.MAX_VALUE - increment ? Long.MAX_VALUE : value + increment;
}
private void resetRetryState() {
failedRevisionKey = "";
consecutiveFailures = 0;
retryNotBeforeMillis = 0L;
}
private void clearSnapshotFence() {
expectedCommittedSnapshot = null;
deferredSnapshots.clear();
}
private record Projection(String hash, List<TenantConfig> tenants, ConfigSnapshot snapshot) {
}
private record ConfigSnapshot(List<NormalizedTenantConfig> tenants) {
private ConfigSnapshot {
tenants = List.copyOf(tenants);
}
}
private record NormalizedTenantConfig(
String name,
String domain,
String url,
String username,
String password
) {
private static final Comparator<String> NULL_SAFE_TEXT = Comparator.nullsFirst(String::compareTo);
private static final Comparator<NormalizedTenantConfig> ORDER =
Comparator.comparing(NormalizedTenantConfig::domain, NULL_SAFE_TEXT)
.thenComparing(NormalizedTenantConfig::name, NULL_SAFE_TEXT)
.thenComparing(NormalizedTenantConfig::url, NULL_SAFE_TEXT)
.thenComparing(NormalizedTenantConfig::username, NULL_SAFE_TEXT)
.thenComparing(NormalizedTenantConfig::password, NULL_SAFE_TEXT);
}
}

View File

@@ -7,6 +7,7 @@ import com.magistr.app.config.tenant.TenantContext;
import com.magistr.app.config.tenant.TenantRoutingDataSource;
import com.magistr.app.model.Role;
import com.magistr.app.service.TenantLifecycleException;
import com.magistr.app.service.TenantLifecycleMutationResult;
import com.magistr.app.service.TenantLifecycleService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
@@ -90,8 +91,11 @@ public class DatabaseController {
public ResponseEntity<Map<String, Object>> addTenant(@RequestBody TenantConfig config) {
try {
return tenantLifecycleService.executeSerialized(() -> {
TenantConfig saved = tenantLifecycleService.addOrUpdateTenant(config);
tenantConfigWatcher.refreshHash();
tenantConfigWatcher.prepareForApiMutation();
TenantLifecycleMutationResult mutation =
tenantLifecycleService.addOrUpdateTenantWithReceipt(config);
tenantConfigWatcher.registerSuccessfulMutation(mutation.receipt());
TenantConfig saved = mutation.tenant();
Map<String, Object> result = new HashMap<>();
result.put("success", true);
result.put("message", "Тенант '" + saved.getDomain() + "' добавлен");
@@ -111,8 +115,11 @@ public class DatabaseController {
public ResponseEntity<Map<String, Object>> removeTenant(@PathVariable("domain") String domain) {
try {
return tenantLifecycleService.executeSerialized(() -> {
TenantConfig removed = tenantLifecycleService.removeTenant(domain);
tenantConfigWatcher.refreshHash();
tenantConfigWatcher.prepareForApiMutation();
TenantLifecycleMutationResult mutation =
tenantLifecycleService.removeTenantWithReceipt(domain);
tenantConfigWatcher.registerSuccessfulMutation(mutation.receipt());
TenantConfig removed = mutation.tenant();
Map<String, Object> result = new HashMap<>();
result.put("success", true);
result.put("message", "Тенант '" + removed.getDomain() + "' удалён");

View File

@@ -0,0 +1,41 @@
package com.magistr.app.service;
import com.magistr.app.config.tenant.TenantConfig;
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
import java.util.Objects;
/**
* Результат успешной lifecycle-мутации тенанта вместе с квитанцией персистенции.
*
* <p>Конфигурация копируется при создании и чтении, чтобы вызывающий код не мог изменить
* уже активированный runtime-снимок через изменяемый DTO.</p>
*/
public final class TenantLifecycleMutationResult {
private final TenantConfig tenant;
private final TenantSecretUpdateReceipt receipt;
public TenantLifecycleMutationResult(TenantConfig tenant, TenantSecretUpdateReceipt receipt) {
this.tenant = copy(Objects.requireNonNull(tenant, "Конфигурация тенанта не задана"));
this.receipt = Objects.requireNonNull(receipt, "Квитанция изменения tenant Secret не задана");
}
public TenantConfig tenant() {
return copy(tenant);
}
public TenantSecretUpdateReceipt receipt() {
return receipt;
}
private static TenantConfig copy(TenantConfig source) {
return new TenantConfig(
source.getName(),
source.getDomain(),
source.getUrl(),
source.getUsername(),
source.getPassword()
);
}
}

View File

@@ -21,10 +21,9 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.Objects;
import java.util.regex.Pattern;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@Service
public class TenantLifecycleService {
@@ -72,6 +71,13 @@ public class TenantLifecycleService {
* Добавляет или обновляет tenant без разрыва действующего подключения.
*/
public synchronized TenantConfig addOrUpdateTenant(TenantConfig requestedConfig) {
return addOrUpdateTenantWithReceipt(requestedConfig).tenant();
}
/**
* Добавляет или обновляет tenant и возвращает подтверждённый результат персистенции.
*/
public synchronized TenantLifecycleMutationResult addOrUpdateTenantWithReceipt(TenantConfig requestedConfig) {
TenantConfig config = validateAndNormalize(requestedConfig);
Map<String, TenantConfig> oldConfigs = routingDataSource.snapshotTenantConfigs();
Map<String, TenantConfig> desiredConfigs = new LinkedHashMap<>(oldConfigs);
@@ -96,7 +102,7 @@ public class TenantLifecycleService {
publishSuccessfulMutation(receipt, sortedConfigs(desiredConfigs), config);
retireDataSource(previous == null ? null : previous.dataSource());
log.info("Lifecycle тенанта '{}' успешно применён", config.getDomain());
return copyConfig(config);
return new TenantLifecycleMutationResult(config, receipt);
} finally {
if (!activated) {
closeCandidateDataSource(candidate);
@@ -108,6 +114,13 @@ public class TenantLifecycleService {
* Сначала сохраняет desired-конфигурацию, затем локально отключает tenant.
*/
public synchronized TenantConfig removeTenant(String requestedDomain) {
return removeTenantWithReceipt(requestedDomain).tenant();
}
/**
* Удаляет tenant и возвращает подтверждённый результат персистенции.
*/
public synchronized TenantLifecycleMutationResult removeTenantWithReceipt(String requestedDomain) {
String domain = validateAndNormalizeDomain(requestedDomain);
Map<String, TenantConfig> oldConfigs = routingDataSource.snapshotTenantConfigs();
TenantConfig existing = oldConfigs.get(domain);
@@ -132,13 +145,14 @@ public class TenantLifecycleService {
publishSuccessfulMutation(receipt, sortedConfigs(desiredConfigs), null);
retireDataSource(removed.dataSource());
log.info("Lifecycle тенанта '{}' удалён", domain);
return copyConfig(existing);
return new TenantLifecycleMutationResult(existing, receipt);
}
/**
* Синхронизирует уже персистированный список из mounted Secret. Метод не пишет
* Secret повторно и использует тот же monitor, что ручные API add/update/remove.
* Обновление параметров существующего domain остаётся отдельной задачей watcher №13.
* Для нового или изменённого домена подключение сначала полностью подготавливается,
* проверяется и мигрируется, после чего атомарно заменяет прежний runtime-снимок.
*/
public synchronized void synchronizeFromPersistedConfig(List<TenantConfig> requestedConfigs) {
if (requestedConfigs == null) {
@@ -146,47 +160,55 @@ public class TenantLifecycleService {
}
List<TenantConfig> configs = requestedConfigs.stream()
.map(this::validateAndNormalize)
.sorted(Comparator.comparing(TenantConfig::getDomain))
.toList();
Set<String> desiredDomains = configs.stream()
.map(TenantConfig::getDomain)
.collect(Collectors.toSet());
readinessRegistry.clearConfigurationFailure();
readinessRegistry.replaceDesired(configs);
if (desiredDomains.size() != configs.size()) {
throw new IllegalArgumentException("Конфигурация содержит повторяющиеся домены тенантов");
}
Map<String, TenantConfig> desiredConfigs = new LinkedHashMap<>();
for (TenantConfig config : configs) {
if (!routingDataSource.hasTenant(config.getDomain())) {
log.info("Активируем нового тенанта '{}' из persisted-конфигурации", config.getDomain());
activatePersistedConfig(config);
if (desiredConfigs.put(config.getDomain(), config) != null) {
throw new IllegalArgumentException("Конфигурация содержит повторяющиеся домены тенантов");
}
}
for (String existingDomain : new ArrayList<>(routingDataSource.snapshotTenantConfigs().keySet())) {
if (!desiredDomains.contains(existingDomain)) {
Map<String, TenantConfig> currentConfigs = routingDataSource.snapshotTenantConfigs();
readinessRegistry.replaceDesired(configs);
for (TenantConfig config : configs) {
TenantConfig currentConfig = currentConfigs.get(config.getDomain());
if (currentConfig == null) {
log.info("Активируем нового тенанта '{}' из persisted-конфигурации", config.getDomain());
activatePersistedConfig(config);
} else if (!sameConfig(currentConfig, config)) {
log.info("Заменяем подключение тенанта '{}' из persisted-конфигурации", config.getDomain());
activatePersistedConfig(config);
} else {
recoverPersistedReadinessIfNeeded(config);
}
}
for (String existingDomain : new ArrayList<>(currentConfigs.keySet())) {
if (!desiredConfigs.containsKey(existingDomain)) {
log.info("Отключаем тенанта '{}' — его нет в persisted-конфигурации", existingDomain);
TenantRoutingDataSource.TenantState removed =
routingDataSource.removeTenantAtomically(existingDomain);
retireDataSource(removed == null ? null : removed.dataSource());
}
}
readinessRegistry.clearConfigurationFailure();
}
private void activatePersistedConfig(TenantConfig config) {
HikariDataSource candidate = null;
boolean activated = false;
TenantRoutingDataSource.TenantState previous;
readinessRegistry.markMigrationStarted(config);
try {
candidate = prepareCandidate(config);
verifyConnection(candidate);
migrateCandidate(candidate);
TenantRoutingDataSource.TenantState previous = routingDataSource.swapTenant(config, candidate);
previous = routingDataSource.swapTenant(config, candidate);
activated = true;
readinessRegistry.markMigrationSucceeded(config);
readinessRegistry.markConnectivity(config.getDomain(), true);
retireDataSource(previous == null ? null : previous.dataSource());
} catch (RuntimeException activationFailure) {
readinessRegistry.markMigrationFailed(config);
throw activationFailure;
@@ -195,6 +217,29 @@ public class TenantLifecycleService {
closeCandidateDataSource(candidate);
}
}
// После swap новый pool уже является рабочим. Ошибка публикации readiness не должна
// ошибочно помечать успешно мигрированный pool как FAILED: следующий retry восстановит
// состояние по активному подключению без повторной замены DataSource.
retireDataSource(previous == null ? null : previous.dataSource());
readinessRegistry.markMigrationSucceeded(config);
readinessRegistry.markConnectivity(config.getDomain(), true);
}
private void recoverPersistedReadinessIfNeeded(TenantConfig config) {
TenantReadinessRegistry.TenantStatus status =
readinessRegistry.snapshot().get(config.getDomain());
if (status == null
|| status.migrationState() == TenantReadinessRegistry.MigrationState.SUCCEEDED) {
return;
}
if (!routingDataSource.testConnection(config.getDomain())) {
throw new TenantLifecycleException(
"Активное подключение тенанта не прошло проверку после предыдущей синхронизации"
);
}
readinessRegistry.markMigrationSucceeded(config);
readinessRegistry.markConnectivity(config.getDomain(), true);
}
private HikariDataSource prepareCandidate(TenantConfig config) {
@@ -398,9 +443,24 @@ public class TenantLifecycleService {
);
}
private boolean sameConfig(TenantConfig first, TenantConfig second) {
TenantConfig normalizedFirst = validateAndNormalize(first);
TenantConfig normalizedSecond = validateAndNormalize(second);
return Objects.equals(normalizedFirst.getName(), normalizedSecond.getName())
&& Objects.equals(normalizedFirst.getDomain(), normalizedSecond.getDomain())
&& Objects.equals(normalizedFirst.getUrl(), normalizedSecond.getUrl())
&& Objects.equals(normalizedFirst.getUsername(), normalizedSecond.getUsername())
&& Objects.equals(normalizedFirst.getPassword(), normalizedSecond.getPassword());
}
private void retireDataSource(DataSource dataSource) {
if (dataSource != null) {
retiredPoolService.retire(dataSource);
try {
retiredPoolService.retire(dataSource);
} catch (RuntimeException retirementFailure) {
logTechnicalFailure("Не удалось передать прежний tenant pool на отложенное закрытие",
retirementFailure);
}
}
}

View File

@@ -511,7 +511,10 @@ class KubernetesTenantSecretUpdaterTest {
files.tokenPath(),
files.namespacePath(),
files.caPath(),
server.url("/").toString(),
// MockWebServer на Windows может вернуть canonical loopback hostname
// kubernetes.docker.internal. Фиксируем имя из SAN тестового сертификата,
// сохраняя реальную проверку hostname в клиенте.
"https://localhost:" + server.getPort(),
"tenants-secret",
objectMapper,
maxAttempts,

View File

@@ -1,33 +1,35 @@
package com.magistr.app.config.tenant;
import com.magistr.app.config.tenant.health.TenantReadinessRegistry;
import com.magistr.app.service.RetiredTenantPoolService;
import com.magistr.app.service.TenantDatabaseMigrationService;
import com.magistr.app.service.TenantLifecycleException;
import com.magistr.app.service.TenantLifecycleService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;
import org.springframework.boot.test.system.CapturedOutput;
import org.springframework.boot.test.system.OutputCaptureExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.LongSupplier;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyList;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.never;
@ExtendWith(OutputCaptureExtension.class)
class TenantConfigWatcherTest {
@TempDir
@@ -48,29 +50,62 @@ class TenantConfigWatcherTest {
.isNotEqualTo(TenantConfigWatcher.configHash("tenant-b"));
}
@Test
void prepareForApiMutationSynchronizesUnappliedBaselineOnce() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String content = json("alpha");
Files.writeString(configPath, content);
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.prepareForApiMutation();
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash(content));
assertThat(ReflectionTestUtils.getField(watcher, "lastAppliedSnapshot")).isNotNull();
}
@Test
void prepareForApiMutationPropagatesReadFailure() throws Exception {
Path configPath = Files.createDirectory(tempDirectory.resolve("tenants.json"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
TenantConfigWatcher watcher = watcher(
lifecycleService,
readinessRegistry,
configPath,
new AtomicLong(1_000L)
);
assertThatThrownBy(watcher::prepareForApiMutation)
.isInstanceOf(TenantLifecycleException.class)
.hasMessage("Не удалось подготовить актуальную конфигурацию тенантов перед изменением");
verify(readinessRegistry).markConfigurationFailure();
verify(lifecycleService, never()).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "configurationUnavailable")).isEqualTo(true);
}
@Test
void advancesHashOnlyAfterSuccessfulLifecycleSynchronization() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String content = """
[{
"name": "Тестовый tenant",
"domain": "alpha",
"url": "jdbc:postgresql://db/alpha",
"username": "alpha",
"password": "тестовый-пароль"
}]
""";
String content = json("alpha");
Files.writeString(configPath, content);
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
doThrow(new IllegalStateException("тестовая ошибка синхронизации"))
.doNothing()
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
AtomicLong now = new AtomicLong(1_000L);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, now);
watcher.watchForChanges();
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash")).isEqualTo("");
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
watcher.watchForChanges();
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash(content));
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
@@ -80,112 +115,441 @@ class TenantConfigWatcherTest {
}
@Test
void successfulHashPreventsRepeatedSynchronizationOfSameContent() throws Exception {
void retriesSameRevisionOnlyAfterBackoffAndResetsStateAfterSuccess() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String content = "[]";
Files.writeString(configPath, content);
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
doNothing().when(lifecycleService).synchronizeFromPersistedConfig(anyList());
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
doThrow(new IllegalStateException("первая тестовая ошибка"))
.doThrow(new IllegalStateException("вторая тестовая ошибка"))
.doNothing()
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
AtomicLong now = new AtomicLong(10_000L);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, now);
watcher.watchForChanges();
watcher.watchForChanges();
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
watcher.watchForChanges();
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS * 2 - 1);
watcher.watchForChanges();
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
now.incrementAndGet();
watcher.watchForChanges();
verify(lifecycleService, times(3)).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(0);
assertThat(ReflectionTestUtils.getField(watcher, "retryNotBeforeMillis")).isEqualTo(0L);
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash(content));
}
@Test
void watcherCannotReadBetweenSerializedApiMutationAndRefreshHash() throws Exception {
void retryDelayGrowsExponentiallyAndIsCapped() {
assertThat(TenantConfigWatcher.retryDelayMillis(1)).isEqualTo(30_000L);
assertThat(TenantConfigWatcher.retryDelayMillis(2)).isEqualTo(60_000L);
assertThat(TenantConfigWatcher.retryDelayMillis(3)).isEqualTo(120_000L);
assertThat(TenantConfigWatcher.retryDelayMillis(4)).isEqualTo(240_000L);
assertThat(TenantConfigWatcher.retryDelayMillis(5)).isEqualTo(300_000L);
assertThat(TenantConfigWatcher.retryDelayMillis(31)).isEqualTo(300_000L);
}
@Test
void newRevisionBypassesBackoffImmediately() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String content = "[]";
Files.writeString(configPath, content);
TenantRoutingDataSource routingDataSource = mock(TenantRoutingDataSource.class);
TenantLifecycleService lifecycleService = new TenantLifecycleService(
routingDataSource,
mock(TenantConfigStore.class),
mock(TenantDatabaseMigrationService.class),
mock(RetiredTenantPoolService.class),
mock(TenantReadinessRegistry.class)
);
TenantConfigWatcher watcher = new TenantConfigWatcher(
Files.writeString(configPath, "[]");
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
doThrow(new IllegalStateException("тестовая ошибка"))
.doNothing()
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, new AtomicLong(1_000L));
watcher.watchForChanges();
Files.writeString(configPath, "[ ]");
watcher.watchForChanges();
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash("[ ]"));
}
@Test
void restoredLastAppliedRevisionIsReconciledAfterFailedChange() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String firstRevision = "[]";
String failedRevision = "[ ]";
Files.writeString(configPath, firstRevision);
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
doNothing()
.doThrow(new IllegalStateException("тестовая ошибка новой ревизии"))
.doNothing()
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, new AtomicLong(1_000L));
watcher.watchForChanges();
Files.writeString(configPath, failedRevision);
watcher.watchForChanges();
Files.writeString(configPath, firstRevision);
watcher.watchForChanges();
verify(lifecycleService, times(3)).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash(firstRevision));
assertThat(ReflectionTestUtils.getField(watcher, "configurationUnavailable")).isEqualTo(false);
}
@Test
void semanticFenceDefersStaleProjectionWithDifferentFormatting() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String baseline = json("alpha");
Files.writeString(configPath, baseline);
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha"),
tenants("alpha", "beta")
));
Files.writeString(configPath, prettyJson("alpha"));
watcher.watchForChanges();
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash(baseline));
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNotNull();
}
@Test
void prepareForNextMutationDoesNotApplyKnownStaleProjection() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
Files.writeString(configPath, json("alpha"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha"),
tenants("alpha", "beta")
));
watcher.prepareForApiMutation();
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
}
@Test
void nonPersistedReceiptDoesNotArmFence() {
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(
lifecycleService,
mock(TenantReadinessRegistry.class)
tempDirectory.resolve("tenants.json")
);
ReflectionTestUtils.setField(watcher, "tenantsConfigPath", configPath.toString());
CountDownLatch apiMutationReachedRefreshWindow = new CountDownLatch(1);
CountDownLatch allowApiRefresh = new CountDownLatch(1);
AtomicReference<Thread> watcherThread = new AtomicReference<>();
ExecutorService executor = Executors.newFixedThreadPool(2);
try {
Future<?> api = executor.submit(() -> lifecycleService.executeSerialized(() -> {
apiMutationReachedRefreshWindow.countDown();
await(allowApiRefresh, "Ожидание разрешения refreshHash прервано");
watcher.refreshHash();
}));
assertThat(apiMutationReachedRefreshWindow.await(5, TimeUnit.SECONDS)).isTrue();
watcher.registerSuccessfulMutation(receipt(
false,
true,
tenants("alpha"),
tenants("alpha", "beta")
));
Future<?> concurrentWatcher = executor.submit(() -> {
watcherThread.set(Thread.currentThread());
watcher.watchForChanges();
});
awaitBlocked(watcherThread);
assertThat(concurrentWatcher).isNotDone();
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
}
allowApiRefresh.countDown();
api.get(5, TimeUnit.SECONDS);
concurrentWatcher.get(5, TimeUnit.SECONDS);
@Test
void persistedNoOpDoesNotSuppressMergedProjection() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
Files.writeString(configPath, json("alpha"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.registerSuccessfulMutation(receipt(
true,
false,
tenants("alpha"),
tenants("alpha")
));
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash"))
.isEqualTo(TenantConfigWatcher.configHash(content));
verify(routingDataSource, never()).snapshotTenantConfigs();
} finally {
allowApiRefresh.countDown();
executor.shutdownNow();
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
}
Files.writeString(configPath, json("alpha", "gamma"));
watcher.watchForChanges();
ArgumentCaptor<List<TenantConfig>> captor = tenantListCaptor();
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(captor.capture());
assertThat(captor.getAllValues().get(1))
.extracting(TenantConfig::getDomain)
.containsExactly("alpha", "gamma");
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
}
@Test
void twoRapidMutationsDeferIntermediateProjectionUntilLatestCommit() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
Files.writeString(configPath, json("alpha"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha"),
tenants("alpha", "beta")
));
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha", "beta"),
tenants("alpha", "beta", "gamma")
));
Files.writeString(configPath, json("alpha", "beta"));
watcher.watchForChanges();
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
Files.writeString(configPath, json("alpha", "beta", "gamma"));
watcher.watchForChanges();
ArgumentCaptor<List<TenantConfig>> captor = tenantListCaptor();
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(captor.capture());
assertThat(captor.getAllValues().get(1))
.extracting(TenantConfig::getDomain)
.containsExactly("alpha", "beta", "gamma");
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
}
@Test
void cyclicRapidMutationsClearFenceWhenLatestCommitIsAlreadyApplied() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
Files.writeString(configPath, json("alpha"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha"),
tenants("alpha", "beta")
));
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha", "beta"),
tenants("alpha")
));
watcher.watchForChanges();
verify(lifecycleService).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
}
@Test
void unknownConcurrentSnapshotSupersedesExpectedAndIsApplied() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
Files.writeString(configPath, json("alpha"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantConfigWatcher watcher = watcher(lifecycleService, configPath);
watcher.prepareForApiMutation();
watcher.registerSuccessfulMutation(receipt(
true,
true,
tenants("alpha"),
tenants("alpha", "beta")
));
Files.writeString(configPath, json("alpha", "gamma"));
watcher.watchForChanges();
ArgumentCaptor<List<TenantConfig>> captor = tenantListCaptor();
verify(lifecycleService, times(2)).synchronizeFromPersistedConfig(captor.capture());
assertThat(captor.getAllValues().get(1))
.extracting(TenantConfig::getDomain)
.containsExactly("alpha", "gamma");
assertThat(ReflectionTestUtils.getField(watcher, "expectedCommittedSnapshot")).isNull();
assertThat((Iterable<?>) ReflectionTestUtils.getField(watcher, "deferredSnapshots")).isEmpty();
}
@Test
void requiredMissingFileUsesBackoff() {
Path configPath = tempDirectory.resolve("tenants.json");
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
AtomicLong now = new AtomicLong(1_000L);
TenantConfigWatcher watcher = watcher(lifecycleService, readinessRegistry, configPath, now);
ReflectionTestUtils.setField(watcher, "tenantsConfigRequired", true);
watcher.watchForChanges();
watcher.watchForChanges();
verify(readinessRegistry).markConfigurationFailure();
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(1);
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
watcher.watchForChanges();
verify(readinessRegistry, times(2)).markConfigurationFailure();
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(2);
}
@Test
void readFailureUsesBackoff() throws Exception {
Path configPath = Files.createDirectory(tempDirectory.resolve("tenants.json"));
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
AtomicLong now = new AtomicLong(1_000L);
TenantConfigWatcher watcher = watcher(lifecycleService, readinessRegistry, configPath, now);
watcher.watchForChanges();
watcher.watchForChanges();
verify(readinessRegistry).markConfigurationFailure();
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(1);
now.addAndGet(TenantConfigWatcher.RETRY_BASE_DELAY_MILLIS);
watcher.watchForChanges();
verify(readinessRegistry, times(2)).markConfigurationFailure();
assertThat(ReflectionTestUtils.getField(watcher, "consecutiveFailures")).isEqualTo(2);
}
@Test
void requiredEmptyConfigMarksFailureAndKeepsHashUnapplied() throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
Files.writeString(configPath, "[]");
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
TenantReadinessRegistry readinessRegistry = mock(TenantReadinessRegistry.class);
TenantConfigWatcher watcher = watcher(
lifecycleService,
readinessRegistry,
configPath,
new AtomicLong(1_000L)
);
ReflectionTestUtils.setField(watcher, "tenantsConfigRequired", true);
watcher.watchForChanges();
verify(readinessRegistry).markConfigurationFailure();
verify(lifecycleService, never()).synchronizeFromPersistedConfig(anyList());
assertThat(ReflectionTestUtils.getField(watcher, "lastConfigHash")).isEqualTo("");
}
@Test
void failureLogsRetryMetadataWithoutSecretsOrFingerprints(CapturedOutput output) throws Exception {
Path configPath = tempDirectory.resolve("tenants.json");
String content = """
[{"domain":"alpha","url":"jdbc:postgresql://secret-host/db",
"username":"secret-user","password":"secret-password"}]
""";
Files.writeString(configPath, content);
TenantLifecycleService lifecycleService = mock(TenantLifecycleService.class);
doThrow(new IllegalStateException("секретные технические детали"))
.when(lifecycleService).synchronizeFromPersistedConfig(anyList());
TenantConfigWatcher watcher = watcher(lifecycleService, configPath, new AtomicLong(1_000L));
watcher.watchForChanges();
assertThat(output)
.contains("Ошибка синхронизации конфигурации тенантов")
.contains("попытка=1")
.contains("через 30000 мс")
.doesNotContain("secret-host")
.doesNotContain("secret-user")
.doesNotContain("secret-password")
.doesNotContain(TenantConfigWatcher.configHash(content))
.doesNotContain("revision=")
.doesNotContain("fingerprint")
.doesNotContain("хеш:");
}
private TenantConfigWatcher watcher(TenantLifecycleService lifecycleService, Path configPath) {
return watcher(lifecycleService, configPath, new AtomicLong(1_000L));
}
private TenantConfigWatcher watcher(TenantLifecycleService lifecycleService,
Path configPath,
AtomicLong now) {
return watcher(lifecycleService, mock(TenantReadinessRegistry.class), configPath, now);
}
private TenantConfigWatcher watcher(TenantLifecycleService lifecycleService,
TenantReadinessRegistry readinessRegistry,
Path configPath,
AtomicLong now) {
doAnswer(invocation -> {
invocation.<Runnable>getArgument(0).run();
return null;
}).when(lifecycleService).executeSerialized(any(Runnable.class));
TenantConfigWatcher watcher = new TenantConfigWatcher(
lifecycleService,
mock(TenantReadinessRegistry.class)
);
TenantConfigWatcher watcher = new TenantConfigWatcher(lifecycleService, readinessRegistry);
ReflectionTestUtils.setField(watcher, "tenantsConfigPath", configPath.toString());
ReflectionTestUtils.setField(watcher, "currentTimeMillis", (LongSupplier) now::get);
return watcher;
}
private void await(CountDownLatch latch, String message) {
try {
if (!latch.await(5, TimeUnit.SECONDS)) {
throw new IllegalStateException(message);
}
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new IllegalStateException(message, exception);
}
private TenantSecretUpdateReceipt receipt(boolean persisted,
boolean changed,
List<TenantConfig> previous,
List<TenantConfig> committed) {
return new TenantSecretUpdateReceipt(
persisted,
changed,
persisted ? "resource-version" : null,
previous,
committed
);
}
private void awaitBlocked(AtomicReference<Thread> threadReference) {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
while (System.nanoTime() < deadline) {
Thread thread = threadReference.get();
if (thread != null && thread.getState() == Thread.State.BLOCKED) {
return;
}
try {
Thread.sleep(10);
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Ожидание блокировки watcher прервано", exception);
}
}
throw new IllegalStateException("Watcher не заблокировался на общем lifecycle coordinator");
private List<TenantConfig> tenants(String... domains) {
return java.util.Arrays.stream(domains).map(this::tenant).toList();
}
private TenantConfig tenant(String domain) {
return new TenantConfig(
domain,
domain,
"jdbc:postgresql://db/" + domain,
domain,
"password-" + domain
);
}
private String json(String... domains) {
return "[" + java.util.Arrays.stream(domains)
.map(domain -> "{\"name\":\"" + domain
+ "\",\"domain\":\"" + domain
+ "\",\"url\":\"jdbc:postgresql://db/" + domain
+ "\",\"username\":\"" + domain
+ "\",\"password\":\"password-" + domain + "\"}")
.collect(java.util.stream.Collectors.joining(",")) + "]";
}
private String prettyJson(String domain) {
return """
[
{
"password": "password-%1$s",
"username": "%1$s",
"url": "jdbc:postgresql://db/%1$s",
"domain": "%1$s",
"name": "%1$s"
}
]
""".formatted(domain);
}
@SuppressWarnings({"unchecked", "rawtypes"})
private ArgumentCaptor<List<TenantConfig>> tenantListCaptor() {
return (ArgumentCaptor) ArgumentCaptor.forClass(List.class);
}
}

View File

@@ -1,23 +1,32 @@
package com.magistr.app.controller;
import com.magistr.app.config.tenant.TenantConfigWatcher;
import com.magistr.app.config.tenant.TenantConfig;
import com.magistr.app.config.tenant.TenantRoutingDataSource;
import com.magistr.app.config.tenant.TenantSecretUpdateReceipt;
import com.magistr.app.service.TenantLifecycleException;
import com.magistr.app.service.TenantLifecycleMutationResult;
import com.magistr.app.service.TenantLifecycleService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.test.web.servlet.MockMvc;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@@ -68,7 +77,128 @@ class DatabaseControllerTest {
.andExpect(jsonPath("$.message")
.value(org.hamcrest.Matchers.not(org.hamcrest.Matchers.containsString("JDBC"))));
verify(tenantConfigWatcher, never()).refreshHash();
verify(tenantConfigWatcher, never()).prepareForApiMutation();
verify(tenantConfigWatcher, never()).registerSuccessfulMutation(any());
verify(lifecycleService).executeSerialized(any(Supplier.class));
}
@Test
void successfulAddPreparesProjectionAndRegistersPersistenceReceipt() throws Exception {
executeSerializedActionsImmediately();
TenantConfig saved = new TenantConfig(
"Тестовый университет",
"test",
"jdbc:postgresql://db/test",
"user",
"password"
);
TenantSecretUpdateReceipt receipt = receipt(saved);
when(lifecycleService.addOrUpdateTenantWithReceipt(any(TenantConfig.class)))
.thenReturn(new TenantLifecycleMutationResult(saved, receipt));
mockMvc.perform(post("/api/database/tenants")
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"name": "Тестовый университет",
"domain": "test",
"url": "jdbc:postgresql://db/test",
"username": "user",
"password": "password"
}
"""))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true));
InOrder order = inOrder(lifecycleService, tenantConfigWatcher);
order.verify(lifecycleService).executeSerialized(any(Supplier.class));
order.verify(tenantConfigWatcher).prepareForApiMutation();
order.verify(lifecycleService).addOrUpdateTenantWithReceipt(any(TenantConfig.class));
order.verify(tenantConfigWatcher).registerSuccessfulMutation(receipt);
}
@Test
void successfulRemovalPreparesProjectionAndRegistersPersistenceReceipt() throws Exception {
executeSerializedActionsImmediately();
TenantConfig removed = new TenantConfig(
"Тестовый университет",
"test",
"jdbc:postgresql://db/test",
"user",
"password"
);
TenantSecretUpdateReceipt receipt = receipt();
when(lifecycleService.removeTenantWithReceipt("test"))
.thenReturn(new TenantLifecycleMutationResult(removed, receipt));
mockMvc.perform(delete("/api/database/tenants/test"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.success").value(true));
InOrder order = inOrder(lifecycleService, tenantConfigWatcher);
order.verify(lifecycleService).executeSerialized(any(Supplier.class));
order.verify(tenantConfigWatcher).prepareForApiMutation();
order.verify(lifecycleService).removeTenantWithReceipt("test");
order.verify(tenantConfigWatcher).registerSuccessfulMutation(receipt);
}
@Test
void projectionReadFailurePreventsAddAndReturnsServiceUnavailable() throws Exception {
executeSerializedActionsImmediately();
doThrow(new TenantLifecycleException(
"Не удалось прочитать конфигурацию тенантов перед изменением",
new IllegalStateException("тестовая ошибка чтения")
)).when(tenantConfigWatcher).prepareForApiMutation();
mockMvc.perform(post("/api/database/tenants")
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"name": "Тестовый университет",
"domain": "test",
"url": "jdbc:postgresql://db/test",
"username": "user",
"password": "password"
}
"""))
.andExpect(status().isServiceUnavailable())
.andExpect(jsonPath("$.success").value(false))
.andExpect(jsonPath("$.message")
.value("Не удалось прочитать конфигурацию тенантов перед изменением"));
verify(lifecycleService, never()).addOrUpdateTenantWithReceipt(any());
verify(tenantConfigWatcher, never()).registerSuccessfulMutation(any());
}
@Test
void projectionReadFailurePreventsRemovalAndReturnsServiceUnavailable() throws Exception {
executeSerializedActionsImmediately();
doThrow(new TenantLifecycleException(
"Не удалось прочитать конфигурацию тенантов перед изменением",
new IllegalStateException("тестовая ошибка чтения")
)).when(tenantConfigWatcher).prepareForApiMutation();
mockMvc.perform(delete("/api/database/tenants/test"))
.andExpect(status().isServiceUnavailable())
.andExpect(jsonPath("$.success").value(false));
verify(lifecycleService, never()).removeTenantWithReceipt(any());
verify(tenantConfigWatcher, never()).registerSuccessfulMutation(any());
}
private void executeSerializedActionsImmediately() {
doAnswer(invocation -> invocation.<Supplier<?>>getArgument(0).get())
.when(lifecycleService)
.executeSerialized(org.mockito.ArgumentMatchers.<Supplier<Object>>any());
}
private TenantSecretUpdateReceipt receipt(TenantConfig... tenants) {
return new TenantSecretUpdateReceipt(
true,
true,
"resource-version",
List.of(),
List.of(tenants)
);
}
}

View File

@@ -178,6 +178,52 @@ class TenantLifecyclePostgreSqlIntegrationTest {
assertThat(currentDatabaseThroughRouter()).isEqualTo(VALID_DATABASE);
}
@Test
void persistedConfigReplacementKeepsOldPoolOnFailureAndSwapsWithoutSecretWrite() throws Exception {
TenantConfig wrongCredentials = config(
"Неверные credentials из проекции",
jdbcUrl(OLD_DATABASE),
POSTGRES.getPassword() + "-wrong"
);
assertThatThrownBy(() -> lifecycleService.synchronizeFromPersistedConfig(List.of(wrongCredentials)))
.isInstanceOf(TenantLifecycleException.class)
.hasMessage("Не удалось подключиться к базе данных тенанта");
assertOldState();
TenantConfig badChecksum = config(
"Повреждённый Flyway из проекции",
jdbcUrl(BAD_CHECKSUM_DATABASE),
POSTGRES.getPassword()
);
assertThatThrownBy(() -> lifecycleService.synchronizeFromPersistedConfig(List.of(badChecksum)))
.isInstanceOf(TenantLifecycleException.class)
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
assertOldState();
TenantConfig validCandidate = config(
"Новый tenant из проекции",
jdbcUrl(VALID_DATABASE),
POSTGRES.getPassword()
);
try (Connection inFlightOldConnection = oldPool.getConnection()) {
lifecycleService.synchronizeFromPersistedConfig(List.of(validCandidate));
assertThat(oldPool.isClosed()).isFalse();
try (Statement statement = inFlightOldConnection.createStatement();
ResultSet result = statement.executeQuery("SELECT current_database()")) {
assertThat(result.next()).isTrue();
assertThat(result.getString(1)).isEqualTo(OLD_DATABASE);
}
}
awaitPoolClosed(oldPool);
assertThat(currentDatabaseThroughRouter()).isEqualTo(VALID_DATABASE);
verify(tenantConfigStore, never()).upsertTenant(any());
verify(tenantConfigStore, never()).removeTenant(any());
verify(tenantConfigStore, never()).compensate(any());
}
private void awaitPoolClosed(HikariDataSource pool) {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3);
while (!pool.isClosed() && System.nanoTime() < deadline) {
@@ -218,6 +264,7 @@ class TenantLifecyclePostgreSqlIntegrationTest {
private void createDatabase(String databaseName) throws Exception {
try (Connection connection = POSTGRES.createConnection("");
Statement statement = connection.createStatement()) {
statement.execute("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)");
statement.execute("CREATE DATABASE " + databaseName);
}
}

View File

@@ -11,6 +11,9 @@ import com.zaxxer.hikari.HikariDataSource;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.Mock;
@@ -20,17 +23,21 @@ import org.mockito.quality.Strictness;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -128,6 +135,25 @@ class TenantLifecycleServiceTest {
verify(candidate, never()).close();
}
@Test
void returnsConfirmedReceiptWithSuccessfulUpsert() {
TenantConfig requested = newAlpha();
TenantSecretUpdateReceipt receipt = new TenantSecretUpdateReceipt(
true,
false,
"resource-version-noop",
List.of(requested),
List.of(requested)
);
when(tenantConfigStore.upsertTenant(any())).thenReturn(receipt);
TenantLifecycleMutationResult result = service.addOrUpdateTenantWithReceipt(requested);
assertThat(result.receipt()).isSameAs(receipt);
assertConfig(result.tenant(),
"Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
}
@Test
void addsNewTenantWithoutRetiringExistingPool() {
Map<String, TenantConfig> createSnapshot = new LinkedHashMap<>();
@@ -296,6 +322,26 @@ class TenantLifecycleServiceTest {
order.verify(retiredPoolService).retire(oldDataSource);
}
@Test
void returnsLocalOnlyReceiptWithSuccessfulRemoval() {
TenantSecretUpdateReceipt receipt = new TenantSecretUpdateReceipt(
false,
false,
null,
List.of(oldAlpha),
List.of(beta)
);
when(tenantConfigStore.removeTenant("alpha")).thenReturn(receipt);
when(routingDataSource.removeTenantAtomically("alpha"))
.thenReturn(new TenantRoutingDataSource.TenantState("alpha", oldAlpha, oldDataSource));
TenantLifecycleMutationResult result = service.removeTenantWithReceipt("alpha");
assertThat(result.receipt()).isSameAs(receipt);
assertConfig(result.tenant(),
"Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "old-password");
}
@Test
void removalFailureCompensatesCommittedReceiptAndKeepsPoolOpen() {
TenantSecretUpdateReceipt receipt = removeReceipt("alpha");
@@ -314,22 +360,156 @@ class TenantLifecycleServiceTest {
@Test
void persistedTenantBecomesReadyOnlyAfterMigrationAndSwap() {
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "user", "password");
when(routingDataSource.hasTenant("gamma")).thenReturn(false);
when(routingDataSource.swapTenant(any(TenantConfig.class), any())).thenReturn(null);
service.synchronizeFromPersistedConfig(List.of(gamma));
InOrder order = inOrder(readinessRegistry, migrationService, routingDataSource);
order.verify(readinessRegistry).clearConfigurationFailure();
order.verify(readinessRegistry).replaceDesired(any());
order.verify(readinessRegistry).markMigrationStarted(any(TenantConfig.class));
order.verify(migrationService).migrate(candidate);
order.verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
order.verify(readinessRegistry).markMigrationSucceeded(any(TenantConfig.class));
order.verify(readinessRegistry).markConnectivity("gamma", true);
order.verify(readinessRegistry).clearConfigurationFailure();
verify(tenantConfigStore, never()).upsertTenant(any());
}
@ParameterizedTest
@MethodSource("changedPersistedAlphaConfigs")
void replacesExistingPersistedTenantWhenAnyNormalizedFieldChanges(TenantConfig changedConfig) {
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of("alpha", oldAlpha));
service.synchronizeFromPersistedConfig(List.of(changedConfig));
ArgumentCaptor<TenantConfig> preparedConfig = ArgumentCaptor.forClass(TenantConfig.class);
verify(routingDataSource).prepareTenantDataSource(preparedConfig.capture());
verify(migrationService).migrate(candidate);
verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
verify(retiredPoolService).retire(oldDataSource);
verify(tenantConfigStore, never()).upsertTenant(any());
verify(tenantConfigStore, never()).removeTenant(any());
assertConfig(
preparedConfig.getValue(),
changedConfig.getName().trim(),
"alpha",
changedConfig.getUrl().trim(),
changedConfig.getUsername(),
changedConfig.getPassword()
);
}
@Test
void normalizedEquivalentPersistedTenantDoesNotRecreatePool() {
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of("alpha", oldAlpha));
TenantConfig equivalent = config(
" Старый Alpha ",
" ALPHA ",
" jdbc:old-alpha ",
"old-user",
"old-password"
);
service.synchronizeFromPersistedConfig(List.of(equivalent));
verify(routingDataSource, never()).prepareTenantDataSource(any());
verify(routingDataSource, never()).swapTenant(any(), any());
verify(routingDataSource, never()).removeTenantAtomically(any());
verify(migrationService, never()).migrate(any());
verify(retiredPoolService, never()).retire(any());
verify(readinessRegistry).clearConfigurationFailure();
}
@Test
void failedPersistedReplacementKeepsOldPoolAndClosesCandidate() {
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of("alpha", oldAlpha));
org.mockito.Mockito.doThrow(new IllegalStateException("тестовая ошибка Flyway"))
.when(migrationService).migrate(candidate);
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(newAlpha())))
.isInstanceOf(TenantLifecycleException.class)
.hasMessage("Не удалось выполнить миграции базы данных тенанта");
verify(routingDataSource, never()).swapTenant(any(), any());
verify(retiredPoolService, never()).retire(oldDataSource);
verify(candidate).close();
verify(readinessRegistry).markMigrationFailed(any(TenantConfig.class));
verify(readinessRegistry, never()).clearConfigurationFailure();
verify(tenantConfigStore, never()).upsertTenant(any());
}
@Test
void retryRestoresReadinessAfterPostSwapPublicationFailureWithoutSecondSwap() {
TenantConfig changed = newAlpha();
Map<String, TenantConfig> activatedSnapshot = new LinkedHashMap<>();
activatedSnapshot.put("alpha", changed);
activatedSnapshot.put("beta", beta);
when(routingDataSource.snapshotTenantConfigs())
.thenReturn(oldSnapshot, activatedSnapshot);
TenantReadinessRegistry.TenantStatus interruptedStatus = new TenantReadinessRegistry.TenantStatus(
"alpha",
"тестовый-fingerprint",
TenantReadinessRegistry.MigrationState.IN_PROGRESS,
TenantReadinessRegistry.ConnectivityState.UNKNOWN,
null,
false,
Instant.EPOCH
);
when(readinessRegistry.snapshot()).thenReturn(Map.of("alpha", interruptedStatus));
when(routingDataSource.testConnection("alpha")).thenReturn(true);
doThrow(new IllegalStateException("тестовый отказ публикации readiness"))
.doNothing()
.when(readinessRegistry).markMigrationSucceeded(any(TenantConfig.class));
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(changed, beta)))
.isInstanceOf(IllegalStateException.class);
service.synchronizeFromPersistedConfig(List.of(changed, beta));
verify(routingDataSource).swapTenant(any(TenantConfig.class), any());
verify(migrationService).migrate(candidate);
verify(retiredPoolService).retire(oldDataSource);
verify(readinessRegistry, never()).markMigrationFailed(any(TenantConfig.class));
verify(routingDataSource).testConnection("alpha");
verify(readinessRegistry).markConnectivity("alpha", true);
verify(readinessRegistry).clearConfigurationFailure();
}
@Test
void appliesMergedPersistedSnapshotWithoutWritingSecret() {
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "gamma-user", "gamma-password");
TenantConfig newBeta = config("Новый Beta", "beta", "jdbc:new-beta", "new-beta-user", "new-beta-password");
when(routingDataSource.snapshotTenantConfigs()).thenReturn(Map.of(
"alpha", oldAlpha,
"gamma", gamma
));
when(routingDataSource.removeTenantAtomically("gamma"))
.thenReturn(new TenantRoutingDataSource.TenantState("gamma", gamma, oldDataSource));
service.synchronizeFromPersistedConfig(List.of(newAlpha(), newBeta));
verify(routingDataSource, times(2)).prepareTenantDataSource(any(TenantConfig.class));
verify(routingDataSource, times(2)).swapTenant(any(TenantConfig.class), any());
verify(routingDataSource).removeTenantAtomically("gamma");
verify(tenantConfigStore, never()).upsertTenant(any());
verify(tenantConfigStore, never()).removeTenant(any());
verify(tenantConfigStore, never()).compensate(any());
}
@Test
void rejectsDuplicateNormalizedDomainsBeforeRuntimeOrReadinessMutation() {
TenantConfig first = config("Alpha 1", "alpha", "jdbc:alpha-1", "user", "password");
TenantConfig second = config("Alpha 2", " ALPHA ", "jdbc:alpha-2", "user", "password");
assertThatThrownBy(() -> service.synchronizeFromPersistedConfig(List.of(first, second)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Конфигурация содержит повторяющиеся домены тенантов");
verify(routingDataSource, never()).snapshotTenantConfigs();
verify(routingDataSource, never()).prepareTenantDataSource(any());
verify(readinessRegistry, never()).replaceDesired(any());
}
@Test
void failedPersistedMigrationLeavesTenantNotReadyForWatcherRetry() {
TenantConfig gamma = config("Gamma", "gamma", "jdbc:gamma", "user", "password");
@@ -376,6 +556,23 @@ class TenantLifecycleServiceTest {
return config("Новый Alpha", "alpha", "jdbc:new-alpha", "new-user", "new-password");
}
private static Stream<Arguments> changedPersistedAlphaConfigs() {
return Stream.of(
Arguments.of(new TenantConfig(
"Новое имя", "alpha", "jdbc:old-alpha", "old-user", "old-password"
)),
Arguments.of(new TenantConfig(
"Старый Alpha", "alpha", "jdbc:new-alpha", "old-user", "old-password"
)),
Arguments.of(new TenantConfig(
"Старый Alpha", "alpha", "jdbc:old-alpha", "new-user", "old-password"
)),
Arguments.of(new TenantConfig(
"Старый Alpha", "alpha", "jdbc:old-alpha", "old-user", "new-password"
))
);
}
private TenantSecretUpdateReceipt upsertReceipt(TenantConfig tenant) {
Map<String, TenantConfig> committed = new LinkedHashMap<>(oldSnapshot);
committed.put(tenant.getDomain(), tenant);