001package io.jstach.rainbowgum.publisher; 002 003import java.lang.System.Logger.Level; 004import java.time.Duration; 005import java.util.Arrays; 006import java.util.Objects; 007import java.util.concurrent.TimeoutException; 008import java.util.concurrent.atomic.AtomicInteger; 009import java.util.concurrent.locks.Condition; 010import java.util.concurrent.locks.ReentrantLock; 011 012import org.jspecify.annotations.Nullable; 013 014import io.jstach.rainbowgum.LevelResolver; 015import io.jstach.rainbowgum.LogAlerts; 016import io.jstach.rainbowgum.LogAppender; 017import io.jstach.rainbowgum.LogConfig; 018import io.jstach.rainbowgum.LogEvent; 019import io.jstach.rainbowgum.LogMetrics; 020import io.jstach.rainbowgum.LogMetrics.Gauge; 021import io.jstach.rainbowgum.LogProperties; 022import io.jstach.rainbowgum.LogProperty; 023import io.jstach.rainbowgum.LogPublisher; 024import io.jstach.rainbowgum.LogPublisher.AsyncLogPublisher; 025import io.jstach.rainbowgum.LogPublisherRegistry; 026import io.jstach.rainbowgum.LogRouter; 027import io.jstach.rainbowgum.annotation.CaseChanging; 028import io.jstach.rainbowgum.annotation.LogConfigurable; 029 030/** 031 * The default bounded async publisher with one output worker. Producers block by default 032 * when the pending buffer is full. Overload options can discard lower-priority events 033 * before the buffer fills or drop incoming events when full. These options never evict 034 * accepted events or write output on the calling thread. Events are delivered in 035 * insertion order, including calls already waiting for space when shutdown begins. New 036 * calls are rejected during shutdown. Select it explicitly with the 037 * {@value LogPublisherRegistry#BATCH_SWAP_SCHEME} URI scheme to keep using it if the 038 * default async implementation changes. 039 * <p> 040 * Up to {@link BatchSwapAsyncLogPublisherBuilder#bufferSize(Integer) bufferSize} events 041 * may wait for delivery while another batch of up to that size is being written. The 042 * queued metric excludes the batch currently being written. Events passed directly to 043 * {@link #log(LogEvent)} must already be {@linkplain LogEvent#freeze() frozen}, as 044 * required by async publishers. 045 * <p> 046 * Closing waits for the configured timeout without interrupting output operations. If it 047 * times out, an error alert is recorded and the worker continues draining before closing 048 * the appender. Logging recursively from the worker is rejected to prevent deadlock. 049 * Rejected events increment {@link LogMetrics#EVENTS_DROPPED_METRIC}, with one error 050 * alert per rejection reason for each publisher. Events discarded by the overload options 051 * increment {@link LogMetrics#EVENTS_DISCARDED_METRIC} instead, with one warning alert 052 * per option for each publisher. An interrupted producer can enqueue when space is 053 * available; interruption while waiting for space drops the event and preserves the 054 * interrupt flag. 055 */ 056public final class BatchSwapAsyncLogPublisher implements AsyncLogPublisher { 057 058 private final String name; 059 060 private final LogMetrics metrics; 061 062 private final AtomicInteger alertedReasons = new AtomicInteger(); 063 064 private final LogAppender appender; 065 066 private final LogAlerts alerts; 067 068 private final Gauge queued; 069 070 private final long shutdownNanos; 071 072 private final Overflow overflow; 073 074 private final int discardAt; 075 076 private final Level discardLevel; 077 078 private final Thread worker; 079 080 private final ReentrantLock lock = new ReentrantLock(); 081 082 private final Condition readable = lock.newCondition(); 083 084 private final Condition writable = lock.newCondition(); 085 086 private final Condition terminated = lock.newCondition(); 087 088 // All mutable queue and lifecycle state is protected by lock. 089 private LogEvent[] pending; 090 091 private int count; 092 093 private int producers; 094 095 private State state = State.NEW; 096 097 private enum State { 098 099 NEW, STARTING, RUNNING, CLOSING, FAILED, CLOSED 100 101 } 102 103 // The method reference is invoked only by start(), after construction has finished. 104 @SuppressWarnings("methodref.receiver.bound") 105 private BatchSwapAsyncLogPublisher(String name, LogConfig config, LogAppender appender, int capacity, 106 long shutdownNanos, Overflow overflow, int discardThreshold, Level discardLevel) { 107 this.name = name; 108 this.metrics = config.metrics(); 109 this.appender = appender; 110 this.alerts = config.alerts(); 111 this.queued = config.metrics().gauge(LogMetrics.EVENTS_QUEUED_METRIC, Level.INFO); 112 this.pending = new LogEvent[capacity]; 113 this.shutdownNanos = shutdownNanos; 114 this.overflow = overflow; 115 this.discardAt = discardThreshold == 0 ? Integer.MAX_VALUE : capacity - discardThreshold; 116 this.discardLevel = LevelResolver.normalizeLevel(discardLevel); 117 this.worker = new Thread(this::consume, "rainbowgum-batch-swap-async-" + name); 118 this.worker.setDaemon(true); 119 } 120 121 /** 122 * Behavior when the pending buffer is full. 123 */ 124 @CaseChanging 125 public enum Overflow { 126 127 /** 128 * Wait for space, preserving accepted events through shutdown. 129 */ 130 BLOCK, 131 /** 132 * Discard the incoming event regardless of level instead of waiting for buffer 133 * space. 134 */ 135 DROP; 136 137 } 138 139 /** 140 * Default full-buffer policy. 141 */ 142 public static final Overflow DEFAULT_OVERFLOW = Overflow.BLOCK; 143 144 /** 145 * Early discarding is disabled by default. 146 */ 147 public static final int DEFAULT_DISCARD_THRESHOLD = 0; 148 149 /** 150 * Highest severity eligible for early discarding. 151 */ 152 public static final Level DEFAULT_DISCARD_LEVEL = Level.INFO; 153 154 /** 155 * Default pending buffer capacity. 156 */ 157 public static final int DEFAULT_BUFFER_SIZE = LogPublisherRegistry.ASYNC_BUFFER_SIZE; 158 159 /** 160 * Default time to wait for delivery and appender closure. 161 */ 162 public static final Duration DEFAULT_SHUTDOWN_TIMEOUT = Duration 163 .ofMillis(LogPublisherRegistry.ASYNC_SHUTDOWN_TIMEOUT); 164 165 /** 166 * Creates a named builder. Call 167 * {@link BatchSwapAsyncLogPublisherBuilder#fromProperties(LogProperties)} to read 168 * {@code logging.route.<name>.publisher.batchswap.bufferSize} and 169 * {@code logging.route.<name>.publisher.batchswap.shutdownTimeout}. Properties 170 * override builder values when present. Select its factory with 171 * {@link LogRouter.Router.Builder#publisher(LogPublisher.PublisherFactory)}. 172 * @param name publisher name used for property lookup. 173 * @return builder. 174 */ 175 public static BatchSwapAsyncLogPublisherBuilder builder(String name) { 176 return new BatchSwapAsyncLogPublisherBuilder(name, LogPublisherRegistry.BATCH_SWAP_SCHEME); 177 } 178 179 /** 180 * Creates a publisher factory with validated options. 181 * @param name publisher name used for property lookup. 182 * @param scheme URI scheme the publisher was selected with, used for property lookup. 183 * @param bufferSize positive pending capacity, excluding the batch being written. 184 * @param shutdownTimeout nonnegative close timeout (property unit: milliseconds). 185 * @param overflow behavior when full: block (default) or drop the incoming event. 186 * @param discardThreshold slots reserved for events above discardLevel. Zero disables 187 * early discarding; otherwise discard when remaining slots are at or below this 188 * number. Must be zero or more and less than bufferSize. 189 * @param discardLevel highest severity discarded early (default INFO). OFF disables 190 * early discarding. ALL is normalized to TRACE. 191 * @return publisher factory. 192 */ 193 @LogConfigurable(name = "BatchSwapAsyncLogPublisherBuilder", 194 prefix = LogProperties.PUBLISHER_PREFIX + "{" + LogPublisherRegistry.SCHEME + "}.") 195 static PublisherFactory of(@LogConfigurable.KeyParameter String name, @LogConfigurable.KeyParameter String scheme, 196 @LogConfigurable.DefaultParameter("DEFAULT_BUFFER_SIZE") Integer bufferSize, 197 @LogConfigurable.DefaultParameter("DEFAULT_SHUTDOWN_TIMEOUT") @LogConfigurable.ConvertParameter("parseShutdownTimeout") Duration shutdownTimeout, 198 @LogConfigurable.DefaultParameter("DEFAULT_OVERFLOW") @LogConfigurable.ConvertParameter("parseOverflow") Overflow overflow, 199 @LogConfigurable.DefaultParameter("DEFAULT_DISCARD_THRESHOLD") Integer discardThreshold, 200 @LogConfigurable.DefaultParameter("DEFAULT_DISCARD_LEVEL") @LogConfigurable.ConvertParameter("parseDiscardLevel") Level discardLevel) { 201 if (bufferSize < 1) { 202 throw new IllegalArgumentException("bufferSize must be positive."); 203 } 204 if (discardThreshold < 0 || discardThreshold >= bufferSize) { 205 throw new IllegalArgumentException("discardThreshold must be zero or more and less than bufferSize."); 206 } 207 if (shutdownTimeout.isNegative()) { 208 throw new IllegalArgumentException("shutdownTimeout must not be negative."); 209 } 210 long shutdownNanos; 211 try { 212 shutdownNanos = shutdownTimeout.toNanos(); 213 } 214 catch (ArithmeticException e) { 215 throw new IllegalArgumentException("shutdownTimeout is too large.", e); 216 } 217 return (n, config, appenders) -> new BatchSwapAsyncLogPublisher(n, config, appenders.asSingle(), bufferSize, 218 shutdownNanos, overflow, discardThreshold, discardLevel); 219 } 220 221 static Overflow parseOverflow(String value) { 222 return LogProperty.enumValue(Overflow.class, value); 223 } 224 225 static Level parseDiscardLevel(String value) { 226 return LevelResolver.parseLevel(value); 227 } 228 229 static Duration parseShutdownTimeout(String value) { 230 return Duration.ofMillis(Long.parseLong(value)); 231 } 232 233 @Override 234 public void start(LogConfig config) { 235 lock.lock(); 236 try { 237 if (state == State.RUNNING) { 238 return; 239 } 240 if (state != State.NEW) { 241 throw new IllegalStateException("BatchSwapAsyncLogPublisher cannot be restarted."); 242 } 243 state = State.STARTING; 244 try { 245 appender.start(config); 246 state = State.RUNNING; 247 worker.start(); 248 } 249 catch (RuntimeException | Error e) { 250 state = State.CLOSED; 251 try { 252 appender.close(); 253 } 254 catch (RuntimeException | Error closeFailure) { 255 e.addSuppressed(closeFailure); 256 } 257 throw e; 258 } 259 } 260 finally { 261 lock.unlock(); 262 } 263 } 264 265 @Override 266 public void log(LogEvent event) { 267 Objects.requireNonNull(event); 268 @Nullable Drop reason; 269 if (Thread.currentThread().equals(worker)) { 270 reason = Drop.REENTRANT; 271 } 272 else { 273 lock.lock(); 274 try { 275 reason = enqueueOrNull(event); 276 } 277 finally { 278 lock.unlock(); 279 } 280 } 281 // Alert listeners can themselves log. Never call them under the queue lock. 282 if (reason != null) { 283 dropped(reason, 1); 284 } 285 } 286 287 // Called with lock held. Null means the event was accepted. 288 private @Nullable Drop enqueueOrNull(LogEvent event) { 289 if (state != State.RUNNING) { 290 return switch (state) { 291 case NEW, STARTING -> Drop.NOT_STARTED; 292 case FAILED -> Drop.WORKER_FAILED; 293 default -> Drop.CLOSED; 294 }; 295 } 296 if (count >= discardAt && discardLevel != Level.OFF 297 && event.level().getSeverity() <= discardLevel.getSeverity()) { 298 return Drop.DISCARD_THRESHOLD; 299 } 300 if (count == pending.length && overflow == Overflow.DROP) { 301 return Drop.BUFFER_FULL; 302 } 303 producers++; 304 try { 305 while (count == pending.length && state != State.FAILED && state != State.CLOSED) { 306 writable.await(); 307 } 308 if (state == State.FAILED || state == State.CLOSED) { 309 return Drop.WORKER_FAILED; 310 } 311 pending[count++] = event; 312 queued.increment(); 313 if (count == 1) { 314 readable.signal(); 315 } 316 return null; 317 } 318 catch (InterruptedException e) { 319 Thread.currentThread().interrupt(); 320 return Drop.INTERRUPTED; 321 } 322 finally { 323 producers--; 324 // Pass available space to one waiter, including when this caller 325 // was interrupted. Avoid waking a herd that races to refill a batch. 326 if (producers != 0 && count < pending.length) { 327 writable.signal(); 328 } 329 if (state == State.CLOSING && producers == 0) { 330 readable.signal(); 331 } 332 } 333 } 334 335 private enum Drop { 336 337 NOT_STARTED(1 << 0, "the publisher has not been started"), CLOSED(1 << 1, "the publisher is closing or closed"), 338 INTERRUPTED(1 << 2, "the logging thread was interrupted while the buffer was full"), 339 REENTRANT(1 << 3, "the publisher's own thread logged recursively"), 340 WORKER_FAILED(1 << 4, "the worker stopped before delivering the event"), 341 // Configured by the application, so warnings counted separately from failures. 342 BUFFER_FULL(1 << 5, "the pending buffer was full and overflow is drop", true), DISCARD_THRESHOLD(1 << 6, 343 "the remaining buffer reached discardThreshold and the event level is at or below discardLevel", true); 344 345 final String description; 346 347 final int bit; 348 349 final boolean configured; 350 351 Drop(int bit, String description) { 352 this(bit, description, false); 353 } 354 355 Drop(int bit, String description, boolean configured) { 356 this.description = description; 357 this.bit = bit; 358 this.configured = configured; 359 } 360 361 } 362 363 private void dropped(Drop reason, int size) { 364 String metric = reason.configured ? LogMetrics.EVENTS_DISCARDED_METRIC : LogMetrics.EVENTS_DROPPED_METRIC; 365 if (reason.configured) { 366 metrics.warnCounter(metric, size); 367 } 368 else { 369 metrics.errorCounter(metric, size); 370 } 371 int previous; 372 do { 373 previous = alertedReasons.get(); 374 if ((previous & reason.bit) != 0) { 375 return; 376 } 377 } 378 while (!alertedReasons.compareAndSet(previous, previous | reason.bit)); 379 if (reason.configured) { 380 alerts.warn(BatchSwapAsyncLogPublisher.class, 381 "Async publisher '" + name + "' discarded " + size + " event(s): " + reason.description 382 + ". Further discards for this reason are only counted in " + metric + "."); 383 return; 384 } 385 alerts.error(BatchSwapAsyncLogPublisher.class, 386 new IllegalStateException("Async publisher '" + name + "' dropped " + size + " event(s): " 387 + reason.description + ". Further drops for this reason are only counted in " + metric + ".")); 388 } 389 390 private void consume() { 391 @Nullable Throwable failure = null; 392 try { 393 LogEvent[] batch = new LogEvent[pending.length]; 394 while (true) { 395 int size; 396 lock.lock(); 397 try { 398 while (count == 0) { 399 if (state == State.CLOSING && producers == 0) { 400 return; 401 } 402 readable.awaitUninterruptibly(); 403 } 404 // Exchange ownership, without copying event references under the 405 // lock. 406 LogEvent[] empty = batch; 407 batch = pending; 408 pending = empty; 409 size = count; 410 count = 0; 411 queued.decrement(size); 412 writable.signal(); 413 } 414 finally { 415 lock.unlock(); 416 } 417 try { 418 appender.append(batch, size); 419 } 420 catch (RuntimeException | Error e) { 421 metrics.errorCounter(LogMetrics.EVENTS_FAILED_METRIC, size); 422 throw e; 423 } 424 finally { 425 Arrays.fill(batch, 0, size, null); 426 } 427 } 428 } 429 catch (RuntimeException | Error e) { 430 failure = e; 431 } 432 finally { 433 finish(failure); 434 } 435 } 436 437 private void finish(@Nullable Throwable failure) { 438 int discarded; 439 lock.lock(); 440 try { 441 state = failure == null ? State.CLOSING : State.FAILED; 442 discarded = count; 443 queued.decrement(count); 444 Arrays.fill(pending, null); 445 count = 0; 446 writable.signalAll(); 447 } 448 finally { 449 lock.unlock(); 450 } 451 try { 452 // Stop admission and release waiters before notifying listeners. 453 if (failure != null) { 454 alerts.error(BatchSwapAsyncLogPublisher.class, "Async worker failed.", failure); 455 } 456 if (discarded != 0) { 457 dropped(Drop.WORKER_FAILED, discarded); 458 } 459 } 460 finally { 461 closeAppender(); 462 } 463 } 464 465 private void closeAppender() { 466 try { 467 appender.close(); 468 } 469 catch (RuntimeException | Error e) { 470 alerts.error(BatchSwapAsyncLogPublisher.class, "Failed to close async appender.", e); 471 } 472 finally { 473 lock.lock(); 474 try { 475 state = State.CLOSED; 476 terminated.signalAll(); 477 } 478 finally { 479 lock.unlock(); 480 } 481 } 482 } 483 484 @Override 485 public void close() { 486 boolean interrupted = Thread.interrupted(); 487 boolean closeUnstarted = false; 488 boolean timedOut = false; 489 @Nullable InterruptedException interruptedWait = null; 490 lock.lock(); 491 try { 492 if (state == State.CLOSED) { 493 return; 494 } 495 if (state == State.STARTING) { 496 throw new IllegalStateException("Cannot close BatchSwapAsyncLogPublisher during appender startup."); 497 } 498 if (state == State.NEW) { 499 state = State.CLOSING; 500 closeUnstarted = true; 501 } 502 else { 503 if (state == State.RUNNING) { 504 state = State.CLOSING; 505 readable.signal(); 506 } 507 if (Thread.currentThread().equals(worker)) { 508 return; 509 } 510 long remaining = shutdownNanos; 511 while (state != State.CLOSED && remaining > 0) { 512 try { 513 remaining = terminated.awaitNanos(remaining); 514 } 515 catch (InterruptedException e) { 516 interrupted = true; 517 interruptedWait = e; 518 break; 519 } 520 } 521 timedOut = state != State.CLOSED; 522 } 523 } 524 finally { 525 lock.unlock(); 526 if (interrupted) { 527 Thread.currentThread().interrupt(); 528 } 529 } 530 if (closeUnstarted) { 531 finish(null); 532 } 533 else if (interruptedWait != null) { 534 alerts.error(BatchSwapAsyncLogPublisher.class, "Interrupted while waiting for async publisher shutdown.", 535 interruptedWait); 536 } 537 else if (timedOut) { 538 alerts.error(BatchSwapAsyncLogPublisher.class, "Async publisher is still draining after close.", 539 new TimeoutException("Shutdown wait ended before the appender closed.")); 540 } 541 } 542 543}