001package io.jstach.rainbowgum.publisher;
002
003import java.lang.System.Logger.Level;
004import java.time.Duration;
005import java.util.Arrays;
006import java.util.Objects;
007import java.util.concurrent.TimeoutException;
008import java.util.concurrent.atomic.AtomicInteger;
009import java.util.concurrent.locks.Condition;
010import java.util.concurrent.locks.ReentrantLock;
011
012import org.jspecify.annotations.Nullable;
013
014import io.jstach.rainbowgum.LevelResolver;
015import io.jstach.rainbowgum.LogAlerts;
016import io.jstach.rainbowgum.LogAppender;
017import io.jstach.rainbowgum.LogConfig;
018import io.jstach.rainbowgum.LogEvent;
019import io.jstach.rainbowgum.LogMetrics;
020import io.jstach.rainbowgum.LogMetrics.Gauge;
021import io.jstach.rainbowgum.LogProperties;
022import io.jstach.rainbowgum.LogProperty;
023import io.jstach.rainbowgum.LogPublisher;
024import io.jstach.rainbowgum.LogPublisher.AsyncLogPublisher;
025import io.jstach.rainbowgum.LogPublisherRegistry;
026import io.jstach.rainbowgum.LogRouter;
027import io.jstach.rainbowgum.annotation.CaseChanging;
028import io.jstach.rainbowgum.annotation.LogConfigurable;
029
030/**
031 * The default bounded async publisher with one output worker. Producers block by default
032 * when the pending buffer is full. Overload options can discard lower-priority events
033 * before the buffer fills or drop incoming events when full. These options never evict
034 * accepted events or write output on the calling thread. Events are delivered in
035 * insertion order, including calls already waiting for space when shutdown begins. New
036 * calls are rejected during shutdown. Select it explicitly with the
037 * {@value LogPublisherRegistry#BATCH_SWAP_SCHEME} URI scheme to keep using it if the
038 * default async implementation changes.
039 * <p>
040 * Up to {@link BatchSwapAsyncLogPublisherBuilder#bufferSize(Integer) bufferSize} events
041 * may wait for delivery while another batch of up to that size is being written. The
042 * queued metric excludes the batch currently being written. Events passed directly to
043 * {@link #log(LogEvent)} must already be {@linkplain LogEvent#freeze() frozen}, as
044 * required by async publishers.
045 * <p>
046 * Closing waits for the configured timeout without interrupting output operations. If it
047 * times out, an error alert is recorded and the worker continues draining before closing
048 * the appender. Logging recursively from the worker is rejected to prevent deadlock.
049 * Rejected events increment {@link LogMetrics#EVENTS_DROPPED_METRIC}, with one error
050 * alert per rejection reason for each publisher. Events discarded by the overload options
051 * increment {@link LogMetrics#EVENTS_DISCARDED_METRIC} instead, with one warning alert
052 * per option for each publisher. An interrupted producer can enqueue when space is
053 * available; interruption while waiting for space drops the event and preserves the
054 * interrupt flag.
055 */
056public final class BatchSwapAsyncLogPublisher implements AsyncLogPublisher {
057
058        private final String name;
059
060        private final LogMetrics metrics;
061
062        private final AtomicInteger alertedReasons = new AtomicInteger();
063
064        private final LogAppender appender;
065
066        private final LogAlerts alerts;
067
068        private final Gauge queued;
069
070        private final long shutdownNanos;
071
072        private final Overflow overflow;
073
074        private final int discardAt;
075
076        private final Level discardLevel;
077
078        private final Thread worker;
079
080        private final ReentrantLock lock = new ReentrantLock();
081
082        private final Condition readable = lock.newCondition();
083
084        private final Condition writable = lock.newCondition();
085
086        private final Condition terminated = lock.newCondition();
087
088        // All mutable queue and lifecycle state is protected by lock.
089        private LogEvent[] pending;
090
091        private int count;
092
093        private int producers;
094
095        private State state = State.NEW;
096
097        private enum State {
098
099                NEW, STARTING, RUNNING, CLOSING, FAILED, CLOSED
100
101        }
102
103        // The method reference is invoked only by start(), after construction has finished.
104        @SuppressWarnings("methodref.receiver.bound")
105        private BatchSwapAsyncLogPublisher(String name, LogConfig config, LogAppender appender, int capacity,
106                        long shutdownNanos, Overflow overflow, int discardThreshold, Level discardLevel) {
107                this.name = name;
108                this.metrics = config.metrics();
109                this.appender = appender;
110                this.alerts = config.alerts();
111                this.queued = config.metrics().gauge(LogMetrics.EVENTS_QUEUED_METRIC, Level.INFO);
112                this.pending = new LogEvent[capacity];
113                this.shutdownNanos = shutdownNanos;
114                this.overflow = overflow;
115                this.discardAt = discardThreshold == 0 ? Integer.MAX_VALUE : capacity - discardThreshold;
116                this.discardLevel = LevelResolver.normalizeLevel(discardLevel);
117                this.worker = new Thread(this::consume, "rainbowgum-batch-swap-async-" + name);
118                this.worker.setDaemon(true);
119        }
120
121        /**
122         * Behavior when the pending buffer is full.
123         */
124        @CaseChanging
125        public enum Overflow {
126
127                /**
128                 * Wait for space, preserving accepted events through shutdown.
129                 */
130                BLOCK,
131                /**
132                 * Discard the incoming event regardless of level instead of waiting for buffer
133                 * space.
134                 */
135                DROP;
136
137        }
138
139        /**
140         * Default full-buffer policy.
141         */
142        public static final Overflow DEFAULT_OVERFLOW = Overflow.BLOCK;
143
144        /**
145         * Early discarding is disabled by default.
146         */
147        public static final int DEFAULT_DISCARD_THRESHOLD = 0;
148
149        /**
150         * Highest severity eligible for early discarding.
151         */
152        public static final Level DEFAULT_DISCARD_LEVEL = Level.INFO;
153
154        /**
155         * Default pending buffer capacity.
156         */
157        public static final int DEFAULT_BUFFER_SIZE = LogPublisherRegistry.ASYNC_BUFFER_SIZE;
158
159        /**
160         * Default time to wait for delivery and appender closure.
161         */
162        public static final Duration DEFAULT_SHUTDOWN_TIMEOUT = Duration
163                .ofMillis(LogPublisherRegistry.ASYNC_SHUTDOWN_TIMEOUT);
164
165        /**
166         * Creates a named builder. Call
167         * {@link BatchSwapAsyncLogPublisherBuilder#fromProperties(LogProperties)} to read
168         * {@code logging.route.<name>.publisher.batchswap.bufferSize} and
169         * {@code logging.route.<name>.publisher.batchswap.shutdownTimeout}. Properties
170         * override builder values when present. Select its factory with
171         * {@link LogRouter.Router.Builder#publisher(LogPublisher.PublisherFactory)}.
172         * @param name publisher name used for property lookup.
173         * @return builder.
174         */
175        public static BatchSwapAsyncLogPublisherBuilder builder(String name) {
176                return new BatchSwapAsyncLogPublisherBuilder(name, LogPublisherRegistry.BATCH_SWAP_SCHEME);
177        }
178
179        /**
180         * Creates a publisher factory with validated options.
181         * @param name publisher name used for property lookup.
182         * @param scheme URI scheme the publisher was selected with, used for property lookup.
183         * @param bufferSize positive pending capacity, excluding the batch being written.
184         * @param shutdownTimeout nonnegative close timeout (property unit: milliseconds).
185         * @param overflow behavior when full: block (default) or drop the incoming event.
186         * @param discardThreshold slots reserved for events above discardLevel. Zero disables
187         * early discarding; otherwise discard when remaining slots are at or below this
188         * number. Must be zero or more and less than bufferSize.
189         * @param discardLevel highest severity discarded early (default INFO). OFF disables
190         * early discarding. ALL is normalized to TRACE.
191         * @return publisher factory.
192         */
193        @LogConfigurable(name = "BatchSwapAsyncLogPublisherBuilder",
194                        prefix = LogProperties.PUBLISHER_PREFIX + "{" + LogPublisherRegistry.SCHEME + "}.")
195        static PublisherFactory of(@LogConfigurable.KeyParameter String name, @LogConfigurable.KeyParameter String scheme,
196                        @LogConfigurable.DefaultParameter("DEFAULT_BUFFER_SIZE") Integer bufferSize,
197                        @LogConfigurable.DefaultParameter("DEFAULT_SHUTDOWN_TIMEOUT") @LogConfigurable.ConvertParameter("parseShutdownTimeout") Duration shutdownTimeout,
198                        @LogConfigurable.DefaultParameter("DEFAULT_OVERFLOW") @LogConfigurable.ConvertParameter("parseOverflow") Overflow overflow,
199                        @LogConfigurable.DefaultParameter("DEFAULT_DISCARD_THRESHOLD") Integer discardThreshold,
200                        @LogConfigurable.DefaultParameter("DEFAULT_DISCARD_LEVEL") @LogConfigurable.ConvertParameter("parseDiscardLevel") Level discardLevel) {
201                if (bufferSize < 1) {
202                        throw new IllegalArgumentException("bufferSize must be positive.");
203                }
204                if (discardThreshold < 0 || discardThreshold >= bufferSize) {
205                        throw new IllegalArgumentException("discardThreshold must be zero or more and less than bufferSize.");
206                }
207                if (shutdownTimeout.isNegative()) {
208                        throw new IllegalArgumentException("shutdownTimeout must not be negative.");
209                }
210                long shutdownNanos;
211                try {
212                        shutdownNanos = shutdownTimeout.toNanos();
213                }
214                catch (ArithmeticException e) {
215                        throw new IllegalArgumentException("shutdownTimeout is too large.", e);
216                }
217                return (n, config, appenders) -> new BatchSwapAsyncLogPublisher(n, config, appenders.asSingle(), bufferSize,
218                                shutdownNanos, overflow, discardThreshold, discardLevel);
219        }
220
221        static Overflow parseOverflow(String value) {
222                return LogProperty.enumValue(Overflow.class, value);
223        }
224
225        static Level parseDiscardLevel(String value) {
226                return LevelResolver.parseLevel(value);
227        }
228
229        static Duration parseShutdownTimeout(String value) {
230                return Duration.ofMillis(Long.parseLong(value));
231        }
232
233        @Override
234        public void start(LogConfig config) {
235                lock.lock();
236                try {
237                        if (state == State.RUNNING) {
238                                return;
239                        }
240                        if (state != State.NEW) {
241                                throw new IllegalStateException("BatchSwapAsyncLogPublisher cannot be restarted.");
242                        }
243                        state = State.STARTING;
244                        try {
245                                appender.start(config);
246                                state = State.RUNNING;
247                                worker.start();
248                        }
249                        catch (RuntimeException | Error e) {
250                                state = State.CLOSED;
251                                try {
252                                        appender.close();
253                                }
254                                catch (RuntimeException | Error closeFailure) {
255                                        e.addSuppressed(closeFailure);
256                                }
257                                throw e;
258                        }
259                }
260                finally {
261                        lock.unlock();
262                }
263        }
264
265        @Override
266        public void log(LogEvent event) {
267                Objects.requireNonNull(event);
268                @Nullable Drop reason;
269                if (Thread.currentThread().equals(worker)) {
270                        reason = Drop.REENTRANT;
271                }
272                else {
273                        lock.lock();
274                        try {
275                                reason = enqueueOrNull(event);
276                        }
277                        finally {
278                                lock.unlock();
279                        }
280                }
281                // Alert listeners can themselves log. Never call them under the queue lock.
282                if (reason != null) {
283                        dropped(reason, 1);
284                }
285        }
286
287        // Called with lock held. Null means the event was accepted.
288        private @Nullable Drop enqueueOrNull(LogEvent event) {
289                if (state != State.RUNNING) {
290                        return switch (state) {
291                                case NEW, STARTING -> Drop.NOT_STARTED;
292                                case FAILED -> Drop.WORKER_FAILED;
293                                default -> Drop.CLOSED;
294                        };
295                }
296                if (count >= discardAt && discardLevel != Level.OFF
297                                && event.level().getSeverity() <= discardLevel.getSeverity()) {
298                        return Drop.DISCARD_THRESHOLD;
299                }
300                if (count == pending.length && overflow == Overflow.DROP) {
301                        return Drop.BUFFER_FULL;
302                }
303                producers++;
304                try {
305                        while (count == pending.length && state != State.FAILED && state != State.CLOSED) {
306                                writable.await();
307                        }
308                        if (state == State.FAILED || state == State.CLOSED) {
309                                return Drop.WORKER_FAILED;
310                        }
311                        pending[count++] = event;
312                        queued.increment();
313                        if (count == 1) {
314                                readable.signal();
315                        }
316                        return null;
317                }
318                catch (InterruptedException e) {
319                        Thread.currentThread().interrupt();
320                        return Drop.INTERRUPTED;
321                }
322                finally {
323                        producers--;
324                        // Pass available space to one waiter, including when this caller
325                        // was interrupted. Avoid waking a herd that races to refill a batch.
326                        if (producers != 0 && count < pending.length) {
327                                writable.signal();
328                        }
329                        if (state == State.CLOSING && producers == 0) {
330                                readable.signal();
331                        }
332                }
333        }
334
335        private enum Drop {
336
337                NOT_STARTED(1 << 0, "the publisher has not been started"), CLOSED(1 << 1, "the publisher is closing or closed"),
338                INTERRUPTED(1 << 2, "the logging thread was interrupted while the buffer was full"),
339                REENTRANT(1 << 3, "the publisher's own thread logged recursively"),
340                WORKER_FAILED(1 << 4, "the worker stopped before delivering the event"),
341                // Configured by the application, so warnings counted separately from failures.
342                BUFFER_FULL(1 << 5, "the pending buffer was full and overflow is drop", true), DISCARD_THRESHOLD(1 << 6,
343                                "the remaining buffer reached discardThreshold and the event level is at or below discardLevel", true);
344
345                final String description;
346
347                final int bit;
348
349                final boolean configured;
350
351                Drop(int bit, String description) {
352                        this(bit, description, false);
353                }
354
355                Drop(int bit, String description, boolean configured) {
356                        this.description = description;
357                        this.bit = bit;
358                        this.configured = configured;
359                }
360
361        }
362
363        private void dropped(Drop reason, int size) {
364                String metric = reason.configured ? LogMetrics.EVENTS_DISCARDED_METRIC : LogMetrics.EVENTS_DROPPED_METRIC;
365                if (reason.configured) {
366                        metrics.warnCounter(metric, size);
367                }
368                else {
369                        metrics.errorCounter(metric, size);
370                }
371                int previous;
372                do {
373                        previous = alertedReasons.get();
374                        if ((previous & reason.bit) != 0) {
375                                return;
376                        }
377                }
378                while (!alertedReasons.compareAndSet(previous, previous | reason.bit));
379                if (reason.configured) {
380                        alerts.warn(BatchSwapAsyncLogPublisher.class,
381                                        "Async publisher '" + name + "' discarded " + size + " event(s): " + reason.description
382                                                        + ". Further discards for this reason are only counted in " + metric + ".");
383                        return;
384                }
385                alerts.error(BatchSwapAsyncLogPublisher.class,
386                                new IllegalStateException("Async publisher '" + name + "' dropped " + size + " event(s): "
387                                                + reason.description + ". Further drops for this reason are only counted in " + metric + "."));
388        }
389
390        private void consume() {
391                @Nullable Throwable failure = null;
392                try {
393                        LogEvent[] batch = new LogEvent[pending.length];
394                        while (true) {
395                                int size;
396                                lock.lock();
397                                try {
398                                        while (count == 0) {
399                                                if (state == State.CLOSING && producers == 0) {
400                                                        return;
401                                                }
402                                                readable.awaitUninterruptibly();
403                                        }
404                                        // Exchange ownership, without copying event references under the
405                                        // lock.
406                                        LogEvent[] empty = batch;
407                                        batch = pending;
408                                        pending = empty;
409                                        size = count;
410                                        count = 0;
411                                        queued.decrement(size);
412                                        writable.signal();
413                                }
414                                finally {
415                                        lock.unlock();
416                                }
417                                try {
418                                        appender.append(batch, size);
419                                }
420                                catch (RuntimeException | Error e) {
421                                        metrics.errorCounter(LogMetrics.EVENTS_FAILED_METRIC, size);
422                                        throw e;
423                                }
424                                finally {
425                                        Arrays.fill(batch, 0, size, null);
426                                }
427                        }
428                }
429                catch (RuntimeException | Error e) {
430                        failure = e;
431                }
432                finally {
433                        finish(failure);
434                }
435        }
436
437        private void finish(@Nullable Throwable failure) {
438                int discarded;
439                lock.lock();
440                try {
441                        state = failure == null ? State.CLOSING : State.FAILED;
442                        discarded = count;
443                        queued.decrement(count);
444                        Arrays.fill(pending, null);
445                        count = 0;
446                        writable.signalAll();
447                }
448                finally {
449                        lock.unlock();
450                }
451                try {
452                        // Stop admission and release waiters before notifying listeners.
453                        if (failure != null) {
454                                alerts.error(BatchSwapAsyncLogPublisher.class, "Async worker failed.", failure);
455                        }
456                        if (discarded != 0) {
457                                dropped(Drop.WORKER_FAILED, discarded);
458                        }
459                }
460                finally {
461                        closeAppender();
462                }
463        }
464
465        private void closeAppender() {
466                try {
467                        appender.close();
468                }
469                catch (RuntimeException | Error e) {
470                        alerts.error(BatchSwapAsyncLogPublisher.class, "Failed to close async appender.", e);
471                }
472                finally {
473                        lock.lock();
474                        try {
475                                state = State.CLOSED;
476                                terminated.signalAll();
477                        }
478                        finally {
479                                lock.unlock();
480                        }
481                }
482        }
483
484        @Override
485        public void close() {
486                boolean interrupted = Thread.interrupted();
487                boolean closeUnstarted = false;
488                boolean timedOut = false;
489                @Nullable InterruptedException interruptedWait = null;
490                lock.lock();
491                try {
492                        if (state == State.CLOSED) {
493                                return;
494                        }
495                        if (state == State.STARTING) {
496                                throw new IllegalStateException("Cannot close BatchSwapAsyncLogPublisher during appender startup.");
497                        }
498                        if (state == State.NEW) {
499                                state = State.CLOSING;
500                                closeUnstarted = true;
501                        }
502                        else {
503                                if (state == State.RUNNING) {
504                                        state = State.CLOSING;
505                                        readable.signal();
506                                }
507                                if (Thread.currentThread().equals(worker)) {
508                                        return;
509                                }
510                                long remaining = shutdownNanos;
511                                while (state != State.CLOSED && remaining > 0) {
512                                        try {
513                                                remaining = terminated.awaitNanos(remaining);
514                                        }
515                                        catch (InterruptedException e) {
516                                                interrupted = true;
517                                                interruptedWait = e;
518                                                break;
519                                        }
520                                }
521                                timedOut = state != State.CLOSED;
522                        }
523                }
524                finally {
525                        lock.unlock();
526                        if (interrupted) {
527                                Thread.currentThread().interrupt();
528                        }
529                }
530                if (closeUnstarted) {
531                        finish(null);
532                }
533                else if (interruptedWait != null) {
534                        alerts.error(BatchSwapAsyncLogPublisher.class, "Interrupted while waiting for async publisher shutdown.",
535                                        interruptedWait);
536                }
537                else if (timedOut) {
538                        alerts.error(BatchSwapAsyncLogPublisher.class, "Async publisher is still draining after close.",
539                                        new TimeoutException("Shutdown wait ended before the appender closed."));
540                }
541        }
542
543}