Skip to content

Revert "Rotate access log if larger than configured limit" #35021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bjormel wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Revert "Rotate access log if larger than configured limit" #35021

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class AccessLogHandler {
AccessLogHandler(AccessLogConfig.FileHandler config, LogWriter<RequestLogEntry> logWriter) {
logFileHandler = new LogFileHandler<>(
toCompression(config), config.bufferSize(), config.pattern(), config.rotation(),
config.symlink(), queueSize(config), config.rotationSize(), "request-logger", logWriter);
config.symlink(), queueSize(config), "request-logger", logWriter);
}

private static int queueSize(AccessLogConfig.FileHandler config) {
Expand Down
84 changes: 10 additions & 74 deletions container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Optional;
import java.util.concurrent.BlockingQueue;
Expand Down Expand Up @@ -50,40 +47,18 @@ enum Compression {NONE, GZIP, ZSTD}

@FunctionalInterface private interface Pollable<T> { Operation<T> poll() throws InterruptedException; }

LogFileHandler(Compression compression, int bufferSize, String filePattern, String rotationTimes, String symlinkName,
int queueSize, long rotationSize, String threadName, LogWriter<LOGTYPE> logWriter, Clock clock) {
this(compression, bufferSize, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, rotationSize, threadName, logWriter, clock);
}

LogFileHandler(Compression compression, int bufferSize, String filePattern, long[] rotationTimes, String symlinkName,
int queueSize, long rotationSize, String threadName, LogWriter<LOGTYPE> logWriter, Clock clock) {
this.logQueue = new LinkedBlockingQueue<>(queueSize);
this.logThread = new LogThread<>(logWriter, filePattern, compression, bufferSize, rotationTimes, symlinkName, rotationSize, threadName, this::poll, clock);
this.logThread.start();
}

LogFileHandler(Compression compression, int bufferSize, String filePattern, String rotationTimes, String symlinkName,
int queueSize, long rotationSize, String threadName, LogWriter<LOGTYPE> logWriter) {
this(compression, bufferSize, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, rotationSize, threadName, logWriter, Clock.systemUTC());
}

LogFileHandler(Compression compression, int bufferSize, String filePattern, long[] rotationTimes, String symlinkName,
int queueSize, long rotationSize, String threadName, LogWriter<LOGTYPE> logWriter) {
this(compression, bufferSize, filePattern, rotationTimes, symlinkName, queueSize, rotationSize, threadName, logWriter, Clock.systemUTC());
}

// Keep backward compatibility constructors
LogFileHandler(Compression compression, int bufferSize, String filePattern, String rotationTimes, String symlinkName,
int queueSize, String threadName, LogWriter<LOGTYPE> logWriter) {
this(compression, bufferSize, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, 0, threadName, logWriter, Clock.systemUTC());
this(compression, bufferSize, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, threadName, logWriter);
}

LogFileHandler(Compression compression, int bufferSize, String filePattern, long[] rotationTimes, String symlinkName,
int queueSize, String threadName, LogWriter<LOGTYPE> logWriter) {
this(compression, bufferSize, filePattern, rotationTimes, symlinkName, queueSize, 0, threadName, logWriter, Clock.systemUTC());
this.logQueue = new LinkedBlockingQueue<>(queueSize);
this.logThread = new LogThread<>(logWriter, filePattern, compression, bufferSize, rotationTimes, symlinkName, threadName, this::poll);
this.logThread.start();
}


private Operation<LOGTYPE> poll() throws InterruptedException {
return logQueue.poll(100, TimeUnit.MILLISECONDS);
}
Expand Down Expand Up @@ -212,10 +187,6 @@ static class LogThread<LOGTYPE> extends Thread {
long lastFlush = 0;
private PageCacheFriendlyFileOutputStream fileOutput = null;
private long nextRotationTime = 0;
private long fileSize = 0;
private final Duration fileSizeCheckInterval = Duration.ofMinutes(1);
private final Clock clock;
private Instant lastFileSizeCheck = Instant.now();
private final String filePattern; // default to current directory, ms time stamp
private volatile String fileName;
private final LogWriter<LOGTYPE> logWriter;
Expand All @@ -225,18 +196,16 @@ static class LogThread<LOGTYPE> extends Thread {
private final String symlinkName;
private final ExecutorService executor = createCompressionTaskExecutor();
private final NativeIO nativeIO = new NativeIO();
private final long rotationSize;


LogThread(LogWriter<LOGTYPE> logWriter,
String filePattern,
Compression compression,
int bufferSize,
long[] rotationTimes,
String symlinkName,
long rotationSize,
String threadName,
Pollable<LOGTYPE> operationProvider,
Clock clock) {
Pollable<LOGTYPE> operationProvider) {
super(threadName);
setDaemon(true);
this.logWriter = logWriter;
Expand All @@ -245,22 +214,7 @@ static class LogThread<LOGTYPE> extends Thread {
this.bufferSize = bufferSize;
this.rotationTimes = rotationTimes;
this.symlinkName = (symlinkName != null && !symlinkName.isBlank()) ? symlinkName : null;
this.rotationSize = rotationSize;
this.operationProvider = operationProvider;
this.clock = clock;
}

LogThread(LogWriter<LOGTYPE> logWriter,
String filePattern,
Compression compression,
int bufferSize,
long[] rotationTimes,
String symlinkName,
long rotationSize,
String threadName,
Pollable<LOGTYPE> operationProvider) {
this(logWriter, filePattern, compression, bufferSize, rotationTimes,
symlinkName, rotationSize, threadName, operationProvider, Clock.systemUTC());
}

private static ExecutorService createCompressionTaskExecutor() {
Expand Down Expand Up @@ -340,18 +294,11 @@ private void internalPublish(LOGTYPE r) {
// first check to see if new file needed.
// if so, use this.internalRotateNow() to do it

long now = clock.millis();
Instant nowInstant = Instant.ofEpochMilli(now);
long now = System.currentTimeMillis();
if (nextRotationTime <= 0) {
nextRotationTime = getNextRotationTime(now); // lazy initialization
}
if (lastFileSizeCheck.plus(fileSizeCheckInterval).isBefore(nowInstant)) {
getFileSize(nowInstant);
}
if (rotationSize > 0 && fileOutput != null && fileSize >= rotationSize) {
nextRotationTime = now; // trigger rotation based on size
}
if (now >= nextRotationTime || fileOutput == null) {
if (now > nextRotationTime || fileOutput == null) {
internalRotateNow();
}
try {
Expand All @@ -362,17 +309,6 @@ private void internalPublish(LOGTYPE r) {
}
}

private void getFileSize(Instant now) {
if (fileOutput != null) {
try {
fileSize = Files.size(Paths.get(fileName));
lastFileSizeCheck = now;
} catch (IOException e) {
logger.log(Level.WARNING, "Failed to get log file size: " + Exceptions.toMessageString(e), e);
}
}
}

/**
* Find next rotation after specified time.
*
Expand All @@ -381,7 +317,7 @@ private void getFileSize(Instant now) {
*/
long getNextRotationTime(long now) {
if (now <= 0) {
now = clock.millis();
now = System.currentTimeMillis();
}
long nowTod = timeOfDayMillis(now);
long next = 0;
Expand Down Expand Up @@ -416,7 +352,7 @@ private void internalRotateNow() {
// figure out new file name, then

String oldFileName = fileName;
long now = clock.millis();
long now = System.currentTimeMillis();
fileName = LogFormatter.insertDate(filePattern, now);
internalClose();
try {
Expand Down
3 changes: 0 additions & 3 deletions container-core/src/main/resources/configdefinitions/container.core.access-log.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,3 @@ fileHandler.queueSize int default=10000

# Buffer size for the output stream has a default of 256k
fileHandler.bufferSize int default=262144

# Maximum file size (in bytes) before rotation. 0 means disabled (only time-based rotation)
fileHandler.rotationSize long default=0
Loading