Commit eb6309a8 authored by Thomas Zahari's avatar Thomas Zahari

Small optimizations

Thanks to https://github.com/driskell
parent 41e4a4d4
...@@ -38,11 +38,12 @@ func (h *Harvester) Harvest(output chan *FileEvent) { ...@@ -38,11 +38,12 @@ func (h *Harvester) Harvest(output chan *FileEvent) {
// TODO(sissel): Make the buffer size tunable at start-time // TODO(sissel): Make the buffer size tunable at start-time
reader := bufio.NewReaderSize(h.file, 16<<10) // 16kb buffer by default reader := bufio.NewReaderSize(h.file, 16<<10) // 16kb buffer by default
buffer := new(bytes.Buffer)
var read_timeout = 10 * time.Second var read_timeout = 10 * time.Second
last_read_time := time.Now() last_read_time := time.Now()
for { for {
text, bytesread, err := h.readline(reader, read_timeout) text, bytesread, err := h.readline(reader, buffer, read_timeout)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
...@@ -116,16 +117,11 @@ func (h *Harvester) open() *os.File { ...@@ -116,16 +117,11 @@ func (h *Harvester) open() *os.File {
return h.file return h.file
} }
func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*string, int, error) { func (h *Harvester) readline(reader *bufio.Reader, buffer *bytes.Buffer, eof_timeout time.Duration) (*string, int, error) {
var buffer bytes.Buffer
var is_partial bool = true var is_partial bool = true
var is_cr_present bool = false var newline_length int = 1
var bufferSize int = 0;
start_time := time.Now() start_time := time.Now()
// Store current offset for seeking back on timeout if the line is not complete
offset, _ := h.file.Seek(0, os.SEEK_CUR)
for { for {
segment, err := reader.ReadBytes('\n') segment, err := reader.ReadBytes('\n')
...@@ -136,17 +132,12 @@ func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (* ...@@ -136,17 +132,12 @@ func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*
// Check if also a CR present // Check if also a CR present
if len(segment) > 1 && segment[len(segment)-2] == '\r' { if len(segment) > 1 && segment[len(segment)-2] == '\r' {
is_cr_present = true; newline_length++
} }
} }
}
if segment != nil && len(segment) > 0 {
// TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it? // TODO(sissel): if buffer exceeds a certain length, maybe report an error condition? chop it?
writelen,_ := buffer.Write(segment) buffer.Write(segment)
bufferSize += writelen;
} }
if err != nil { if err != nil {
...@@ -156,11 +147,6 @@ func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (* ...@@ -156,11 +147,6 @@ func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*
// Give up waiting for data after a certain amount of time. // Give up waiting for data after a certain amount of time.
// If we time out, return the error (eof) // If we time out, return the error (eof)
if time.Since(start_time) > eof_timeout { if time.Since(start_time) > eof_timeout {
// If we read a partial line then we seek back otherwise we miss this part
if len(segment) > 0 || bufferSize > 0 {
h.file.Seek(offset, os.SEEK_SET)
}
return nil, 0, err return nil, 0, err
} }
continue continue
...@@ -170,15 +156,14 @@ func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (* ...@@ -170,15 +156,14 @@ func (h *Harvester) readline(reader *bufio.Reader, eof_timeout time.Duration) (*
} }
} }
// If we got a full line, return the whole line without the EOL chars (CRLF or LF)
if !is_partial { if !is_partial {
// If we got a full line, return the whole line without the EOL chars (CRLF or LF) // Get the str length with the EOL chars (LF or CRLF)
bufferSize := buffer.Len()
str := new(string) str := new(string)
if !is_cr_present { *str = buffer.String()[:bufferSize - newline_length]
*str = buffer.String()[:bufferSize-1] // Reset the buffer for the next line
} else { buffer.Reset()
*str = buffer.String()[:bufferSize-2]
}
// bufferSize returns the str length with the EOL chars (LF or CRLF)
return str, bufferSize, nil return str, bufferSize, nil
} }
} /* forever read chunks */ } /* forever read chunks */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment