Commit b0622f00 authored by Gurvinder Singh's avatar Gurvinder Singh

fixed from beginning and flood logs issue

parent 9b8fb710
......@@ -53,6 +53,21 @@ rpm deb: | build-all
--url "https://github.com/elasticsearch/logstash-forwarder" \
build/bin/logstash-forwarder=$(PREFIX)/bin/ \
build/bin/logstash-forwarder.sh=$(PREFIX)/bin/ \
logrouter_uninett.pem=/etc/ssl/certs/ \
logstash-forwarder.conf=/etc/logstashforwarder/config.json.example \
logstash-forwarder.init=/etc/init.d/logstash-forwarder
rpm32 deb32: PREFIX=/opt/logstash-forwarder
rpm32 deb32: | build-all
fpm -a i386 -s dir -t deb -n logstash-forwarder -v $(VERSION) \
--replaces lumberjack \
--exclude '*.a' --exclude 'lib/pkgconfig/zlib.pc' \
--description "a log shipping tool" \
--url "https://github.com/elasticsearch/logstash-forwarder" \
build/bin/logstash-forwarder=$(PREFIX)/bin/ \
build/bin/logstash-forwarder.sh=$(PREFIX)/bin/ \
logrouter_uninett.pem=/etc/ssl/certs/ \
logstash-forwarder.conf=/etc/logstashforwarder/config.json.example \
logstash-forwarder.init=/etc/init.d/logstash-forwarder
# Vendor'd dependencies
......
......@@ -106,10 +106,10 @@ func (h *Harvester) open() *os.File {
}
// TODO(sissel): Only seek if the file is a file, not a pipe or socket.
if h.Offset > 0 {
h.file.Seek(h.Offset, os.SEEK_SET)
} else if *from_beginning {
if *from_beginning {
h.file.Seek(0, os.SEEK_SET)
} else if h.Offset > 0 {
h.file.Seek(h.Offset, os.SEEK_SET)
} else {
h.file.Seek(0, os.SEEK_END)
}
......
{
"network": {
"servers": [ "localhost:5043" ],
"ssl certificate": "./lumberjack.crt",
"ssl key": "./lumberjack.key",
"ssl ca": "./lumberjack_ca.crt"
},
"files": [
{
"paths": [
"/var/log/*.log",
"/var/log/messages"
],
"fields": { "type": "syslog" }
}, {
"paths": [ "/var/log/apache2/access.log" ],
"fields": { "type": "apache" }
}
]
}
......@@ -10,6 +10,7 @@ import (
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var spool_size = flag.Uint64("spool-size", 1024, "Maximum number of events to spool before a flush is forced.")
var age_limit = flag.Duration("age_limit", 1, "Maximum number of days old files to read.")
var idle_timeout = flag.Duration("idle-flush-time", 5*time.Second, "Maximum time to wait for a full spool before flushing anyway")
var config_file = flag.String("config", "", "The config file to load")
var use_syslog = flag.Bool("log-to-syslog", false, "Log to syslog instead of stdout")
......@@ -60,7 +61,7 @@ func main() {
// Prospect the globs/paths given on the command line and launch harvesters
for _, fileconfig := range config.Files {
go Prospect(fileconfig, event_chan)
go Prospect(fileconfig, event_chan, *age_limit)
}
// Harvesters dump events into the spooler.
......
......@@ -17,7 +17,7 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="log shipper"
NAME=logstash-forwarder
DAEMON=/opt/logstash-forwarder/bin/logstash-forwarder
DAEMON_ARGS="-config /etc/logstash-forwarder -spool-size 100 -log-to-syslog"
DAEMON_ARGS="-config /etc/logstashforwarder/config.json -spool-size 100 -log-to-syslog"
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
......
......@@ -8,7 +8,7 @@ import (
"time"
)
func Prospect(fileconfig FileConfig, output chan *FileEvent) {
func Prospect(fileconfig FileConfig, output chan *FileEvent, age_limit time.Duration) {
fileinfo := make(map[string]os.FileInfo)
// Handle any "-" (stdin) paths
......@@ -27,7 +27,7 @@ func Prospect(fileconfig FileConfig, output chan *FileEvent) {
for {
for _, path := range fileconfig.Paths {
prospector_scan(path, fileconfig.Fields, fileinfo, output)
prospector_scan(path, fileconfig.Fields, fileinfo, output, age_limit)
}
// Defer next scan for a bit.
......@@ -72,7 +72,7 @@ func resume_tracking(fileconfig FileConfig, fileinfo map[string]os.FileInfo, out
func prospector_scan(path string, fields map[string]string,
fileinfo map[string]os.FileInfo,
output chan *FileEvent) {
output chan *FileEvent, age_limit time.Duration) {
//log.Printf("Prospecting %s\n", path)
// Evaluate the path as a wildcards/shell glob
......@@ -114,7 +114,7 @@ func prospector_scan(path string, fields map[string]string,
if !is_known {
// TODO(sissel): Skip files with modification dates older than N
// TODO(sissel): Make the 'ignore if older than N' tunable
if time.Since(info.ModTime()) > 24*time.Hour {
if time.Since(info.ModTime()) > 30*24*time.Hour {
log.Printf("Skipping old file: %s\n", file)
} else if is_file_renamed(file, info, fileinfo) {
// Check to see if this file was simply renamed (known inode+dev)
......
......@@ -5,8 +5,8 @@ import (
)
func Registrar(input chan []*FileEvent) {
for events := range input {
state := make(map[string]*FileState)
for events := range input {
log.Printf("Registrar received %d events\n", len(events))
// Take the last event found for each file source
for _, event := range events {
......
......@@ -3,14 +3,16 @@
package main
import (
"os"
"log"
"log/syslog"
// "log/syslog"
)
func configureSyslog() {
writer, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "logstash-forwarder")
//writer, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "logstash-forwarder")
writer, err := os.OpenFile("/var/log/logstash-forwarder.log", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
if err != nil {
log.Fatalf("Failed to open syslog: %s\n", err)
log.Fatalf("Failed to open log: %s\n", err)
return
}
log.SetOutput(writer)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment