Commit b0622f00 authored by Gurvinder Singh's avatar Gurvinder Singh

fixed from beginning and flood logs issue

parent 9b8fb710
...@@ -53,6 +53,21 @@ rpm deb: | build-all ...@@ -53,6 +53,21 @@ rpm deb: | build-all
--url "https://github.com/elasticsearch/logstash-forwarder" \ --url "https://github.com/elasticsearch/logstash-forwarder" \
build/bin/logstash-forwarder=$(PREFIX)/bin/ \ build/bin/logstash-forwarder=$(PREFIX)/bin/ \
build/bin/logstash-forwarder.sh=$(PREFIX)/bin/ \ build/bin/logstash-forwarder.sh=$(PREFIX)/bin/ \
logrouter_uninett.pem=/etc/ssl/certs/ \
logstash-forwarder.conf=/etc/logstashforwarder/config.json.example \
logstash-forwarder.init=/etc/init.d/logstash-forwarder
rpm32 deb32: PREFIX=/opt/logstash-forwarder
rpm32 deb32: | build-all
fpm -a i386 -s dir -t deb -n logstash-forwarder -v $(VERSION) \
--replaces lumberjack \
--exclude '*.a' --exclude 'lib/pkgconfig/zlib.pc' \
--description "a log shipping tool" \
--url "https://github.com/elasticsearch/logstash-forwarder" \
build/bin/logstash-forwarder=$(PREFIX)/bin/ \
build/bin/logstash-forwarder.sh=$(PREFIX)/bin/ \
logrouter_uninett.pem=/etc/ssl/certs/ \
logstash-forwarder.conf=/etc/logstashforwarder/config.json.example \
logstash-forwarder.init=/etc/init.d/logstash-forwarder logstash-forwarder.init=/etc/init.d/logstash-forwarder
# Vendor'd dependencies # Vendor'd dependencies
......
...@@ -106,10 +106,10 @@ func (h *Harvester) open() *os.File { ...@@ -106,10 +106,10 @@ func (h *Harvester) open() *os.File {
} }
// TODO(sissel): Only seek if the file is a file, not a pipe or socket. // TODO(sissel): Only seek if the file is a file, not a pipe or socket.
if h.Offset > 0 { if *from_beginning {
h.file.Seek(h.Offset, os.SEEK_SET)
} else if *from_beginning {
h.file.Seek(0, os.SEEK_SET) h.file.Seek(0, os.SEEK_SET)
} else if h.Offset > 0 {
h.file.Seek(h.Offset, os.SEEK_SET)
} else { } else {
h.file.Seek(0, os.SEEK_END) h.file.Seek(0, os.SEEK_END)
} }
......
{
"network": {
"servers": [ "localhost:5043" ],
"ssl certificate": "./lumberjack.crt",
"ssl key": "./lumberjack.key",
"ssl ca": "./lumberjack_ca.crt"
},
"files": [
{
"paths": [
"/var/log/*.log",
"/var/log/messages"
],
"fields": { "type": "syslog" }
}, {
"paths": [ "/var/log/apache2/access.log" ],
"fields": { "type": "apache" }
}
]
}
...@@ -10,6 +10,7 @@ import ( ...@@ -10,6 +10,7 @@ import (
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var spool_size = flag.Uint64("spool-size", 1024, "Maximum number of events to spool before a flush is forced.") var spool_size = flag.Uint64("spool-size", 1024, "Maximum number of events to spool before a flush is forced.")
var age_limit = flag.Duration("age_limit", 1, "Maximum number of days old files to read.")
var idle_timeout = flag.Duration("idle-flush-time", 5*time.Second, "Maximum time to wait for a full spool before flushing anyway") var idle_timeout = flag.Duration("idle-flush-time", 5*time.Second, "Maximum time to wait for a full spool before flushing anyway")
var config_file = flag.String("config", "", "The config file to load") var config_file = flag.String("config", "", "The config file to load")
var use_syslog = flag.Bool("log-to-syslog", false, "Log to syslog instead of stdout") var use_syslog = flag.Bool("log-to-syslog", false, "Log to syslog instead of stdout")
...@@ -60,7 +61,7 @@ func main() { ...@@ -60,7 +61,7 @@ func main() {
// Prospect the globs/paths given on the command line and launch harvesters // Prospect the globs/paths given on the command line and launch harvesters
for _, fileconfig := range config.Files { for _, fileconfig := range config.Files {
go Prospect(fileconfig, event_chan) go Prospect(fileconfig, event_chan, *age_limit)
} }
// Harvesters dump events into the spooler. // Harvesters dump events into the spooler.
......
...@@ -17,7 +17,7 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin ...@@ -17,7 +17,7 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="log shipper" DESC="log shipper"
NAME=logstash-forwarder NAME=logstash-forwarder
DAEMON=/opt/logstash-forwarder/bin/logstash-forwarder DAEMON=/opt/logstash-forwarder/bin/logstash-forwarder
DAEMON_ARGS="-config /etc/logstash-forwarder -spool-size 100 -log-to-syslog" DAEMON_ARGS="-config /etc/logstashforwarder/config.json -spool-size 100 -log-to-syslog"
PIDFILE=/var/run/$NAME.pid PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME SCRIPTNAME=/etc/init.d/$NAME
......
...@@ -8,7 +8,7 @@ import ( ...@@ -8,7 +8,7 @@ import (
"time" "time"
) )
func Prospect(fileconfig FileConfig, output chan *FileEvent) { func Prospect(fileconfig FileConfig, output chan *FileEvent, age_limit time.Duration) {
fileinfo := make(map[string]os.FileInfo) fileinfo := make(map[string]os.FileInfo)
// Handle any "-" (stdin) paths // Handle any "-" (stdin) paths
...@@ -27,7 +27,7 @@ func Prospect(fileconfig FileConfig, output chan *FileEvent) { ...@@ -27,7 +27,7 @@ func Prospect(fileconfig FileConfig, output chan *FileEvent) {
for { for {
for _, path := range fileconfig.Paths { for _, path := range fileconfig.Paths {
prospector_scan(path, fileconfig.Fields, fileinfo, output) prospector_scan(path, fileconfig.Fields, fileinfo, output, age_limit)
} }
// Defer next scan for a bit. // Defer next scan for a bit.
...@@ -72,7 +72,7 @@ func resume_tracking(fileconfig FileConfig, fileinfo map[string]os.FileInfo, out ...@@ -72,7 +72,7 @@ func resume_tracking(fileconfig FileConfig, fileinfo map[string]os.FileInfo, out
func prospector_scan(path string, fields map[string]string, func prospector_scan(path string, fields map[string]string,
fileinfo map[string]os.FileInfo, fileinfo map[string]os.FileInfo,
output chan *FileEvent) { output chan *FileEvent, age_limit time.Duration) {
//log.Printf("Prospecting %s\n", path) //log.Printf("Prospecting %s\n", path)
// Evaluate the path as a wildcards/shell glob // Evaluate the path as a wildcards/shell glob
...@@ -114,7 +114,7 @@ func prospector_scan(path string, fields map[string]string, ...@@ -114,7 +114,7 @@ func prospector_scan(path string, fields map[string]string,
if !is_known { if !is_known {
// TODO(sissel): Skip files with modification dates older than N // TODO(sissel): Skip files with modification dates older than N
// TODO(sissel): Make the 'ignore if older than N' tunable // TODO(sissel): Make the 'ignore if older than N' tunable
if time.Since(info.ModTime()) > 24*time.Hour { if time.Since(info.ModTime()) > 30*24*time.Hour {
log.Printf("Skipping old file: %s\n", file) log.Printf("Skipping old file: %s\n", file)
} else if is_file_renamed(file, info, fileinfo) { } else if is_file_renamed(file, info, fileinfo) {
// Check to see if this file was simply renamed (known inode+dev) // Check to see if this file was simply renamed (known inode+dev)
......
...@@ -5,8 +5,8 @@ import ( ...@@ -5,8 +5,8 @@ import (
) )
func Registrar(input chan []*FileEvent) { func Registrar(input chan []*FileEvent) {
state := make(map[string]*FileState)
for events := range input { for events := range input {
state := make(map[string]*FileState)
log.Printf("Registrar received %d events\n", len(events)) log.Printf("Registrar received %d events\n", len(events))
// Take the last event found for each file source // Take the last event found for each file source
for _, event := range events { for _, event := range events {
......
...@@ -3,14 +3,16 @@ ...@@ -3,14 +3,16 @@
package main package main
import ( import (
"os"
"log" "log"
"log/syslog" // "log/syslog"
) )
func configureSyslog() { func configureSyslog() {
writer, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "logstash-forwarder") //writer, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "logstash-forwarder")
writer, err := os.OpenFile("/var/log/logstash-forwarder.log", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
if err != nil { if err != nil {
log.Fatalf("Failed to open syslog: %s\n", err) log.Fatalf("Failed to open log: %s\n", err)
return return
} }
log.SetOutput(writer) log.SetOutput(writer)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment