Commit 8f6547aa authored by Jordan Sissel's avatar Jordan Sissel

Rename lumberjack to logstash-forwarder

parent 5a816d62
...@@ -20,3 +20,5 @@ nacl.secret ...@@ -20,3 +20,5 @@ nacl.secret
.lumberjack .lumberjack
.lumberjack.new .lumberjack.new
.rbx .rbx
lumberjack
logstash-forwarder
# lumberjack # logstash-forwarder
o/~ I'm a lumberjack and I'm ok! I sleep when idle, then I ship logs all day! I parse your logs, I eat the JVM agent for lunch! o/~ o/~ I'm a lumberjack and I'm ok! I sleep when idle, then I ship logs all day! I parse your logs, I eat the JVM agent for lunch! o/~
...@@ -18,20 +18,21 @@ Perceived Problems: Some users view logstash releases as "large" or have a gener ...@@ -18,20 +18,21 @@ Perceived Problems: Some users view logstash releases as "large" or have a gener
Actual Problems: Logstash, for right now, runs with a footprint that is not Actual Problems: Logstash, for right now, runs with a footprint that is not
friendly to underprovisioned systems such as EC2 micro instances; on other friendly to underprovisioned systems such as EC2 micro instances; on other
systems it is fine. Lumberjack will exist until that is resolved. systems it is fine. This project will exist until that is resolved.
### Transport Problems ### Transport Problems
Few log transport mechanisms provide security, low latency, and reliability. Few log transport mechanisms provide security, low latency, and reliability.
lumberjack exists to provide a network protocol for transmission that is The lumberjack protocol used by this project exists to provide a network
secure, low latency, low resource usage, and reliable. protocol for transmission that is secure, low latency, low resource usage, and
reliable.
## Configuring ## Configuring
lumberjack is configured with a json file you specify with the -config flag: logstash-forwarder is configured with a json file you specify with the -config flag:
`lumberjack -config yourstuff.json` `logstash-forwarder -config yourstuff.json`
Here's a sample, with comments in-line to describe the settings. Please please Here's a sample, with comments in-line to describe the settings. Please please
please keep in mind that comments are technically invalid in JSON, so you can't please keep in mind that comments are technically invalid in JSON, so you can't
...@@ -41,24 +42,24 @@ include them in your config.: ...@@ -41,24 +42,24 @@ include them in your config.:
# The network section covers network configuration :) # The network section covers network configuration :)
"network": { "network": {
# A list of downstream servers listening for our messages. # A list of downstream servers listening for our messages.
# lumberjack will pick one at random and only switch if # logstash-forwarder will pick one at random and only switch if
# the selected one appears to be dead or unresponsive # the selected one appears to be dead or unresponsive
"servers": [ "localhost:5043" ], "servers": [ "localhost:5043" ],
# The path to your client ssl certificate (optional) # The path to your client ssl certificate (optional)
"ssl certificate": "./lumberjack.crt", "ssl certificate": "./logstash-forwarder.crt",
# The path to your client ssl key (optional) # The path to your client ssl key (optional)
"ssl key": "./lumberjack.key", "ssl key": "./logstash-forwarder.key",
# The path to your trusted ssl CA file. This is used # The path to your trusted ssl CA file. This is used
# to authenticate your downstream server. # to authenticate your downstream server.
"ssl ca": "./lumberjack_ca.crt", "ssl ca": "./logstash-forwarder.crt",
# Network timeout in seconds. This is most important for lumberjack # Network timeout in seconds. This is most important for
# determining whether to stop waiting for an acknowledgement from the # logstash-forwarder determining whether to stop waiting for an
# downstream server. If an timeout is reached, lumberjack will assume # acknowledgement from the downstream server. If an timeout is reached,
# the connection or server is bad and will connect to a server chosen # logstash-forwarder will assume the connection or server is bad and
# at random from the servers list. # will connect to a server chosen at random from the servers list.
"timeout": 15 "timeout": 15
}, },
...@@ -98,46 +99,49 @@ include them in your config.: ...@@ -98,46 +99,49 @@ include them in your config.:
* Easy to deploy with minimal moving parts. * Easy to deploy with minimal moving parts.
* Simple inputs only: * Simple inputs only:
* Follows files and respects rename/truncation conditions. * Follows files and respects rename/truncation conditions.
* Accepts `STDIN`, useful for things like `varnishlog | lumberjack...`. * Accepts `STDIN`, useful for things like `varnishlog | logstash-forwarder...`.
## Building it ## Building it
1. Install [FPM](https://github.com/jordansissel/fpm) 1. Install [go](http://golang.org/doc/install)
$ sudo gem install fpm 2. Compile logstash-forwarder
2. Install [go](http://golang.org/doc/install) git clone git://github.com/elasticsearch/logstash-forwarder.git
cd logstash-forwarder
go build
## Packaging it (optional)
3. Compile lumberjack You can make native packages of logstash-forwarder.
$ git clone git://github.com/jordansissel/lumberjack.git To build the packages, you will need ruby and fpm installed.
$ cd lumberback
$ go build
4. Make packages, either: gem install fpm
$ make rpm Now build an rpm:
Or: make rpm
$ make deb Or:
make deb
## Installing it (via packages only) ## Installing it (via packages only)
If you don't use rpm or deb make targets as above, you can skip this section. If you don't use rpm or deb make targets as above, you can skip this section.
Packages install to `/opt/lumberjack`. Lumberjack builds all necessary Packages install to `/opt/logstash-forwarder`.
dependencies itself, so there should be no run-time dependencies you
need. There are no run-time dependencies.
## Running it ## Running it
Generally: Generally:
$ lumberjack.sh -config lumberjack.conf logstash-forwarder -config lumberjack.conf
See `lumberjack.sh -help` for all the flags See `logstash-forwarder -help` for all the flags
The config file is documented further up in this file. The config file is documented further up in this file.
...@@ -152,9 +156,9 @@ The config file is documented further up in this file. ...@@ -152,9 +156,9 @@ The config file is documented further up in this file.
Logstash supports all certificates, including self-signed certificates. To generate a certificate, you can run the following command: Logstash supports all certificates, including self-signed certificates. To generate a certificate, you can run the following command:
$ openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout lumberjack.key -out lumberjack.crt $ openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout logstash-forwarder.key -out logstash-forwarder.crt
This will generate a key at `lumberjack.key` and the certificate at `lumberjack.crt`. Both the server that is running lumberjack as well as the logstash instances receiving logs will require these files on disk to verify the authenticity of messages. This will generate a key at `logstash-forwarder.key` and the certificate at `logstash-forwarder.crt`. Both the server that is running logstash-forwarder as well as the logstash instances receiving logs will require these files on disk to verify the authenticity of messages.
Recommended file locations: Recommended file locations:
...@@ -199,7 +203,7 @@ Below is valid as of 2012/09/19 ...@@ -199,7 +203,7 @@ Below is valid as of 2012/09/19
### Configurable event data ### Configurable event data
* The protocol lumberjack uses supports sending a `string:string` map. * The protocol supports sending a `string:string` map.
### Easy deployment ### Easy deployment
......
#!/bin/sh
dir=`dirname $0`
LD_LIBRARY_PATH="${dir}/../lib"
export LD_LIBRARY_PATH
exec "${dir}/lumberjack" "$@"
...@@ -37,7 +37,7 @@ func Prospect(fileconfig FileConfig, output chan *FileEvent) { ...@@ -37,7 +37,7 @@ func Prospect(fileconfig FileConfig, output chan *FileEvent) {
func resume_tracking(fileconfig FileConfig, fileinfo map[string]os.FileInfo, output chan *FileEvent) { func resume_tracking(fileconfig FileConfig, fileinfo map[string]os.FileInfo, output chan *FileEvent) {
// Start up with any registrar data. // Start up with any registrar data.
history, err := os.Open(".lumberjack") history, err := os.Open(".logstash-forwarder")
if err == nil { if err == nil {
historical_state := make(map[string]*FileState) historical_state := make(map[string]*FileState)
log.Printf("Loading registrar data\n") log.Printf("Loading registrar data\n")
......
...@@ -30,7 +30,7 @@ func Registrar(input chan []*FileEvent) { ...@@ -30,7 +30,7 @@ func Registrar(input chan []*FileEvent) {
} }
if len(state) > 0 { if len(state) > 0 {
WriteRegistry(state, ".lumberjack") WriteRegistry(state, ".logstash-forwarder")
} }
} }
} }
......
...@@ -9,9 +9,9 @@ import ( ...@@ -9,9 +9,9 @@ import (
func WriteRegistry(state map[string]*FileState, path string) { func WriteRegistry(state map[string]*FileState, path string) {
// Open tmp file, write, flush, rename // Open tmp file, write, flush, rename
file, err := os.Create(".lumberjack.new") file, err := os.Create(".logstash-forwarder.new")
if err != nil { if err != nil {
log.Printf("Failed to open .lumberjack.new for writing: %s\n", err) log.Printf("Failed to open .logstash-forwarder.new for writing: %s\n", err)
return return
} }
defer file.Close() defer file.Close()
...@@ -19,5 +19,5 @@ func WriteRegistry(state map[string]*FileState, path string) { ...@@ -19,5 +19,5 @@ func WriteRegistry(state map[string]*FileState, path string) {
encoder := json.NewEncoder(file) encoder := json.NewEncoder(file)
encoder.Encode(state) encoder.Encode(state)
os.Rename(".lumberjack.new", path) os.Rename(".logstash-forwarder.new", path)
} }
...@@ -10,7 +10,7 @@ func WriteRegistry(state map[string]*FileState, path string) { ...@@ -10,7 +10,7 @@ func WriteRegistry(state map[string]*FileState, path string) {
tmp := path + ".new" tmp := path + ".new"
file, err := os.Create(tmp) file, err := os.Create(tmp)
if err != nil { if err != nil {
log.Printf("Failed to open .lumberjack.new for writing: %s\n", err) log.Printf("Failed to open .logstash-forwarder.new for writing: %s\n", err)
return return
} }
......
...@@ -7,7 +7,7 @@ import ( ...@@ -7,7 +7,7 @@ import (
) )
func configureSyslog() { func configureSyslog() {
writer, err := syslog.New(syslog.LOG_INFO | syslog.LOG_DAEMON, "lumberjack") writer, err := syslog.New(syslog.LOG_INFO | syslog.LOG_DAEMON, "logstash-forwarder")
if err != nil { if err != nil {
log.Fatalf("Failed to open syslog: %s\n", err) log.Fatalf("Failed to open syslog: %s\n", err)
return return
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment