Skip to content

GitLab

L
logstash-forwarder
Commit 2e20f954 authored by Jordan Sissel's avatar Jordan Sissel
Browse files
Options

- build with the go version now 

- gozmqfix'd publisher.go
- keep lumberjack.sh
parent 48efc23b
Hide whitespace changes
Inline Side-by-side
Showing with 213 additions and 124 deletions
Makefile
View file @ 2e20f954
VERSION=0.0.30
VERSION=0.1.0
# By default, all dependencies (zeromq, etc) will be downloaded and installed
# locally. You can change this if you are deploying your own.
VENDOR?=zeromq jemalloc openssl zlib
VENDOR?=zeromq libsodium
# Where to install to.
PREFIX?=/opt/lumberjack
FETCH=sh fetch.sh
CFLAGS+=-D_POSIX_C_SOURCE=199309 -std=c99 -Wall -Wextra -Werror -pipe
CFLAGS+=-g
CFLAGS+=-Wno-unused-function
LDFLAGS+=-pthread
LIBS=-lzmq -ljemalloc -lssl -lcrypto -luuid -lz
MAKE?=make
CFLAGS+=-Ibuild/include
LDFLAGS+=-Lbuild/lib -Wl,-rpath,'$$ORIGIN/../lib'
......@@ -24,22 +16,24 @@ default: build-all
build-all: build/bin/lumberjack build/bin/lumberjack.sh
include Makefile.ext
ifeq ($(UNAME),Linux)
# clock_gettime is in librt on linux.
LIBS+=-lrt
endif
clean:
-@rm -fr lumberjack unixsock *.o build
deps-clean:
rm -fr src/code.google.com/
rm -fr src/github.com/ugorji/go-msgpack
rm -fr src/github.com/alecthomas/gozmq
vendor-clean:
-$(MAKE) -C vendor/msgpack/ clean
-$(MAKE) -C vendor/jansson/ clean
-$(MAKE) -C vendor/jemalloc/ clean
-$(MAKE) -C vendor/libuuid/ clean
-$(MAKE) -C vendor/zeromq/ clean
-$(MAKE) -C vendor/zlib/ clean
-$(MAKE) -C vendor/apr/ clean
$(MAKE) -C vendor/apr/ clean
$(MAKE) -C vendor/jansson/ clean
$(MAKE) -C vendor/jemalloc/ clean
$(MAKE) -C vendor/libsodium/ clean
$(MAKE) -C vendor/libuuid/ clean
$(MAKE) -C vendor/lz4/ clean
$(MAKE) -C vendor/msgpack/ clean
$(MAKE) -C vendor/openssl/ clean
$(MAKE) -C vendor/zeromq/ clean
$(MAKE) -C vendor/zlib/ clean
rpm deb: | build-all
fpm -s dir -t $@ -n lumberjack -v $(VERSION) --prefix /opt/lumberjack \
......@@ -48,103 +42,43 @@ rpm deb: | build-all
--url "https://github.com/jordansissel/lumberjack" \
bin/lumberjack bin/lumberjack.sh lib
#install: build/bin/lumberjack build/lib/libzmq.$(LIBEXT)
# install -d -m 755 build/bin/* $(PREFIX)/bin/lumberjack
# install -d build/lib/* $(PREFIX)/lib
flog.o: flog.c flog.h
strlist.o: strlist.h
emitter.o: strlist.h
backoff.o: backoff.c backoff.h
harvester.o: harvester.c harvester.h proto.h str.h sleepdefs.h flog.h
emitter.o: emitter.c emitter.h ring.h sleepdefs.h flog.h
lumberjack.o: lumberjack.c backoff.h harvester.h emitter.h flog.h
str.o: str.c str.h
proto.o: proto.c proto.h str.h sleepdefs.h flog.h
ring.o: ring.c ring.h
harvester.o: build/include/insist.h
lumberjack.o: build/include/insist.h
# Vendor'd dependencies
# If VENDOR contains 'zeromq' download and build it.
ifeq ($(filter zeromq,$(VENDOR)),zeromq)
emitter.o: build/include/zmq.h
harvester.o: build/include/zmq.h
lumberjack.o: build/include/zmq.h
build/bin/lumberjack: | build/bin build/lib/libzmq.$(LIBEXT)
bin/lumberjack: | build/bin build/lib/libzmq.$(LIBEXT)
pkg/linux_amd64/github.com/alecthomas/gozmq.a: | build/lib/libzmq.$(LIBEXT)
src/github.com/alecthomas/gozmq/zmq.go: | build/lib/libzmq.$(LIBEXT)
endif # zeromq
ifeq ($(filter jemalloc,$(VENDOR)),jemalloc)
harvester.o lumberjack.o ring.o str.o: build/include/jemalloc/jemalloc.h
build/bin/lumberjack: | build/lib/libjemalloc.$(LIBEXT)
endif # jemalloc
ifeq ($(filter libsodium,$(VENDOR)),libsodium)
bin/keygen: | build/bin build/lib/libsodium.$(LIBEXT)
endif # libsodium
ifeq ($(filter openssl,$(VENDOR)),openssl)
proto.o: build/include/openssl/ssl.h
lumberjack.o: build/include/openssl/ssl.h
build/bin/lumberjack: | build/lib/libssl.$(LIBEXT)
build/bin/lumberjack: | build/lib/libcrypto.$(LIBEXT)
endif # openssl
build/bin/lumberjack.sh: lumberjack.sh | build/bin
install -m 755 $^ $@
ifeq ($(filter zlib,$(VENDOR)),zlib)
proto.o: build/include/zlib.h
build/bin/lumberjack: | build/lib/libz.$(LIBEXT)
endif # zlib
build/bin/lumberjack: | build/bin bin/lumberjack
cp bin/lumberjack build/bin/lumberjack
.PHONY: test
test: | build/test/test_ring
build/test/test_ring
bin/lumberjack: pkg/linux_amd64/github.com/alecthomas/gozmq.a
go install -ldflags '-r $$ORIGIN/../lib' lumberjack
# Tests
test_ring.o: ring.h build/include/jemalloc/jemalloc.h build/include/insist.h
build/test/test_ring: test_ring.o ring.o | build/test
$(CC) $(LDFLAGS) -o $@ $^ -ljemalloc
# gozmq
src/github.com/alecthomas/gozmq/zmq.go:
go get -d github.com/alecthomas/gozmq
#rm src/github.com/alecthomas/gozmq/zmq_2_*.go
build/bin/lumberjack.sh: lumberjack.sh | build/bin
install -m 755 $^ $@
build/bin/lumberjack: | build/bin
build/bin/lumberjack: lumberjack.o backoff.o harvester.o emitter.o str.o proto.o ring.o strlist.o flog.o
$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
@echo " => Build complete: $@"
@echo " => Run '$(MAKE) rpm' to build an rpm (or deb or tarball)"
pkg/linux_amd64/github.com/alecthomas/gozmq.a: src/github.com/alecthomas/gozmq/zmq.go
PKG_CONFIG_PATH=$$PWD/build/lib/pkgconfig \
go install -tags zmq_3_x github.com/alecthomas/gozmq
build/include/insist.h: | build/include
PATH=$$PWD:$$PATH fetch.sh -o $@ https://raw.github.com/jordansissel/experiments/master/c/better-assert/insist.h
bin/keygen:
go install -ldflags '-r $$ORIGIN/../lib' lumberjack
build/include/zmq.h build/lib/libzmq.$(LIBEXT): | build
@echo " => Building zeromq"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zeromq/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/msgpack.h build/lib/libmsgpack.$(LIBEXT): | build
@echo " => Building msgpack"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/msgpack/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/jemalloc/jemalloc.h build/lib/libjemalloc.$(LIBEXT): | build
@echo " => Building jemalloc"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/jemalloc/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/sodium/sodium.h build/lib/libsodium.$(LIBEXT): | build
@echo " => Building libsodium"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/libsodium/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/lz4.h build/lib/liblz4.$(LIBEXT): | build
@echo " => Building lz4"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/lz4/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/zlib.h build/lib/libz.$(LIBEXT): | build
@echo " => Building zlib"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zlib/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/openssl/ssl.h build/lib/libssl.$(LIBEXT) build/lib/libcrypto.$(LIBEXT): | build
@echo " => Building openssl"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/openssl install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/apr-1/apr.h build/lib/libapr-1.$(LIBEXT): | build
@echo " => Building apr"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/apr install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build:
mkdir $@
......
c/Makefile 0 → 100644
View file @ 2e20f954
VERSION=0.0.30
# By default, all dependencies (zeromq, etc) will be downloaded and installed
# locally. You can change this if you are deploying your own.
VENDOR?=zeromq jemalloc openssl zlib
# Where to install to.
PREFIX?=/opt/lumberjack
FETCH=sh fetch.sh
CFLAGS+=-D_POSIX_C_SOURCE=199309 -std=c99 -Wall -Wextra -Werror -pipe
CFLAGS+=-g
CFLAGS+=-Wno-unused-function
LDFLAGS+=-pthread
LIBS=-lzmq -ljemalloc -lssl -lcrypto -luuid -lz
MAKE?=make
CFLAGS+=-Ibuild/include
LDFLAGS+=-Lbuild/lib -Wl,-rpath,'$$ORIGIN/../lib'
default: build-all
build-all: build/bin/lumberjack build/bin/lumberjack.sh
include Makefile.ext
ifeq ($(UNAME),Linux)
# clock_gettime is in librt on linux.
LIBS+=-lrt
endif
clean:
-@rm -fr lumberjack unixsock *.o build
vendor-clean:
-$(MAKE) -C vendor/msgpack/ clean
-$(MAKE) -C vendor/jansson/ clean
-$(MAKE) -C vendor/jemalloc/ clean
-$(MAKE) -C vendor/libuuid/ clean
-$(MAKE) -C vendor/zeromq/ clean
-$(MAKE) -C vendor/zlib/ clean
-$(MAKE) -C vendor/apr/ clean
rpm deb: | build-all
fpm -s dir -t $@ -n lumberjack -v $(VERSION) --prefix /opt/lumberjack \
--exclude '*.a' --exclude 'lib/pkgconfig/zlib.pc' -C build \
--description "a log shipping tool" \
--url "https://github.com/jordansissel/lumberjack" \
bin/lumberjack bin/lumberjack.sh lib
#install: build/bin/lumberjack build/lib/libzmq.$(LIBEXT)
# install -d -m 755 build/bin/* $(PREFIX)/bin/lumberjack
# install -d build/lib/* $(PREFIX)/lib
flog.o: flog.c flog.h
strlist.o: strlist.h
emitter.o: strlist.h
backoff.o: backoff.c backoff.h
harvester.o: harvester.c harvester.h proto.h str.h sleepdefs.h flog.h
emitter.o: emitter.c emitter.h ring.h sleepdefs.h flog.h
lumberjack.o: lumberjack.c backoff.h harvester.h emitter.h flog.h
str.o: str.c str.h
proto.o: proto.c proto.h str.h sleepdefs.h flog.h
ring.o: ring.c ring.h
harvester.o: build/include/insist.h
lumberjack.o: build/include/insist.h
# Vendor'd dependencies
# If VENDOR contains 'zeromq' download and build it.
ifeq ($(filter zeromq,$(VENDOR)),zeromq)
emitter.o: build/include/zmq.h
harvester.o: build/include/zmq.h
lumberjack.o: build/include/zmq.h
build/bin/lumberjack: | build/bin build/lib/libzmq.$(LIBEXT)
endif # zeromq
ifeq ($(filter jemalloc,$(VENDOR)),jemalloc)
harvester.o lumberjack.o ring.o str.o: build/include/jemalloc/jemalloc.h
build/bin/lumberjack: | build/lib/libjemalloc.$(LIBEXT)
endif # jemalloc
ifeq ($(filter openssl,$(VENDOR)),openssl)
proto.o: build/include/openssl/ssl.h
lumberjack.o: build/include/openssl/ssl.h
build/bin/lumberjack: | build/lib/libssl.$(LIBEXT)
build/bin/lumberjack: | build/lib/libcrypto.$(LIBEXT)
endif # openssl
ifeq ($(filter zlib,$(VENDOR)),zlib)
proto.o: build/include/zlib.h
build/bin/lumberjack: | build/lib/libz.$(LIBEXT)
endif # zlib
.PHONY: test
test: | build/test/test_ring
build/test/test_ring
# Tests
test_ring.o: ring.h build/include/jemalloc/jemalloc.h build/include/insist.h
build/test/test_ring: test_ring.o ring.o | build/test
$(CC) $(LDFLAGS) -o $@ $^ -ljemalloc
build/bin/lumberjack.sh: lumberjack.sh | build/bin
install -m 755 $^ $@
build/bin/lumberjack: | build/bin
build/bin/lumberjack: lumberjack.o backoff.o harvester.o emitter.o str.o proto.o ring.o strlist.o flog.o
$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
@echo " => Build complete: $@"
@echo " => Run '$(MAKE) rpm' to build an rpm (or deb or tarball)"
build/include/insist.h: | build/include
PATH=$$PWD:$$PATH fetch.sh -o $@ https://raw.github.com/jordansissel/experiments/master/c/better-assert/insist.h
build/include/zmq.h build/lib/libzmq.$(LIBEXT): | build
@echo " => Building zeromq"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zeromq/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/msgpack.h build/lib/libmsgpack.$(LIBEXT): | build
@echo " => Building msgpack"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/msgpack/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/jemalloc/jemalloc.h build/lib/libjemalloc.$(LIBEXT): | build
@echo " => Building jemalloc"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/jemalloc/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/sodium/sodium.h build/lib/libsodium.$(LIBEXT): | build
@echo " => Building libsodium"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/libsodium/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/lz4.h build/lib/liblz4.$(LIBEXT): | build
@echo " => Building lz4"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/lz4/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/zlib.h build/lib/libz.$(LIBEXT): | build
@echo " => Building zlib"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zlib/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/openssl/ssl.h build/lib/libssl.$(LIBEXT) build/lib/libcrypto.$(LIBEXT): | build
@echo " => Building openssl"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/openssl install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/apr-1/apr.h build/lib/libapr-1.$(LIBEXT): | build
@echo " => Building apr"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/apr install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build:
mkdir $@
build/include build/bin build/test: | build
mkdir $@
src/liblumberjack/publisher.go
View file @ 2e20f954
package liblumberjack
import (
"log"
"bytes"
"encoding/json"
zmq "github.com/alecthomas/gozmq"
"log"
"math/big"
"syscall"
"bytes"
"time"
//"syscall"
"compress/zlib"
......@@ -14,7 +14,8 @@ import (
"crypto/rand"
)
var context zmq.Context
var context *zmq.Context
func init() {
context, _ = zmq.NewContext()
}
......@@ -30,9 +31,9 @@ type FFS struct {
SendTimeout time.Duration
RecvTimeout time.Duration
endpoint string // the current endpoint in use
socket zmq.Socket // the current zmq socket
connected bool // are we connected?
endpoint string // the current endpoint in use
socket *zmq.Socket // the current zmq socket
connected bool // are we connected?
}
func (s *FFS) Send(data []byte, flags zmq.SendRecvOption) (err error) {
......@@ -40,18 +41,18 @@ func (s *FFS) Send(data []byte, flags zmq.SendRecvOption) (err error) {
s.ensure_connect()
pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLOUT}}
count, err := zmq.Poll(pi, int64(s.SendTimeout.Nanoseconds() / 1000))
count, err := zmq.Poll(pi, s.SendTimeout)
if count == 0 {
// not ready in time, fail the socket and try again.
log.Printf("%s: timed out waiting to Send(): %s\n",
s.endpoint, err)
s.endpoint, err)
s.fail_socket()
} else {
//log.Printf("%s: sending %d payload\n", s.endpoint, len(data))
err = s.socket.Send(data, flags)
if err != nil {
log.Printf("%s: Failed to Send() %d byte message: %s\n",
s.endpoint, len(data), err)
s.endpoint, len(data), err)
s.fail_socket()
} else {
// Success!
......@@ -66,20 +67,20 @@ func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) {
s.ensure_connect()
pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLIN}}
count, err := zmq.Poll(pi, int64(s.RecvTimeout.Nanoseconds() / 1000))
count, err := zmq.Poll(pi, s.RecvTimeout)
if count == 0 {
// not ready in time, fail the socket and try again.
s.fail_socket()
err = syscall.ETIMEDOUT
log.Printf("%s: timed out waiting to Recv(): %s\n",
s.endpoint, err)
s.endpoint, err)
return nil, err
} else {
data, err = s.socket.Recv(flags)
if err != nil {
log.Printf("%s: Failed to Recv() %d byte message: %s\n",
s.endpoint, len(data), err)
s.endpoint, len(data), err)
s.fail_socket()
return nil, err
} else {
......@@ -91,7 +92,9 @@ func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) {
func (s *FFS) Close() (err error) {
err = s.socket.Close()
if err != nil { return }
if err != nil {
return
}
s.socket = nil
s.connected = false
......@@ -113,13 +116,13 @@ func (s *FFS) ensure_connect() {
if s.SocketType == 0 {
log.Panicf("No socket type set on zmq socket")
}
if s.socket != nil {
if s.socket != nil {
s.socket.Close()
s.socket = nil
}
var err error
s.socket, err = context.NewSocket(s.SocketType)
s.socket, err = context.NewSocket(s.SocketType)
if err != nil {
log.Panicf("zmq.NewSocket(%d) failed: %s\n", s.SocketType, err)
}
......@@ -148,19 +151,21 @@ func (s *FFS) ensure_connect() {
}
func (s *FFS) fail_socket() {
if !s.connected { return }
if !s.connected {
return
}
s.Close()
}
func Publish(input chan []*FileEvent, server_list []string,
server_timeout time.Duration) {
server_timeout time.Duration) {
var buffer bytes.Buffer
//key := "abcdefghijklmnop"
//cipher, err := aes.NewCipher([]byte(key))
socket := FFS{
Endpoints: server_list,
SocketType: zmq.REQ,
Endpoints: server_list,
SocketType: zmq.REQ,
RecvTimeout: server_timeout,
SendTimeout: server_timeout,
}
......@@ -210,5 +215,3 @@ func Publish(input chan []*FileEvent, server_list []string,
// TODO(sissel): notify registrar of success
} /* for each event payload */
} // Publish
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment