Commit 2e20f954 authored by Jordan Sissel's avatar Jordan Sissel

- build with the go version now

- gozmqfix'd publisher.go
- keep lumberjack.sh
parent 48efc23b
VERSION=0.0.30 VERSION=0.1.0
# By default, all dependencies (zeromq, etc) will be downloaded and installed # By default, all dependencies (zeromq, etc) will be downloaded and installed
# locally. You can change this if you are deploying your own. # locally. You can change this if you are deploying your own.
VENDOR?=zeromq jemalloc openssl zlib VENDOR?=zeromq libsodium
# Where to install to. # Where to install to.
PREFIX?=/opt/lumberjack PREFIX?=/opt/lumberjack
FETCH=sh fetch.sh FETCH=sh fetch.sh
CFLAGS+=-D_POSIX_C_SOURCE=199309 -std=c99 -Wall -Wextra -Werror -pipe
CFLAGS+=-g
CFLAGS+=-Wno-unused-function
LDFLAGS+=-pthread
LIBS=-lzmq -ljemalloc -lssl -lcrypto -luuid -lz
MAKE?=make MAKE?=make
CFLAGS+=-Ibuild/include CFLAGS+=-Ibuild/include
LDFLAGS+=-Lbuild/lib -Wl,-rpath,'$$ORIGIN/../lib' LDFLAGS+=-Lbuild/lib -Wl,-rpath,'$$ORIGIN/../lib'
...@@ -24,22 +16,24 @@ default: build-all ...@@ -24,22 +16,24 @@ default: build-all
build-all: build/bin/lumberjack build/bin/lumberjack.sh build-all: build/bin/lumberjack build/bin/lumberjack.sh
include Makefile.ext include Makefile.ext
ifeq ($(UNAME),Linux)
# clock_gettime is in librt on linux.
LIBS+=-lrt
endif
clean: clean:
-@rm -fr lumberjack unixsock *.o build -@rm -fr lumberjack unixsock *.o build
deps-clean:
rm -fr src/code.google.com/
rm -fr src/github.com/ugorji/go-msgpack
rm -fr src/github.com/alecthomas/gozmq
vendor-clean: vendor-clean:
-$(MAKE) -C vendor/msgpack/ clean $(MAKE) -C vendor/apr/ clean
-$(MAKE) -C vendor/jansson/ clean $(MAKE) -C vendor/jansson/ clean
-$(MAKE) -C vendor/jemalloc/ clean $(MAKE) -C vendor/jemalloc/ clean
-$(MAKE) -C vendor/libuuid/ clean $(MAKE) -C vendor/libsodium/ clean
-$(MAKE) -C vendor/zeromq/ clean $(MAKE) -C vendor/libuuid/ clean
-$(MAKE) -C vendor/zlib/ clean $(MAKE) -C vendor/lz4/ clean
-$(MAKE) -C vendor/apr/ clean $(MAKE) -C vendor/msgpack/ clean
$(MAKE) -C vendor/openssl/ clean
$(MAKE) -C vendor/zeromq/ clean
$(MAKE) -C vendor/zlib/ clean
rpm deb: | build-all rpm deb: | build-all
fpm -s dir -t $@ -n lumberjack -v $(VERSION) --prefix /opt/lumberjack \ fpm -s dir -t $@ -n lumberjack -v $(VERSION) --prefix /opt/lumberjack \
...@@ -48,103 +42,43 @@ rpm deb: | build-all ...@@ -48,103 +42,43 @@ rpm deb: | build-all
--url "https://github.com/jordansissel/lumberjack" \ --url "https://github.com/jordansissel/lumberjack" \
bin/lumberjack bin/lumberjack.sh lib bin/lumberjack bin/lumberjack.sh lib
#install: build/bin/lumberjack build/lib/libzmq.$(LIBEXT)
# install -d -m 755 build/bin/* $(PREFIX)/bin/lumberjack
# install -d build/lib/* $(PREFIX)/lib
flog.o: flog.c flog.h
strlist.o: strlist.h
emitter.o: strlist.h
backoff.o: backoff.c backoff.h
harvester.o: harvester.c harvester.h proto.h str.h sleepdefs.h flog.h
emitter.o: emitter.c emitter.h ring.h sleepdefs.h flog.h
lumberjack.o: lumberjack.c backoff.h harvester.h emitter.h flog.h
str.o: str.c str.h
proto.o: proto.c proto.h str.h sleepdefs.h flog.h
ring.o: ring.c ring.h
harvester.o: build/include/insist.h
lumberjack.o: build/include/insist.h
# Vendor'd dependencies # Vendor'd dependencies
# If VENDOR contains 'zeromq' download and build it. # If VENDOR contains 'zeromq' download and build it.
ifeq ($(filter zeromq,$(VENDOR)),zeromq) ifeq ($(filter zeromq,$(VENDOR)),zeromq)
emitter.o: build/include/zmq.h bin/lumberjack: | build/bin build/lib/libzmq.$(LIBEXT)
harvester.o: build/include/zmq.h pkg/linux_amd64/github.com/alecthomas/gozmq.a: | build/lib/libzmq.$(LIBEXT)
lumberjack.o: build/include/zmq.h src/github.com/alecthomas/gozmq/zmq.go: | build/lib/libzmq.$(LIBEXT)
build/bin/lumberjack: | build/bin build/lib/libzmq.$(LIBEXT)
endif # zeromq endif # zeromq
ifeq ($(filter jemalloc,$(VENDOR)),jemalloc) ifeq ($(filter libsodium,$(VENDOR)),libsodium)
harvester.o lumberjack.o ring.o str.o: build/include/jemalloc/jemalloc.h bin/keygen: | build/bin build/lib/libsodium.$(LIBEXT)
build/bin/lumberjack: | build/lib/libjemalloc.$(LIBEXT) endif # libsodium
endif # jemalloc
ifeq ($(filter openssl,$(VENDOR)),openssl) build/bin/lumberjack.sh: lumberjack.sh | build/bin
proto.o: build/include/openssl/ssl.h install -m 755 $^ $@
lumberjack.o: build/include/openssl/ssl.h
build/bin/lumberjack: | build/lib/libssl.$(LIBEXT)
build/bin/lumberjack: | build/lib/libcrypto.$(LIBEXT)
endif # openssl
ifeq ($(filter zlib,$(VENDOR)),zlib) build/bin/lumberjack: | build/bin bin/lumberjack
proto.o: build/include/zlib.h cp bin/lumberjack build/bin/lumberjack
build/bin/lumberjack: | build/lib/libz.$(LIBEXT)
endif # zlib
.PHONY: test bin/lumberjack: pkg/linux_amd64/github.com/alecthomas/gozmq.a
test: | build/test/test_ring go install -ldflags '-r $$ORIGIN/../lib' lumberjack
build/test/test_ring
# Tests # gozmq
test_ring.o: ring.h build/include/jemalloc/jemalloc.h build/include/insist.h src/github.com/alecthomas/gozmq/zmq.go:
build/test/test_ring: test_ring.o ring.o | build/test go get -d github.com/alecthomas/gozmq
$(CC) $(LDFLAGS) -o $@ $^ -ljemalloc #rm src/github.com/alecthomas/gozmq/zmq_2_*.go
build/bin/lumberjack.sh: lumberjack.sh | build/bin pkg/linux_amd64/github.com/alecthomas/gozmq.a: src/github.com/alecthomas/gozmq/zmq.go
install -m 755 $^ $@ PKG_CONFIG_PATH=$$PWD/build/lib/pkgconfig \
go install -tags zmq_3_x github.com/alecthomas/gozmq
build/bin/lumberjack: | build/bin
build/bin/lumberjack: lumberjack.o backoff.o harvester.o emitter.o str.o proto.o ring.o strlist.o flog.o
$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
@echo " => Build complete: $@"
@echo " => Run '$(MAKE) rpm' to build an rpm (or deb or tarball)"
build/include/insist.h: | build/include bin/keygen:
PATH=$$PWD:$$PATH fetch.sh -o $@ https://raw.github.com/jordansissel/experiments/master/c/better-assert/insist.h go install -ldflags '-r $$ORIGIN/../lib' lumberjack
build/include/zmq.h build/lib/libzmq.$(LIBEXT): | build build/include/zmq.h build/lib/libzmq.$(LIBEXT): | build
@echo " => Building zeromq" @echo " => Building zeromq"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zeromq/ install PREFIX=$$PWD/build DEBUG=$(DEBUG) PATH=$$PWD:$$PATH $(MAKE) -C vendor/zeromq/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/msgpack.h build/lib/libmsgpack.$(LIBEXT): | build
@echo " => Building msgpack"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/msgpack/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/jemalloc/jemalloc.h build/lib/libjemalloc.$(LIBEXT): | build
@echo " => Building jemalloc"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/jemalloc/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/sodium/sodium.h build/lib/libsodium.$(LIBEXT): | build
@echo " => Building libsodium"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/libsodium/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/lz4.h build/lib/liblz4.$(LIBEXT): | build
@echo " => Building lz4"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/lz4/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/zlib.h build/lib/libz.$(LIBEXT): | build
@echo " => Building zlib"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zlib/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/openssl/ssl.h build/lib/libssl.$(LIBEXT) build/lib/libcrypto.$(LIBEXT): | build
@echo " => Building openssl"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/openssl install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/apr-1/apr.h build/lib/libapr-1.$(LIBEXT): | build
@echo " => Building apr"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/apr install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build: build:
mkdir $@ mkdir $@
......
VERSION=0.0.30
# By default, all dependencies (zeromq, etc) will be downloaded and installed
# locally. You can change this if you are deploying your own.
VENDOR?=zeromq jemalloc openssl zlib
# Where to install to.
PREFIX?=/opt/lumberjack
FETCH=sh fetch.sh
CFLAGS+=-D_POSIX_C_SOURCE=199309 -std=c99 -Wall -Wextra -Werror -pipe
CFLAGS+=-g
CFLAGS+=-Wno-unused-function
LDFLAGS+=-pthread
LIBS=-lzmq -ljemalloc -lssl -lcrypto -luuid -lz
MAKE?=make
CFLAGS+=-Ibuild/include
LDFLAGS+=-Lbuild/lib -Wl,-rpath,'$$ORIGIN/../lib'
default: build-all
build-all: build/bin/lumberjack build/bin/lumberjack.sh
include Makefile.ext
ifeq ($(UNAME),Linux)
# clock_gettime is in librt on linux.
LIBS+=-lrt
endif
clean:
-@rm -fr lumberjack unixsock *.o build
vendor-clean:
-$(MAKE) -C vendor/msgpack/ clean
-$(MAKE) -C vendor/jansson/ clean
-$(MAKE) -C vendor/jemalloc/ clean
-$(MAKE) -C vendor/libuuid/ clean
-$(MAKE) -C vendor/zeromq/ clean
-$(MAKE) -C vendor/zlib/ clean
-$(MAKE) -C vendor/apr/ clean
rpm deb: | build-all
fpm -s dir -t $@ -n lumberjack -v $(VERSION) --prefix /opt/lumberjack \
--exclude '*.a' --exclude 'lib/pkgconfig/zlib.pc' -C build \
--description "a log shipping tool" \
--url "https://github.com/jordansissel/lumberjack" \
bin/lumberjack bin/lumberjack.sh lib
#install: build/bin/lumberjack build/lib/libzmq.$(LIBEXT)
# install -d -m 755 build/bin/* $(PREFIX)/bin/lumberjack
# install -d build/lib/* $(PREFIX)/lib
flog.o: flog.c flog.h
strlist.o: strlist.h
emitter.o: strlist.h
backoff.o: backoff.c backoff.h
harvester.o: harvester.c harvester.h proto.h str.h sleepdefs.h flog.h
emitter.o: emitter.c emitter.h ring.h sleepdefs.h flog.h
lumberjack.o: lumberjack.c backoff.h harvester.h emitter.h flog.h
str.o: str.c str.h
proto.o: proto.c proto.h str.h sleepdefs.h flog.h
ring.o: ring.c ring.h
harvester.o: build/include/insist.h
lumberjack.o: build/include/insist.h
# Vendor'd dependencies
# If VENDOR contains 'zeromq' download and build it.
ifeq ($(filter zeromq,$(VENDOR)),zeromq)
emitter.o: build/include/zmq.h
harvester.o: build/include/zmq.h
lumberjack.o: build/include/zmq.h
build/bin/lumberjack: | build/bin build/lib/libzmq.$(LIBEXT)
endif # zeromq
ifeq ($(filter jemalloc,$(VENDOR)),jemalloc)
harvester.o lumberjack.o ring.o str.o: build/include/jemalloc/jemalloc.h
build/bin/lumberjack: | build/lib/libjemalloc.$(LIBEXT)
endif # jemalloc
ifeq ($(filter openssl,$(VENDOR)),openssl)
proto.o: build/include/openssl/ssl.h
lumberjack.o: build/include/openssl/ssl.h
build/bin/lumberjack: | build/lib/libssl.$(LIBEXT)
build/bin/lumberjack: | build/lib/libcrypto.$(LIBEXT)
endif # openssl
ifeq ($(filter zlib,$(VENDOR)),zlib)
proto.o: build/include/zlib.h
build/bin/lumberjack: | build/lib/libz.$(LIBEXT)
endif # zlib
.PHONY: test
test: | build/test/test_ring
build/test/test_ring
# Tests
test_ring.o: ring.h build/include/jemalloc/jemalloc.h build/include/insist.h
build/test/test_ring: test_ring.o ring.o | build/test
$(CC) $(LDFLAGS) -o $@ $^ -ljemalloc
build/bin/lumberjack.sh: lumberjack.sh | build/bin
install -m 755 $^ $@
build/bin/lumberjack: | build/bin
build/bin/lumberjack: lumberjack.o backoff.o harvester.o emitter.o str.o proto.o ring.o strlist.o flog.o
$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
@echo " => Build complete: $@"
@echo " => Run '$(MAKE) rpm' to build an rpm (or deb or tarball)"
build/include/insist.h: | build/include
PATH=$$PWD:$$PATH fetch.sh -o $@ https://raw.github.com/jordansissel/experiments/master/c/better-assert/insist.h
build/include/zmq.h build/lib/libzmq.$(LIBEXT): | build
@echo " => Building zeromq"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zeromq/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/msgpack.h build/lib/libmsgpack.$(LIBEXT): | build
@echo " => Building msgpack"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/msgpack/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/jemalloc/jemalloc.h build/lib/libjemalloc.$(LIBEXT): | build
@echo " => Building jemalloc"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/jemalloc/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/sodium/sodium.h build/lib/libsodium.$(LIBEXT): | build
@echo " => Building libsodium"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/libsodium/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/lz4.h build/lib/liblz4.$(LIBEXT): | build
@echo " => Building lz4"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/lz4/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/zlib.h build/lib/libz.$(LIBEXT): | build
@echo " => Building zlib"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/zlib/ install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/openssl/ssl.h build/lib/libssl.$(LIBEXT) build/lib/libcrypto.$(LIBEXT): | build
@echo " => Building openssl"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/openssl install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build/include/apr-1/apr.h build/lib/libapr-1.$(LIBEXT): | build
@echo " => Building apr"
PATH=$$PWD:$$PATH $(MAKE) -C vendor/apr install PREFIX=$$PWD/build DEBUG=$(DEBUG)
build:
mkdir $@
build/include build/bin build/test: | build
mkdir $@
package liblumberjack package liblumberjack
import ( import (
"log" "bytes"
"encoding/json" "encoding/json"
zmq "github.com/alecthomas/gozmq" zmq "github.com/alecthomas/gozmq"
"log"
"math/big" "math/big"
"syscall" "syscall"
"bytes"
"time" "time"
//"syscall" //"syscall"
"compress/zlib" "compress/zlib"
...@@ -14,7 +14,8 @@ import ( ...@@ -14,7 +14,8 @@ import (
"crypto/rand" "crypto/rand"
) )
var context zmq.Context var context *zmq.Context
func init() { func init() {
context, _ = zmq.NewContext() context, _ = zmq.NewContext()
} }
...@@ -31,7 +32,7 @@ type FFS struct { ...@@ -31,7 +32,7 @@ type FFS struct {
RecvTimeout time.Duration RecvTimeout time.Duration
endpoint string // the current endpoint in use endpoint string // the current endpoint in use
socket zmq.Socket // the current zmq socket socket *zmq.Socket // the current zmq socket
connected bool // are we connected? connected bool // are we connected?
} }
...@@ -40,7 +41,7 @@ func (s *FFS) Send(data []byte, flags zmq.SendRecvOption) (err error) { ...@@ -40,7 +41,7 @@ func (s *FFS) Send(data []byte, flags zmq.SendRecvOption) (err error) {
s.ensure_connect() s.ensure_connect()
pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLOUT}} pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLOUT}}
count, err := zmq.Poll(pi, int64(s.SendTimeout.Nanoseconds() / 1000)) count, err := zmq.Poll(pi, s.SendTimeout)
if count == 0 { if count == 0 {
// not ready in time, fail the socket and try again. // not ready in time, fail the socket and try again.
log.Printf("%s: timed out waiting to Send(): %s\n", log.Printf("%s: timed out waiting to Send(): %s\n",
...@@ -66,7 +67,7 @@ func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) { ...@@ -66,7 +67,7 @@ func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) {
s.ensure_connect() s.ensure_connect()
pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLIN}} pi := zmq.PollItems{zmq.PollItem{Socket: s.socket, Events: zmq.POLLIN}}
count, err := zmq.Poll(pi, int64(s.RecvTimeout.Nanoseconds() / 1000)) count, err := zmq.Poll(pi, s.RecvTimeout)
if count == 0 { if count == 0 {
// not ready in time, fail the socket and try again. // not ready in time, fail the socket and try again.
s.fail_socket() s.fail_socket()
...@@ -91,7 +92,9 @@ func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) { ...@@ -91,7 +92,9 @@ func (s *FFS) Recv(flags zmq.SendRecvOption) (data []byte, err error) {
func (s *FFS) Close() (err error) { func (s *FFS) Close() (err error) {
err = s.socket.Close() err = s.socket.Close()
if err != nil { return } if err != nil {
return
}
s.socket = nil s.socket = nil
s.connected = false s.connected = false
...@@ -148,7 +151,9 @@ func (s *FFS) ensure_connect() { ...@@ -148,7 +151,9 @@ func (s *FFS) ensure_connect() {
} }
func (s *FFS) fail_socket() { func (s *FFS) fail_socket() {
if !s.connected { return } if !s.connected {
return
}
s.Close() s.Close()
} }
...@@ -210,5 +215,3 @@ func Publish(input chan []*FileEvent, server_list []string, ...@@ -210,5 +215,3 @@ func Publish(input chan []*FileEvent, server_list []string,
// TODO(sissel): notify registrar of success // TODO(sissel): notify registrar of success
} /* for each event payload */ } /* for each event payload */
} // Publish } // Publish
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment