Compare commits

...
10 Commits
11 changed files with 155 additions and 51 deletions
+16 -3
View File
@@ -1,11 +1,24 @@
cmake_minimum_required(VERSION 2.8) cmake_minimum_required(VERSION 2.8)
project(syslog-bulk-uploader) project(syslog-bulk-uploader)
set(SLBU_DESC "Syslog Bulk Uploader") set(SLBU_DESC "Syslog Bulk Uploader")
set(SLBU_VER "1.0.0-alpha.1") set(SLBU_VER "1.0.0")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror")
add_definitions(-std=c++11) if(UNIX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror")
add_definitions(-std=c++11)
endif()
if(WIN32)
add_definitions(-DWINVER=0x0601)
add_definitions(-D_WIN32_WINNT=0x0601)
endif()
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_MULTITHREADED ON)
add_subdirectory(src) add_subdirectory(src)
add_subdirectory(test) add_subdirectory(test)
enable_testing() enable_testing()
add_test(NAME SyslogBulkUploaderTests COMMAND SyslogBulkUploaderTests) add_test(NAME SyslogBulkUploaderTests COMMAND SyslogBulkUploaderTests)
add_test(NAME SyslogMessageTests COMMAND SyslogMessageTests) add_test(NAME SyslogMessageTests COMMAND SyslogMessageTests)
+9 -2
View File
@@ -69,9 +69,16 @@ In order to build syslog-bulk-uploader from source, you will need to have a coup
### Building on Windows systems ### Building on Windows systems
Not available yet. Check back soon. After installing and setting up Visual Studio, git, cmake and boost, building is simple:
git clone git@github.com:zmarko/syslog-bulk-uploader.git
cd syslog-bulk-uploader\build
cmake ..
and you will find Visual Studio solution (.sln) file in syslog-bulk-uploader\build directory.
You can use `cmake .. -G "NMake Makefiles"` in order to generate nmake makefile, so you can run build from command line.
### Installers ### Installers
During the build it is also possible to build installer packages. Supported installers at this time are: During the build it is also possible to build installer packages. Supported installers at this time are:
**rpm**, **deb** and plain **.tar.gz** archive. Installers are built by simly running `make package` in `build` **rpm**, **deb** and plain **.tar.gz** archive. Installers are built by simply running `make package` in `build`
sub-directory. sub-directory.
+39 -21
View File
@@ -1,11 +1,16 @@
find_package(Boost COMPONENTS date_time filesystem system thread program_options REQUIRED) set(BOOST_LIBS date_time filesystem system thread program_options)
if(WIN32)
list(APPEND BOOST_LIBS regex chrono)
endif()
find_package(Boost COMPONENTS ${BOOST_LIBS} REQUIRED)
find_package(Threads) find_package(Threads)
configure_file("config.h.in" "config.h") configure_file("config.h.in" "config.h")
include_directories( include_directories(
${PROJECT_BINARY_DIR} ${PROJECT_BINARY_DIR}
${Boost_INLUDE_DIRS} ${Boost_INCLUDE_DIRS}
) )
add_library(slbu-lib add_library(slbu-lib
@@ -29,17 +34,28 @@ target_link_libraries(syslog-bulk-uploader
slbu-lib slbu-lib
) )
if(WIN32)
target_link_libraries(syslog-bulk-uploader
${Boost_REGEX_LIBRARY}
${Boost_CHRONO_LIBRARY}
)
endif()
install(TARGETS syslog-bulk-uploader DESTINATION bin) install(TARGETS syslog-bulk-uploader DESTINATION bin)
## ##
## Installers configuration ## Installers configuration
## ##
include(InstallRequiredSystemLibraries) include(InstallRequiredSystemLibraries)
set(CPACK_GENERATOR "RPM;DEB;TGZ") if(WIN32)
set(CPACK_GENERATOR "ZIP")
else()
set(CPACK_GENERATOR "RPM;DEB;TGZ")
endif()
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${VER_NAME}") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${VER_NAME}")
set(CPACK_PACKAGE_VENDOR "Marko Zivanovic <marko@zivanovic.in.rs>") set(CPACK_PACKAGE_VENDOR "Marko Zivanovic <marko@zivanovic.in.rs>")
set(CPACK_PACKAGE_CONTACT "${CPACK_PACKAGE_VENDOR}") set(CPACK_PACKAGE_CONTACT "${CPACK_PACKAGE_VENDOR}")
set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../README.md") set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/PackageDescription.txt")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Utility to send messages from file to remote syslog server") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Utility to send messages from file to remote syslog server")
set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE")
set(CPACK_PACKAGE_VERSION "${SLBU_VER}") set(CPACK_PACKAGE_VERSION "${SLBU_VER}")
@@ -54,22 +70,24 @@ include(CPack)
## ##
## Generate documentation ## Generate documentation
## ##
find_program(A2X_EXECUTABLE NAMES a2x) if(UNIX)
set(A2X_OPTS --doctype manpage --format manpage -D ${CMAKE_CURRENT_BINARY_DIR}) find_program(A2X_EXECUTABLE NAMES a2x)
set(A2X_OPTS --doctype manpage --format manpage -D ${CMAKE_CURRENT_BINARY_DIR})
set(MAN_NAMES syslog-bulk-uploader.1) set(MAN_NAMES syslog-bulk-uploader.1)
set(MAN_FILES) set(MAN_FILES)
foreach(m IN LISTS MAN_NAMES) foreach(m IN LISTS MAN_NAMES)
set(mf ${CMAKE_CURRENT_BINARY_DIR}/${m}) set(mf ${CMAKE_CURRENT_BINARY_DIR}/${m})
set(ms ${CMAKE_CURRENT_SOURCE_DIR}/${m}.txt) set(ms ${CMAKE_CURRENT_SOURCE_DIR}/${m}.txt)
add_custom_command(OUTPUT ${mf} add_custom_command(OUTPUT ${mf}
COMMAND ${A2X_EXECUTABLE} ${A2X_OPTS} ${ms} COMMAND ${A2X_EXECUTABLE} ${A2X_OPTS} ${ms}
DEPENDS ${ms} DEPENDS ${ms}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Building manpage ${mf}" COMMENT "Building manpage ${mf}"
VERBATIM) VERBATIM)
list(APPEND MAN_FILES ${mf}) list(APPEND MAN_FILES ${mf})
endforeach() endforeach()
add_custom_target(man ALL DEPENDS ${MAN_FILES}) add_custom_target(man ALL DEPENDS ${MAN_FILES})
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/syslog-bulk-uploader.1 DESTINATION man/man1/) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/syslog-bulk-uploader.1 DESTINATION man/man1/)
endif()
+1
View File
@@ -0,0 +1 @@
Syslog-bulk-uploader is non-interactive utility designed to read syslog records from one or more files and send them to remote syslog server.
+4
View File
@@ -34,6 +34,10 @@ void SyslogBulkUploader::run() {
while (auto msg = _reader.nextMessage()) { while (auto msg = _reader.nextMessage()) {
freqLimit.tick(); freqLimit.tick();
if (_preSendCallback)
_preSendCallback(msg);
_writer.sendMessage(msg); _writer.sendMessage(msg);
if (_postSendCallback)
_postSendCallback(msg);
} }
} }
+15
View File
@@ -26,23 +26,38 @@ SOFTWARE.
#define SYSLOGBULKUPLOADER_H #define SYSLOGBULKUPLOADER_H
#include <boost/noncopyable.hpp> #include <boost/noncopyable.hpp>
#include <functional>
#include <memory>
class Reader; class Reader;
class Writer; class Writer;
class SyslogMessage;
class SyslogBulkUploader : boost::noncopyable { class SyslogBulkUploader : boost::noncopyable {
public: public:
typedef std::function<void(std::shared_ptr<const SyslogMessage>)> Callback;
SyslogBulkUploader(Reader& reader, Writer& writer, const size_t& mps = DEFAULT_MPS) : _reader(reader), SyslogBulkUploader(Reader& reader, Writer& writer, const size_t& mps = DEFAULT_MPS) : _reader(reader),
_writer(writer), _mps(mps) { _writer(writer), _mps(mps) {
}; };
void run(); void run();
void setPreSendCallback(Callback cb) {
_preSendCallback = cb;
}
void setPostSendCallback(Callback cb) {
_postSendCallback = cb;
}
private: private:
const static size_t DEFAULT_MPS = 1000; const static size_t DEFAULT_MPS = 1000;
Reader& _reader; Reader& _reader;
Writer& _writer; Writer& _writer;
const size_t _mps; const size_t _mps;
Callback _preSendCallback;
Callback _postSendCallback;
}; };
#endif /* SYSLOGBULKUPLOADER_H */ #endif /* SYSLOGBULKUPLOADER_H */
+18 -4
View File
@@ -33,12 +33,16 @@ SOFTWARE.
namespace po = boost::program_options; namespace po = boost::program_options;
using namespace std; using namespace std;
using namespace boost::posix_time;
po::options_description desc("Supported options"); po::options_description desc("Supported options");
size_t mps; size_t mps;
string dest; string dest;
uint16_t port; uint16_t port;
vector<string> files; vector<string> files;
ptime start;
ptime lastPrint;
time_duration printInterval = seconds(1);
string version() { string version() {
stringstream ss; stringstream ss;
@@ -50,6 +54,14 @@ void help() {
desc.print(cout); desc.print(cout);
} }
void preSendCallback(shared_ptr<const SyslogMessage>) {
ptime now = second_clock::local_time();
if (now - lastPrint > printInterval) {
cout << "." << flush;
lastPrint = now;
}
}
int main(int argc, char** argv) { int main(int argc, char** argv) {
desc.add_options() desc.add_options()
("help,h", "display help message") ("help,h", "display help message")
@@ -60,7 +72,7 @@ int main(int argc, char** argv) {
("file,f", po::value<vector < string >> (&files), "input file(s)") ("file,f", po::value<vector < string >> (&files), "input file(s)")
; ;
po::positional_options_description pos; po::positional_options_description pos;
pos.add("files", -1); pos.add("file", -1);
po::variables_map vars; po::variables_map vars;
try { try {
@@ -87,7 +99,7 @@ int main(int argc, char** argv) {
return -1; return -1;
} }
if (!vars.count("files")) { if (!vars.count("file")) {
cout << "ERROR: You must specify at least one input file" << endl; cout << "ERROR: You must specify at least one input file" << endl;
help(); help();
return -1; return -1;
@@ -95,14 +107,16 @@ int main(int argc, char** argv) {
UDPWriter w(dest, port); UDPWriter w(dest, port);
lastPrint = start = second_clock::local_time();
for (auto& file : files) { for (auto& file : files) {
try { try {
FileReader r(file); FileReader r(file);
SyslogBulkUploader uploader(r, w, mps); SyslogBulkUploader uploader(r, w, mps);
uploader.setPostSendCallback(bind(preSendCallback, placeholders::_1));
cout << "Sending logs from " << file << " to udp://" << dest << ":" << port << " at a max rate of " << cout << "Sending logs from " << file << " to udp://" << dest << ":" << port << " at a max rate of " <<
mps << " messages per second ... " << flush; mps << " messages per second " << flush;
uploader.run(); uploader.run();
cout << "done" << endl; cout << " - DONE" << endl;
} catch (string& ex) { } catch (string& ex) {
cout << ex << endl; cout << ex << endl;
} catch (exception& ex) { } catch (exception& ex) {
+41 -11
View File
@@ -1,42 +1,72 @@
find_package(Boost COMPONENTS unit_test_framework date_time filesystem system thread REQUIRED) set(BOOST_LIBS unit_test_framework date_time filesystem system thread)
if(WIN32)
list(APPEND BOOST_LIBS regex chrono)
endif()
find_package(Boost COMPONENTS ${BOOST_LIBS} REQUIRED)
find_package(Threads) find_package(Threads)
include_directories( include_directories(
${TEST_SOURCE_DIR/src} ${TEST_SOURCE_DIR/src}
${Boost_INLUDE_DIRS} ${Boost_INCLUDE_DIRS}
) )
add_definitions(-DBOOST_TEST_DYN_LINK) #add_definitions(-DBOOST_TEST_DYN_LINK)
add_executable(SyslogBulkUploaderTests SyslogBulkUploaderTests.cpp) add_executable(SyslogBulkUploaderTests SyslogBulkUploaderTests.cpp)
target_link_libraries(SyslogBulkUploaderTests target_link_libraries(SyslogBulkUploaderTests
slbu-lib slbu-lib
${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
${Boost_DATE_TIME_LIBRARY} ${Boost_DATE_TIME_LIBRARY}
${Boost_FILESYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY}
${CMAKE_THREAD_LIBS_INIT} ${CMAKE_THREAD_LIBS_INIT}
${Boost_THREAD_LIBRARY} ${Boost_THREAD_LIBRARY}
) )
if(WIN32)
target_link_libraries(SyslogBulkUploaderTests
${Boost_REGEX_LIBRARY}
${Boost_CHRONO_LIBRARY}
)
endif()
add_executable(SyslogMessageTests SyslogMessageTests.cpp) add_executable(SyslogMessageTests SyslogMessageTests.cpp)
target_link_libraries(SyslogMessageTests target_link_libraries(SyslogMessageTests
slbu-lib slbu-lib
${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
${Boost_DATE_TIME_LIBRARY} ${Boost_DATE_TIME_LIBRARY}
) )
if(WIN32)
target_link_libraries(SyslogMessageTests
${Boost_REGEX_LIBRARY}
${Boost_CHRONO_LIBRARY}
)
endif()
add_executable(RFC3164FormattedSyslogMessageTests RFC3164FormattedSyslogMessageTests.cpp) add_executable(RFC3164FormattedSyslogMessageTests RFC3164FormattedSyslogMessageTests.cpp)
target_link_libraries(RFC3164FormattedSyslogMessageTests target_link_libraries(RFC3164FormattedSyslogMessageTests
slbu-lib slbu-lib
${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
${Boost_DATE_TIME_LIBRARY} ${Boost_DATE_TIME_LIBRARY}
) )
if(WIN32)
target_link_libraries(RFC3164FormattedSyslogMessageTests
${Boost_REGEX_LIBRARY}
${Boost_CHRONO_LIBRARY}
)
endif()
add_executable(FrequencyLimitTests FrequencyLimitTests.cpp) add_executable(FrequencyLimitTests FrequencyLimitTests.cpp)
target_link_libraries(FrequencyLimitTests target_link_libraries(FrequencyLimitTests
slbu-lib slbu-lib
${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
${Boost_DATE_TIME_LIBRARY} ${Boost_DATE_TIME_LIBRARY}
${Boost_THREAD_LIBRARY} ${Boost_THREAD_LIBRARY}
${Boost_SYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY}
${CMAKE_THREAD_LIBS_INIT}
) )
if(WIN32)
target_link_libraries(FrequencyLimitTests
${Boost_REGEX_LIBRARY}
${Boost_CHRONO_LIBRARY}
)
endif()
+2 -2
View File
@@ -42,10 +42,10 @@ size_t countInLoop(size_t freq, time_duration duration) {
BOOST_AUTO_TEST_CASE(test_slow) { BOOST_AUTO_TEST_CASE(test_slow) {
size_t c = countInLoop(5, seconds(5)); size_t c = countInLoop(5, seconds(5));
BOOST_WARN_CLOSE((float) c, 25, 10); BOOST_WARN_CLOSE((float) c, 25., 10.);
} }
BOOST_AUTO_TEST_CASE(test_fast) { BOOST_AUTO_TEST_CASE(test_fast) {
size_t c = countInLoop(100, seconds(5)); size_t c = countInLoop(100, seconds(5));
BOOST_WARN_CLOSE((float) c, 500, 10); BOOST_WARN_CLOSE((float) c, 500., 10.);
} }
+8 -7
View File
@@ -22,6 +22,13 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. SOFTWARE.
*/ */
#define BOOST_TEST_MODULE SyslogBulkUploaderTests
#include <boost/test/unit_test.hpp>
#include <memory>
#define BOOST_FILESYSTEM_NO_DEPRECATED
#include <boost/filesystem.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include "../src/SyslogBulkUploader.h" #include "../src/SyslogBulkUploader.h"
#include "../src/Reader.h" #include "../src/Reader.h"
#include "../src/Writer.h" #include "../src/Writer.h"
@@ -29,12 +36,6 @@ SOFTWARE.
#include "../src/UDPWriter.h" #include "../src/UDPWriter.h"
#include "UdpSyslogServer.h" #include "UdpSyslogServer.h"
#include "../src/RFC3164FormattedSyslogMessage.h" #include "../src/RFC3164FormattedSyslogMessage.h"
#define BOOST_TEST_MODULE SyslogBulkUploaderTests
#include <boost/test/unit_test.hpp>
#include <memory>
#define BOOST_FILESYSTEM_NO_DEPRECATED
#include <boost/filesystem.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
class MockReader : public Reader { class MockReader : public Reader {
public: public:
@@ -89,7 +90,7 @@ BOOST_AUTO_TEST_CASE(test_udp_writer) {
ptime start(second_clock::local_time()); ptime start(second_clock::local_time());
boost::asio::io_service ios; boost::asio::io_service ios;
FileReader r("../test/sample1"); FileReader r("../test/sample1");
UDPWriter w("localhost", 51514); UDPWriter w("127.0.0.1", 51514);
SyslogBulkUploader ul(r, w); SyslogBulkUploader ul(r, w);
UdpSyslogServer server(ios, 51514, 2000); UdpSyslogServer server(ios, 51514, 2000);
ul.run(); ul.run();
+2 -1
View File
@@ -71,7 +71,8 @@ private:
} }
void deadlineHandler() { void deadlineHandler() {
if (_dt.expires_at() <= deadline_timer::traits_type::now()) { auto delta = deadline_timer::traits_type::now() - _dt.expires_at();
if (delta != not_a_date_time && delta >= milliseconds(0)) {
_socket.cancel(); _socket.cancel();
_dt.expires_at(boost::posix_time::pos_infin); _dt.expires_at(boost::posix_time::pos_infin);
} }