We first need to create the executable file: src/bin/tdg-analyzer.rs
.
We start by declaring our dependent external crates
extern crate kafka;
extern crate clap;
We then declare the modules we will be using.
use kafka::consumer::{Consumer, FetchOffset, GroupOffsetStorage};
use clap::{Arg, App};
use std::path::Path;
use test_data_generation::{Profile, shared};
Next, we add some supportive functions outside of the main
function.
fn make_consumer(data_topic: String) -> Result<Consumer, kafka::Error> {
Consumer::from_hosts(vec!("localhost:9092".to_owned()))
.with_topic(data_topic.to_owned())
.with_fallback_offset(FetchOffset::Earliest)
.with_group("my-group".to_owned())
.with_offset_storage(GroupOffsetStorage::Kafka)
.create()
}
fn analyse_data(topic: &str, data: &str) -> Result<bool, std::io::Error>{
let profile_file = shared::string_to_static_str(format!("{}/{}", WORKSPACE_LOCAL_STORAGE, topic));
let mut profile = match Path::new(&format!("{}/{}.json", WORKSPACE_LOCAL_STORAGE, topic)).exists() {
true => {
// use existing file
Profile::from_file(&profile_file)
},
false => {
// create new file
println!("Creating new profile: {}",topic);
Profile::new_with_id(topic.to_string())
},
};
profile.analyze(&data);
profile.pre_generate();
profile.save(&profile_file)
}
Finally, we write the main function that will be called.
fn main() {
let matches = App::new("Sample Data Analyzer")
.arg(Arg::with_name("topic")
.help("Specify the Kafak topic to pull data from")
.required(true)
.short("t")
.long("topic")
.takes_value(true))
.get_matches();
// Gets a value for config if supplied by user, or defaults to "default.conf"
let data_topic = matches.value_of("topic").unwrap();
println!("Listening to the {} topic ...", data_topic);
let mut consumer = match make_consumer(data_topic.to_string()) {
Ok(c) => c,
Err(err) => {
panic!("{}",err);
},
};
loop {
for ms in consumer.poll().unwrap().iter() {
for m in ms.messages() {
let data = String::from_utf8(m.value.to_vec()).unwrap();
println!("{}", data);
match analyse_data(data_topic, &data) {
Ok(_) =>{
//do nothing
println!("Analyzed the data.");
},
Err(_err) => {
println!("Unable to analyze the data for the topic {}!", data_topic);
},
}
}
consumer.consume_messageset(ms);
}
consumer.commit_consumed().unwrap();
}
}
When you are finished, the tdg-analyzer.rs
file should look like this:
extern crate kafka;
extern crate clap;
use kafka::consumer::{Consumer, FetchOffset, GroupOffsetStorage};
use clap::{Arg, App};
use std::path::Path;
use test_data_generation::{Profile, shared};
static WORKSPACE_LOCAL_STORAGE: &str = "../profiles";
fn make_consumer(data_topic: String) -> Result<Consumer, kafka::Error> {
Consumer::from_hosts(vec!("localhost:9092".to_owned()))
.with_topic(data_topic.to_owned())
.with_fallback_offset(FetchOffset::Earliest)
.with_group("my-group".to_owned())
.with_offset_storage(GroupOffsetStorage::Kafka)
.create()
}
fn analyse_data(topic: &str, data: &str) -> Result<bool, std::io::Error>{
let profile_file = shared::string_to_static_str(format!("{}/{}", WORKSPACE_LOCAL_STORAGE, topic));
let mut profile = match Path::new(&format!("{}/{}.json", WORKSPACE_LOCAL_STORAGE, topic)).exists() {
true => {
// use existing file
Profile::from_file(&profile_file)
},
false => {
// create new file
println!("Creating new profile: {}",topic);
Profile::new_with_id(topic.to_string())
},
};
profile.analyze(&data);
profile.pre_generate();
profile.save(&profile_file)
}
fn main() {
let matches = App::new("Sample Data Analyzer")
.arg(Arg::with_name("topic")
.help("Specify the Kafak topic to pull data from")
.required(true)
.short("t")
.long("topic")
.takes_value(true))
.get_matches();
// Gets a value for config if supplied by user, or defaults to "default.conf"
let data_topic = matches.value_of("topic").unwrap();
println!("Listening to the {} topic ...", data_topic);
let mut consumer = match make_consumer(data_topic.to_string()) {
Ok(c) => c,
Err(err) => {
panic!("{}",err);
},
};
loop {
for ms in consumer.poll().unwrap().iter() {
for m in ms.messages() {
let data = String::from_utf8(m.value.to_vec()).unwrap();
println!("{}", data);
match analyse_data(data_topic, &data) {
Ok(_) =>{
//do nothing
println!("Analyzed the data.");
},
Err(_err) => {
println!("Unable to analyze the data for the topic {}!", data_topic);
},
}
}
consumer.consume_messageset(ms);
}
consumer.commit_consumed().unwrap();
}
}